From 690b3410e9b11fa37c0d985d26c398be888767e4 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sat, 25 Jan 2025 13:47:10 +0100 Subject: [PATCH 01/19] Interfaces for poTokens + WebView implementation --- app/build.gradle | 4 +- app/src/main/assets/po_token.html | 211 +++++++++++++ app/src/main/java/org/schabi/newpipe/App.java | 32 ++ .../newpipe/util/potoken/PoTokenException.kt | 3 + .../newpipe/util/potoken/PoTokenGenerator.kt | 26 ++ .../newpipe/util/potoken/PoTokenWebView.kt | 297 ++++++++++++++++++ 6 files changed, 571 insertions(+), 2 deletions(-) create mode 100644 app/src/main/assets/po_token.html create mode 100644 app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt create mode 100644 app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt create mode 100644 app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt diff --git a/app/build.gradle b/app/build.gradle index 8d9981271..1a3dc7535 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -185,7 +185,7 @@ afterEvaluate { if (!System.properties.containsKey('skipFormatKtlint')) { preDebugBuild.dependsOn formatKtlint } - preDebugBuild.dependsOn runCheckstyle, runKtlint + //preDebugBuild.dependsOn runCheckstyle, runKtlint } sonar { @@ -208,7 +208,7 @@ dependencies { implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' // WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with // the corresponding commit hash, since JitPack is sometimes buggy - implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.24.4' + implementation 'com.github.FireMasterK:NewPipeExtractor:5528d5c31b400aac8e8930fef16f7b981b5cc0a4' implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0' /** Checkstyle **/ diff --git a/app/src/main/assets/po_token.html b/app/src/main/assets/po_token.html new file mode 100644 index 000000000..572c3016e --- /dev/null +++ b/app/src/main/assets/po_token.html @@ -0,0 +1,211 @@ + + diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 9bc25d55d..74f2c687b 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -3,7 +3,15 @@ package org.schabi.newpipe; import android.app.Application; import android.content.Context; import android.content.SharedPreferences; +import android.os.Build; +import android.os.UserManager; import android.util.Log; +import android.view.View; +import android.webkit.JavascriptInterface; +import android.webkit.WebSettings; +import android.webkit.WebView; +import android.webkit.WebViewClient; +import android.widget.RelativeLayout; import androidx.annotation.NonNull; import androidx.core.app.NotificationChannelCompat; @@ -17,6 +25,8 @@ import org.acra.config.CoreConfigurationBuilder; import org.schabi.newpipe.error.ReCaptchaActivity; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.ktx.ExceptionUtils; import org.schabi.newpipe.settings.NewPipeSettings; import org.schabi.newpipe.util.BridgeStateSaverInitializer; @@ -26,6 +36,7 @@ import org.schabi.newpipe.util.StateSaver; import org.schabi.newpipe.util.image.ImageStrategy; import org.schabi.newpipe.util.image.PicassoHelper; import org.schabi.newpipe.util.image.PreferredImageQuality; +import org.schabi.newpipe.util.potoken.PoTokenWebView; import java.io.IOException; import java.io.InterruptedIOException; @@ -33,12 +44,16 @@ import java.net.SocketException; import java.util.List; import java.util.Objects; +import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; +import io.reactivex.rxjava3.core.Single; +import io.reactivex.rxjava3.disposables.CompositeDisposable; import io.reactivex.rxjava3.exceptions.CompositeException; import io.reactivex.rxjava3.exceptions.MissingBackpressureException; import io.reactivex.rxjava3.exceptions.OnErrorNotImplementedException; import io.reactivex.rxjava3.exceptions.UndeliverableException; import io.reactivex.rxjava3.functions.Consumer; import io.reactivex.rxjava3.plugins.RxJavaPlugins; +import kotlin.Pair; /* * Copyright (C) Hans-Christoph Steiner 2016 @@ -118,6 +133,23 @@ public class App extends Application { && prefs.getBoolean(getString(R.string.show_image_indicators_key), false)); configureRxJavaErrorHandler(); + + CompositeDisposable disposable = new CompositeDisposable(); + disposable.add(PoTokenWebView.Companion.newPoTokenGenerator(this) + .observeOn(AndroidSchedulers.mainThread()) + .subscribeOn(AndroidSchedulers.mainThread()) + .flatMap(poTokenGenerator -> Single.zip( + poTokenGenerator.generatePoToken(YoutubeParsingHelper + .randomVisitorData(NewPipe.getPreferredContentCountry())), + poTokenGenerator.generatePoToken("i_SsnRdgitA"), + Pair::new + )) + .subscribe( + pots -> Log.e(TAG, "success! " + pots.getSecond().poToken + + ",web.gvs+" + pots.getFirst().poToken + + ";visitor_data=" + pots.getFirst().visitorData), + error -> Log.e(TAG, "error", error) + )); } @Override diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt new file mode 100644 index 000000000..896c53a68 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt @@ -0,0 +1,3 @@ +package org.schabi.newpipe.util.potoken + +class PoTokenException(message: String) : Exception(message) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt new file mode 100644 index 000000000..c48afc5c7 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt @@ -0,0 +1,26 @@ +package org.schabi.newpipe.util.potoken + +import android.content.Context +import io.reactivex.rxjava3.core.Single +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import java.io.Closeable + +interface PoTokenGenerator : Closeable { + /** + * Generates a poToken for the provided identifier, using the `integrityToken` and + * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be + * called multiple times. + */ + fun generatePoToken(identifier: String): Single + + interface Factory { + /** + * Initializes a [PoTokenGenerator] by loading the BotGuard VM, running it, and obtaining + * an `integrityToken`. Can then be used multiple times to generate multiple poTokens with + * [generatePoToken]. + * + * @param context used e.g. to load the HTML asset or to instantiate a WebView + */ + fun newPoTokenGenerator(context: Context): Single + } +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt new file mode 100644 index 000000000..626ebea45 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -0,0 +1,297 @@ +package org.schabi.newpipe.util.potoken + +import android.content.Context +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.webkit.JavascriptInterface +import android.webkit.WebView +import androidx.annotation.MainThread +import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers +import io.reactivex.rxjava3.core.Single +import io.reactivex.rxjava3.core.SingleEmitter +import io.reactivex.rxjava3.disposables.CompositeDisposable +import io.reactivex.rxjava3.schedulers.Schedulers +import org.schabi.newpipe.DownloaderImpl +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult + +class PoTokenWebView private constructor( + context: Context, + // to be used exactly once only during initialization! + private val generatorEmitter: SingleEmitter, +) : PoTokenGenerator { + private val webView = WebView(context) + private val disposables = CompositeDisposable() // used only during initialization + private val poTokenEmitters = mutableListOf>>() + + //region Initialization + init { + val webviewSettings = webView.settings + //noinspection SetJavaScriptEnabled we want to use JavaScript! + webviewSettings.javaScriptEnabled = true + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + webviewSettings.safeBrowsingEnabled = false + } + webviewSettings.userAgentString = USER_AGENT + webviewSettings.blockNetworkLoads = true // the WebView does not need internet access + + // so that we can run async functions and get back the result + webView.addJavascriptInterface(this, "PoTokenWebView") + } + + /** + * Must be called right after instantiating [PoTokenWebView] to perform the actual + * initialization. This will asynchronously go through all the steps needed to load BotGuard, + * run it, and obtain an `integrityToken`. + */ + private fun loadHtmlAndObtainBotguard(context: Context) { + disposables.add( + Single.fromCallable { + val html = context.assets.open("po_token.html").bufferedReader() + .use { it.readText() } + return@fromCallable html + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe( + { html -> + webView.loadDataWithBaseURL( + "https://www.youtube.com", + html, + "text/html", + "utf-8", + null, + ) + downloadAndRunBotguard() + }, + this::onInitializationErrorCloseAndCancel + ) + ) + } + + /** + * Called during initialization after the WebView content has been loaded. + */ + private fun downloadAndRunBotguard() { + makeJnnPaGoogleapisRequest( + "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/Create", + "[ \"$REQUEST_KEY\" ]", + ) { responseBody -> + webView.evaluateJavascript( + """(async function() { + try { + data = JSON.parse(String.raw`$responseBody`) + result = await runBotGuard(data) + globalThis.webPoSignalOutput = result.webPoSignalOutput + PoTokenWebView.onRunBotguardResult(result.botguardResponse) + } catch (error) { + PoTokenWebView.onJsInitializationError(error.toString()) + } + })();""", + ) {} + } + } + + /** + * Called during initialization by the JavaScript snippets from either + * [downloadAndRunBotguard] or [onRunBotguardResult]. + */ + @JavascriptInterface + fun onJsInitializationError(error: String) { + Log.e(TAG, "Initialization error from JavaScript: $error") + onInitializationErrorCloseAndCancel(PoTokenException(error)) + } + + /** + * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after + * obtaining the BotGuard execution output [botguardResponse]. + */ + @JavascriptInterface + fun onRunBotguardResult(botguardResponse: String) { + Log.e(TAG, "botguardResponse: $botguardResponse") + makeJnnPaGoogleapisRequest( + "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT", + "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", + ) { responseBody -> + webView.evaluateJavascript( + """(async function() { + try { + globalThis.integrityToken = JSON.parse(String.raw`$responseBody`) + PoTokenWebView.onInitializationFinished() + } catch (error) { + PoTokenWebView.onJsInitializationError(error.toString()) + } + })();""", + ) {} + } + } + + /** + * Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the + * `integrityToken` has been received by JavaScript. + */ + @JavascriptInterface + fun onInitializationFinished() { + generatorEmitter.onSuccess(this) + } + //endregion + + //region Obtaining poTokens + /** + * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that + * multiple poToken requests can be generated invparallel, and the results will be notified to + * the right emitters. + */ + private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { + synchronized(poTokenEmitters) { + poTokenEmitters.add(Pair(identifier, emitter)) + } + } + + /** + * Extracts and removes from the [poTokenEmitters] list a [SingleEmitter] based on its + * [identifier]. The emitter is supposed to be used immediately after to either signal a success + * or an error. + */ + private fun popPoTokenEmitter(identifier: String): SingleEmitter? { + return synchronized(poTokenEmitters) { + poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let { + poTokenEmitters.removeAt(it).second + } + } + } + + @MainThread + override fun generatePoToken(identifier: String): Single = + Single.create { emitter -> + addPoTokenEmitter(identifier, emitter) + + webView.evaluateJavascript( + """(async function() { + identifier = String.raw`$identifier` + try { + poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier) + PoTokenWebView.onObtainPoTokenResult(identifier, poToken) + } catch (error) { + PoTokenWebView.onObtainPoTokenError(identifier, error.toString()) + } + })();""", + ) {} + } + + /** + * Called by the JavaScript snippet from [generatePoToken] when an error occurs in calling the + * JavaScript `obtainPoToken()` function. + */ + @JavascriptInterface + fun onObtainPoTokenError(identifier: String, error: String) { + Log.e(TAG, "obtainPoToken error from JavaScript: $error") + popPoTokenEmitter(identifier)?.onError(PoTokenException(error)) + } + + /** + * Called by the JavaScript snippet from [generatePoToken] with the original identifier and the + * result of the JavaScript `obtainPoToken()` function. + */ + @JavascriptInterface + fun onObtainPoTokenResult(identifier: String, poToken: String) { + Log.e(TAG, "identifier=$identifier") + Log.e(TAG, "poToken=$poToken") + popPoTokenEmitter(identifier)?.onSuccess(PoTokenResult(identifier, poToken)) + } + //endregion + + //region Utils + /** + * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls + * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response + * does not have HTTP code 200, therefore this is supposed to be used only during + * initialization. Calls [handleResponseBody] with the response body if the response is + * successful. The request is performed in the background and a disposable is added to + * [disposables]. + */ + private fun makeJnnPaGoogleapisRequest( + url: String, + data: String, + handleResponseBody: (String) -> Unit, + ) { + disposables.add( + Single.fromCallable { + return@fromCallable DownloaderImpl.getInstance().post( + url, + mapOf( + // replace the downloader user agent + "User-Agent" to listOf(USER_AGENT), + "Accept" to listOf("application/json"), + "Content-Type" to listOf("application/json+protobuf"), + "x-goog-api-key" to listOf(GOOGLE_API_KEY), + "x-user-agent" to listOf("grpc-web-javascript/0.1"), + ), + data.toByteArray() + ) + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe( + { response -> + val httpCode = response.responseCode() + if (httpCode != 200) { + onInitializationErrorCloseAndCancel( + PoTokenException("Invalid response code: $httpCode") + ) + return@subscribe + } + val responseBody = response.responseBody() + handleResponseBody(responseBody) + }, + this::onInitializationErrorCloseAndCancel + ) + ) + } + + /** + * Handles any error happening during initialization, releasing resources and sending the error + * to [generatorEmitter]. + */ + private fun onInitializationErrorCloseAndCancel(error: Throwable) { + Handler(Looper.getMainLooper()).post { + close() + generatorEmitter.onError(error) + } + } + + /** + * Releases all [webView] and [disposables] resources. + */ + override fun close() { + disposables.dispose() + + webView.clearHistory() + // clears RAM cache and disk cache (globally for all WebViews) + webView.clearCache(true) + + // ensures that the WebView isn't doing anything when destroying it + webView.loadUrl("about:blank") + + webView.onPause() + webView.removeAllViews() + webView.destroy() + } + //endregion + + companion object : PoTokenGenerator.Factory { + private val TAG = PoTokenWebView::class.simpleName + private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" + private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" + private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + + @MainThread + override fun newPoTokenGenerator(context: Context): Single = + Single.create { emitter -> + val potWv = PoTokenWebView(context, emitter) + potWv.loadHtmlAndObtainBotguard(context) + emitter.setDisposable(potWv.disposables) + } + } +} From 6010c4ea7fd1efa85a2741f5b0230a43759e3a12 Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 26 Jan 2025 12:28:09 +0100 Subject: [PATCH 02/19] Connect poToken generation to extractor --- app/src/main/java/org/schabi/newpipe/App.java | 34 +--------- .../org/schabi/newpipe/util/DeviceUtils.java | 14 ++++ .../newpipe/util/potoken/PoTokenGenerator.kt | 13 +++- .../util/potoken/PoTokenProviderImpl.kt | 59 +++++++++++++++++ .../newpipe/util/potoken/PoTokenWebView.kt | 64 +++++++++++-------- 5 files changed, 125 insertions(+), 59 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 74f2c687b..8ce161eec 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -3,15 +3,7 @@ package org.schabi.newpipe; import android.app.Application; import android.content.Context; import android.content.SharedPreferences; -import android.os.Build; -import android.os.UserManager; import android.util.Log; -import android.view.View; -import android.webkit.JavascriptInterface; -import android.webkit.WebSettings; -import android.webkit.WebView; -import android.webkit.WebViewClient; -import android.widget.RelativeLayout; import androidx.annotation.NonNull; import androidx.core.app.NotificationChannelCompat; @@ -25,8 +17,7 @@ import org.acra.config.CoreConfigurationBuilder; import org.schabi.newpipe.error.ReCaptchaActivity; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.ktx.ExceptionUtils; import org.schabi.newpipe.settings.NewPipeSettings; import org.schabi.newpipe.util.BridgeStateSaverInitializer; @@ -36,7 +27,7 @@ import org.schabi.newpipe.util.StateSaver; import org.schabi.newpipe.util.image.ImageStrategy; import org.schabi.newpipe.util.image.PicassoHelper; import org.schabi.newpipe.util.image.PreferredImageQuality; -import org.schabi.newpipe.util.potoken.PoTokenWebView; +import org.schabi.newpipe.util.potoken.PoTokenProviderImpl; import java.io.IOException; import java.io.InterruptedIOException; @@ -44,16 +35,12 @@ import java.net.SocketException; import java.util.List; import java.util.Objects; -import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; -import io.reactivex.rxjava3.core.Single; -import io.reactivex.rxjava3.disposables.CompositeDisposable; import io.reactivex.rxjava3.exceptions.CompositeException; import io.reactivex.rxjava3.exceptions.MissingBackpressureException; import io.reactivex.rxjava3.exceptions.OnErrorNotImplementedException; import io.reactivex.rxjava3.exceptions.UndeliverableException; import io.reactivex.rxjava3.functions.Consumer; import io.reactivex.rxjava3.plugins.RxJavaPlugins; -import kotlin.Pair; /* * Copyright (C) Hans-Christoph Steiner 2016 @@ -134,22 +121,7 @@ public class App extends Application { configureRxJavaErrorHandler(); - CompositeDisposable disposable = new CompositeDisposable(); - disposable.add(PoTokenWebView.Companion.newPoTokenGenerator(this) - .observeOn(AndroidSchedulers.mainThread()) - .subscribeOn(AndroidSchedulers.mainThread()) - .flatMap(poTokenGenerator -> Single.zip( - poTokenGenerator.generatePoToken(YoutubeParsingHelper - .randomVisitorData(NewPipe.getPreferredContentCountry())), - poTokenGenerator.generatePoToken("i_SsnRdgitA"), - Pair::new - )) - .subscribe( - pots -> Log.e(TAG, "success! " + pots.getSecond().poToken + - ",web.gvs+" + pots.getFirst().poToken + - ";visitor_data=" + pots.getFirst().visitorData), - error -> Log.e(TAG, "error", error) - )); + YoutubeStreamExtractor.setPoTokenProvider(PoTokenProviderImpl.INSTANCE); } @Override diff --git a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java index e9678c2b0..a037e48d3 100644 --- a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java +++ b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java @@ -17,6 +17,7 @@ import android.view.InputDevice; import android.view.KeyEvent; import android.view.WindowInsets; import android.view.WindowManager; +import android.webkit.CookieManager; import androidx.annotation.Dimension; import androidx.annotation.NonNull; @@ -335,4 +336,17 @@ public final class DeviceUtils { && !TX_50JXW834 && !HMB9213NW; } + + /** + * @return whether the device has support for WebView, see + * https://stackoverflow.com/a/69626735 + */ + public static boolean supportsWebView() { + try { + CookieManager.getInstance(); + return true; + } catch (Throwable ignored) { + return false; + } + } } diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt index c48afc5c7..6446ecc72 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt @@ -2,16 +2,25 @@ package org.schabi.newpipe.util.potoken import android.content.Context import io.reactivex.rxjava3.core.Single -import org.schabi.newpipe.extractor.services.youtube.PoTokenResult import java.io.Closeable +/** + * This interface was created to allow for multiple methods to generate poTokens in the future (e.g. + * via WebView and via a local DOM implementation) + */ interface PoTokenGenerator : Closeable { /** * Generates a poToken for the provided identifier, using the `integrityToken` and * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be * called multiple times. */ - fun generatePoToken(identifier: String): Single + fun generatePoToken(identifier: String): Single + + /** + * @return whether the `integrityToken` is expired, in which case all tokens generated by + * [generatePoToken] will be invalid + */ + fun isExpired(): Boolean interface Factory { /** diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt new file mode 100644 index 000000000..a00e19f2d --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -0,0 +1,59 @@ +package org.schabi.newpipe.util.potoken + +import android.util.Log +import org.schabi.newpipe.App +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper +import org.schabi.newpipe.util.DeviceUtils + +object PoTokenProviderImpl : PoTokenProvider { + val TAG = PoTokenProviderImpl::class.simpleName + private val webViewSupported by lazy { DeviceUtils.supportsWebView() } + + private object WebPoTokenGenLock + private var webPoTokenVisitorData: String? = null + private var webPoTokenStreamingPot: String? = null + private var webPoTokenGenerator: PoTokenGenerator? = null + + override fun getWebClientPoToken(videoId: String): PoTokenResult? { + if (!webViewSupported) { + return null + } + + val (poTokenGenerator, visitorData, streamingPot) = synchronized(WebPoTokenGenLock) { + if (webPoTokenGenerator == null || webPoTokenGenerator!!.isExpired()) { + webPoTokenGenerator = PoTokenWebView.newPoTokenGenerator(App.getApp()).blockingGet() + webPoTokenVisitorData = YoutubeParsingHelper + .randomVisitorData(NewPipe.getPreferredContentCountry()) + + // The streaming poToken needs to be generated exactly once before generating any + // other (player) tokens. + webPoTokenStreamingPot = webPoTokenGenerator!! + .generatePoToken(webPoTokenVisitorData!!).blockingGet() + } + return@synchronized Triple( + webPoTokenGenerator!!, webPoTokenVisitorData!!, webPoTokenStreamingPot!! + ) + } + + // Not using synchronized here, since poTokenGenerator would be able to generate multiple + // poTokens in parallel if needed. The only important thing is for exactly one + // visitorData/streaming poToken to be generated before anything else. + val playerPot = poTokenGenerator.generatePoToken(videoId).blockingGet() + Log.e(TAG, "success($videoId) $playerPot,web.gvs+$streamingPot;visitor_data=$visitorData") + + return PoTokenResult( + webPoTokenVisitorData!!, + playerPot, + webPoTokenStreamingPot!!, + ) + } + + override fun getWebEmbedClientPoToken(videoId: String): PoTokenResult? = null + + override fun getAndroidClientPoToken(videoId: String): PoTokenResult? = null + + override fun getIosClientPoToken(videoId: String): PoTokenResult? = null +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 626ebea45..008477790 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -7,14 +7,13 @@ import android.os.Looper import android.util.Log import android.webkit.JavascriptInterface import android.webkit.WebView -import androidx.annotation.MainThread import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.core.SingleEmitter import io.reactivex.rxjava3.disposables.CompositeDisposable import io.reactivex.rxjava3.schedulers.Schedulers import org.schabi.newpipe.DownloaderImpl -import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import java.time.Instant class PoTokenWebView private constructor( context: Context, @@ -23,7 +22,8 @@ class PoTokenWebView private constructor( ) : PoTokenGenerator { private val webView = WebView(context) private val disposables = CompositeDisposable() // used only during initialization - private val poTokenEmitters = mutableListOf>>() + private val poTokenEmitters = mutableListOf>>() + private lateinit var expirationInstant: Instant //region Initialization init { @@ -114,11 +114,12 @@ class PoTokenWebView private constructor( "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT", "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", ) { responseBody -> + Log.e(TAG, "GenerateIT response: $responseBody") webView.evaluateJavascript( """(async function() { try { globalThis.integrityToken = JSON.parse(String.raw`$responseBody`) - PoTokenWebView.onInitializationFinished() + PoTokenWebView.onInitializationFinished(integrityToken[1]) } catch (error) { PoTokenWebView.onJsInitializationError(error.toString()) } @@ -130,9 +131,14 @@ class PoTokenWebView private constructor( /** * Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the * `integrityToken` has been received by JavaScript. + * + * @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found + * in `integrityToken[1]` */ @JavascriptInterface - fun onInitializationFinished() { + fun onInitializationFinished(expirationTimeInSeconds: Long) { + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) generatorEmitter.onSuccess(this) } //endregion @@ -143,7 +149,7 @@ class PoTokenWebView private constructor( * multiple poToken requests can be generated invparallel, and the results will be notified to * the right emitters. */ - private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { + private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { synchronized(poTokenEmitters) { poTokenEmitters.add(Pair(identifier, emitter)) } @@ -154,7 +160,7 @@ class PoTokenWebView private constructor( * [identifier]. The emitter is supposed to be used immediately after to either signal a success * or an error. */ - private fun popPoTokenEmitter(identifier: String): SingleEmitter? { + private fun popPoTokenEmitter(identifier: String): SingleEmitter? { return synchronized(poTokenEmitters) { poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let { poTokenEmitters.removeAt(it).second @@ -162,22 +168,23 @@ class PoTokenWebView private constructor( } } - @MainThread - override fun generatePoToken(identifier: String): Single = + override fun generatePoToken(identifier: String): Single = Single.create { emitter -> addPoTokenEmitter(identifier, emitter) - - webView.evaluateJavascript( - """(async function() { - identifier = String.raw`$identifier` - try { - poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier) - PoTokenWebView.onObtainPoTokenResult(identifier, poToken) - } catch (error) { - PoTokenWebView.onObtainPoTokenError(identifier, error.toString()) - } - })();""", - ) {} + Handler(Looper.getMainLooper()).post { + webView.evaluateJavascript( + """(async function() { + identifier = String.raw`$identifier` + try { + poToken = await obtainPoToken(webPoSignalOutput, integrityToken, + identifier) + PoTokenWebView.onObtainPoTokenResult(identifier, poToken) + } catch (error) { + PoTokenWebView.onObtainPoTokenError(identifier, error.toString()) + } + })();""", + ) {} + } } /** @@ -198,7 +205,11 @@ class PoTokenWebView private constructor( fun onObtainPoTokenResult(identifier: String, poToken: String) { Log.e(TAG, "identifier=$identifier") Log.e(TAG, "poToken=$poToken") - popPoTokenEmitter(identifier)?.onSuccess(PoTokenResult(identifier, poToken)) + popPoTokenEmitter(identifier)?.onSuccess(poToken) + } + + override fun isExpired(): Boolean { + return Instant.now().isAfter(expirationInstant) } //endregion @@ -286,12 +297,13 @@ class PoTokenWebView private constructor( private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" - @MainThread override fun newPoTokenGenerator(context: Context): Single = Single.create { emitter -> - val potWv = PoTokenWebView(context, emitter) - potWv.loadHtmlAndObtainBotguard(context) - emitter.setDisposable(potWv.disposables) + Handler(Looper.getMainLooper()).post { + val potWv = PoTokenWebView(context, emitter) + potWv.loadHtmlAndObtainBotguard(context) + emitter.setDisposable(potWv.disposables) + } } } } From 3bdae81c0a1358a7440004d054d49e17d60c4b2d Mon Sep 17 00:00:00 2001 From: Stypox Date: Sun, 26 Jan 2025 16:21:17 +0100 Subject: [PATCH 03/19] Fix checkstyle --- app/build.gradle | 4 ++-- app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 1a3dc7535..8b181c94b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -185,7 +185,7 @@ afterEvaluate { if (!System.properties.containsKey('skipFormatKtlint')) { preDebugBuild.dependsOn formatKtlint } - //preDebugBuild.dependsOn runCheckstyle, runKtlint + preDebugBuild.dependsOn runCheckstyle, runKtlint } sonar { @@ -208,7 +208,7 @@ dependencies { implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' // WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with // the corresponding commit hash, since JitPack is sometimes buggy - implementation 'com.github.FireMasterK:NewPipeExtractor:5528d5c31b400aac8e8930fef16f7b981b5cc0a4' + implementation 'com.github.FireMasterK:NewPipeExtractor:d2cbd09089e8af933738f98b671ad58236a79d6e' implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0' /** Checkstyle **/ diff --git a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java index a037e48d3..9eb9fab37 100644 --- a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java +++ b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java @@ -345,7 +345,7 @@ public final class DeviceUtils { try { CookieManager.getInstance(); return true; - } catch (Throwable ignored) { + } catch (final Throwable ignored) { return false; } } From 0066b322e13ecf9b421a61f238ef90eaf04bf49e Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 27 Jan 2025 11:20:57 +0100 Subject: [PATCH 04/19] Unify running on main thread --- .../newpipe/util/potoken/PoTokenWebView.kt | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 008477790..55cf23d93 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -170,8 +170,8 @@ class PoTokenWebView private constructor( override fun generatePoToken(identifier: String): Single = Single.create { emitter -> - addPoTokenEmitter(identifier, emitter) - Handler(Looper.getMainLooper()).post { + runOnMainThread(emitter) { + addPoTokenEmitter(identifier, emitter) webView.evaluateJavascript( """(async function() { identifier = String.raw`$identifier` @@ -266,7 +266,7 @@ class PoTokenWebView private constructor( * to [generatorEmitter]. */ private fun onInitializationErrorCloseAndCancel(error: Throwable) { - Handler(Looper.getMainLooper()).post { + runOnMainThread(generatorEmitter) { close() generatorEmitter.onError(error) } @@ -295,15 +295,29 @@ class PoTokenWebView private constructor( private val TAG = PoTokenWebView::class.simpleName private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" - private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" override fun newPoTokenGenerator(context: Context): Single = Single.create { emitter -> - Handler(Looper.getMainLooper()).post { + runOnMainThread(emitter) { val potWv = PoTokenWebView(context, emitter) potWv.loadHtmlAndObtainBotguard(context) emitter.setDisposable(potWv.disposables) } } + + /** + * Runs [runnable] on the main thread using `Handler(Looper.getMainLooper()).post()`, and + * if the `post` fails emits an error on [emitterIfPostFails]. + */ + private fun runOnMainThread( + emitterIfPostFails: SingleEmitter, + runnable: () -> Unit, + ) { + if (!Handler(Looper.getMainLooper()).post(runnable)) { + emitterIfPostFails.onError(PoTokenException("Could not run on main thread")) + } + } } } From f856bd9306154e793ce0c7ddc4fe2165378190eb Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 27 Jan 2025 12:06:19 +0100 Subject: [PATCH 05/19] Recreate poToken generator if current is broken This will be tried only once, and afterwards an error will be thrown --- .../util/potoken/PoTokenProviderImpl.kt | 81 ++++++++++++++----- .../newpipe/util/potoken/PoTokenWebView.kt | 2 + 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index a00e19f2d..da2aa9658 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -1,5 +1,7 @@ package org.schabi.newpipe.util.potoken +import android.os.Handler +import android.os.Looper import android.util.Log import org.schabi.newpipe.App import org.schabi.newpipe.extractor.NewPipe @@ -22,33 +24,68 @@ object PoTokenProviderImpl : PoTokenProvider { return null } - val (poTokenGenerator, visitorData, streamingPot) = synchronized(WebPoTokenGenLock) { - if (webPoTokenGenerator == null || webPoTokenGenerator!!.isExpired()) { - webPoTokenGenerator = PoTokenWebView.newPoTokenGenerator(App.getApp()).blockingGet() - webPoTokenVisitorData = YoutubeParsingHelper - .randomVisitorData(NewPipe.getPreferredContentCountry()) + return getWebClientPoToken(videoId = videoId, forceRecreate = false) + } - // The streaming poToken needs to be generated exactly once before generating any - // other (player) tokens. - webPoTokenStreamingPot = webPoTokenGenerator!! - .generatePoToken(webPoTokenVisitorData!!).blockingGet() + /** + * @param forceRecreate whether to force the recreation of [webPoTokenGenerator], to be used in + * case the current [webPoTokenGenerator] threw an error last time + * [PoTokenGenerator.generatePoToken] was called + */ + private fun getWebClientPoToken(videoId: String, forceRecreate: Boolean): PoTokenResult { + // just a helper class since Kotlin does not have builtin support for 4-tuples + data class Quadruple(val t1: T1, val t2: T2, val t3: T3, val t4: T4) + + val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) = + synchronized(WebPoTokenGenLock) { + val shouldRecreate = webPoTokenGenerator == null || forceRecreate || + webPoTokenGenerator!!.isExpired() + + if (shouldRecreate) { + // close the current webPoTokenGenerator on the main thread + webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } + + // create a new webPoTokenGenerator + webPoTokenGenerator = PoTokenWebView + .newPoTokenGenerator(App.getApp()).blockingGet() + webPoTokenVisitorData = YoutubeParsingHelper + .randomVisitorData(NewPipe.getPreferredContentCountry()) + + // The streaming poToken needs to be generated exactly once before generating + // any other (player) tokens. + webPoTokenStreamingPot = webPoTokenGenerator!! + .generatePoToken(webPoTokenVisitorData!!).blockingGet() + } + + return@synchronized Quadruple( + webPoTokenGenerator!!, + webPoTokenVisitorData!!, + webPoTokenStreamingPot!!, + shouldRecreate + ) + } + + val playerPot = try { + // Not using synchronized here, since poTokenGenerator would be able to generate + // multiple poTokens in parallel if needed. The only important thing is for exactly one + // visitorData/streaming poToken to be generated before anything else. + poTokenGenerator.generatePoToken(videoId).blockingGet() + } catch (throwable: Throwable) { + if (hasBeenRecreated) { + // the poTokenGenerator has just been recreated (and possibly this is already the + // second time we try), so there is likely nothing we can do + throw throwable + } else { + // retry, this time recreating the [webPoTokenGenerator] from scratch; + // this might happen for example if NewPipe goes in the background and the WebView + // content is lost + Log.e(TAG, "Failed to obtain poToken, retrying", throwable) + return getWebClientPoToken(videoId = videoId, forceRecreate = true) } - return@synchronized Triple( - webPoTokenGenerator!!, webPoTokenVisitorData!!, webPoTokenStreamingPot!! - ) } - // Not using synchronized here, since poTokenGenerator would be able to generate multiple - // poTokens in parallel if needed. The only important thing is for exactly one - // visitorData/streaming poToken to be generated before anything else. - val playerPot = poTokenGenerator.generatePoToken(videoId).blockingGet() Log.e(TAG, "success($videoId) $playerPot,web.gvs+$streamingPot;visitor_data=$visitorData") - - return PoTokenResult( - webPoTokenVisitorData!!, - playerPot, - webPoTokenStreamingPot!!, - ) + return PoTokenResult(visitorData, playerPot, streamingPot) } override fun getWebEmbedClientPoToken(videoId: String): PoTokenResult? = null diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 55cf23d93..30e19df9f 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -7,6 +7,7 @@ import android.os.Looper import android.util.Log import android.webkit.JavascriptInterface import android.webkit.WebView +import androidx.annotation.MainThread import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.core.SingleEmitter @@ -275,6 +276,7 @@ class PoTokenWebView private constructor( /** * Releases all [webView] and [disposables] resources. */ + @MainThread override fun close() { disposables.dispose() From 2b183a057651a79aa826a110e3eebf126befdd6c Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 27 Jan 2025 12:10:49 +0100 Subject: [PATCH 06/19] Wrap logs in BuildConfig.DEBUG --- .../util/potoken/PoTokenProviderImpl.kt | 12 ++++++++-- .../newpipe/util/potoken/PoTokenWebView.kt | 24 +++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index da2aa9658..4cfcc6eb6 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -4,6 +4,7 @@ import android.os.Handler import android.os.Looper import android.util.Log import org.schabi.newpipe.App +import org.schabi.newpipe.BuildConfig import org.schabi.newpipe.extractor.NewPipe import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider import org.schabi.newpipe.extractor.services.youtube.PoTokenResult @@ -39,7 +40,7 @@ object PoTokenProviderImpl : PoTokenProvider { val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) = synchronized(WebPoTokenGenLock) { val shouldRecreate = webPoTokenGenerator == null || forceRecreate || - webPoTokenGenerator!!.isExpired() + webPoTokenGenerator!!.isExpired() if (shouldRecreate) { // close the current webPoTokenGenerator on the main thread @@ -84,7 +85,14 @@ object PoTokenProviderImpl : PoTokenProvider { } } - Log.e(TAG, "success($videoId) $playerPot,web.gvs+$streamingPot;visitor_data=$visitorData") + if (BuildConfig.DEBUG) { + Log.d( + TAG, + "poToken for $videoId: playerPot=$playerPot, " + + "streamingPot=$streamingPot, visitor_data=$visitorData" + ) + } + return PoTokenResult(visitorData, playerPot, streamingPot) } diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 30e19df9f..b5d4bfb71 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -13,6 +13,7 @@ import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.core.SingleEmitter import io.reactivex.rxjava3.disposables.CompositeDisposable import io.reactivex.rxjava3.schedulers.Schedulers +import org.schabi.newpipe.BuildConfig import org.schabi.newpipe.DownloaderImpl import java.time.Instant @@ -100,7 +101,9 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onJsInitializationError(error: String) { - Log.e(TAG, "Initialization error from JavaScript: $error") + if (BuildConfig.DEBUG) { + Log.e(TAG, "Initialization error from JavaScript: $error") + } onInitializationErrorCloseAndCancel(PoTokenException(error)) } @@ -110,12 +113,16 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onRunBotguardResult(botguardResponse: String) { - Log.e(TAG, "botguardResponse: $botguardResponse") + if (BuildConfig.DEBUG) { + Log.d(TAG, "botguardResponse: $botguardResponse") + } makeJnnPaGoogleapisRequest( "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT", "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", ) { responseBody -> - Log.e(TAG, "GenerateIT response: $responseBody") + if (BuildConfig.DEBUG) { + Log.d(TAG, "GenerateIT response: $responseBody") + } webView.evaluateJavascript( """(async function() { try { @@ -194,7 +201,9 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onObtainPoTokenError(identifier: String, error: String) { - Log.e(TAG, "obtainPoToken error from JavaScript: $error") + if (BuildConfig.DEBUG) { + Log.e(TAG, "obtainPoToken error from JavaScript: $error") + } popPoTokenEmitter(identifier)?.onError(PoTokenException(error)) } @@ -204,8 +213,9 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onObtainPoTokenResult(identifier: String, poToken: String) { - Log.e(TAG, "identifier=$identifier") - Log.e(TAG, "poToken=$poToken") + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") + } popPoTokenEmitter(identifier)?.onSuccess(poToken) } @@ -298,7 +308,7 @@ class PoTokenWebView private constructor( private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" override fun newPoTokenGenerator(context: Context): Single = Single.create { emitter -> From e7fe84f2c74aeeb8509b7e529fd3c590af528927 Mon Sep 17 00:00:00 2001 From: Stypox Date: Mon, 27 Jan 2025 13:21:03 +0100 Subject: [PATCH 07/19] Make sure downloadAndRunBotguard() is called after ", + // calls downloadAndRunBotguard() when the page has finished loading + "\n$JS_INTERFACE.downloadAndRunBotguard()" + ), "text/html", "utf-8", null, ) - downloadAndRunBotguard() }, this::onInitializationErrorCloseAndCancel ) @@ -73,9 +80,15 @@ class PoTokenWebView private constructor( } /** - * Called during initialization after the WebView content has been loaded. + * Called during initialization by the JavaScript snippet appended to the HTML page content in + * [loadHtmlAndObtainBotguard] after the WebView content has been loaded. */ - private fun downloadAndRunBotguard() { + @JavascriptInterface + fun downloadAndRunBotguard() { + if (BuildConfig.DEBUG) { + Log.d(TAG, "downloadAndRunBotguard() called") + } + makeJnnPaGoogleapisRequest( "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/Create", "[ \"$REQUEST_KEY\" ]", @@ -86,9 +99,9 @@ class PoTokenWebView private constructor( data = JSON.parse(String.raw`$responseBody`) result = await runBotGuard(data) globalThis.webPoSignalOutput = result.webPoSignalOutput - PoTokenWebView.onRunBotguardResult(result.botguardResponse) + $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) } catch (error) { - PoTokenWebView.onJsInitializationError(error.toString()) + $JS_INTERFACE.onJsInitializationError(error.toString()) } })();""", ) {} @@ -127,9 +140,9 @@ class PoTokenWebView private constructor( """(async function() { try { globalThis.integrityToken = JSON.parse(String.raw`$responseBody`) - PoTokenWebView.onInitializationFinished(integrityToken[1]) + $JS_INTERFACE.onInitializationFinished(integrityToken[1]) } catch (error) { - PoTokenWebView.onJsInitializationError(error.toString()) + $JS_INTERFACE.onJsInitializationError(error.toString()) } })();""", ) {} @@ -145,6 +158,9 @@ class PoTokenWebView private constructor( */ @JavascriptInterface fun onInitializationFinished(expirationTimeInSeconds: Long) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "onInitializationFinished() called, expiration=${expirationTimeInSeconds}s") + } // leave 10 minutes of margin just to be sure expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) generatorEmitter.onSuccess(this) @@ -178,6 +194,9 @@ class PoTokenWebView private constructor( override fun generatePoToken(identifier: String): Single = Single.create { emitter -> + if (BuildConfig.DEBUG) { + Log.d(TAG, "generatePoToken() called with identifier $identifier") + } runOnMainThread(emitter) { addPoTokenEmitter(identifier, emitter) webView.evaluateJavascript( @@ -186,9 +205,9 @@ class PoTokenWebView private constructor( try { poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier) - PoTokenWebView.onObtainPoTokenResult(identifier, poToken) + $JS_INTERFACE.onObtainPoTokenResult(identifier, poToken) } catch (error) { - PoTokenWebView.onObtainPoTokenError(identifier, error.toString()) + $JS_INTERFACE.onObtainPoTokenError(identifier, error.toString()) } })();""", ) {} @@ -309,6 +328,7 @@ class PoTokenWebView private constructor( private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + private const val JS_INTERFACE = "PoTokenWebView" override fun newPoTokenGenerator(context: Context): Single = Single.create { emitter -> From 46d0bc1004aa13a1f892cd01cf017ade8439dbbd Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 22:28:08 +0100 Subject: [PATCH 08/19] Update NewPipeExtractor --- app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build.gradle b/app/build.gradle index 8b181c94b..92983b4a0 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -208,7 +208,7 @@ dependencies { implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' // WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with // the corresponding commit hash, since JitPack is sometimes buggy - implementation 'com.github.FireMasterK:NewPipeExtractor:d2cbd09089e8af933738f98b671ad58236a79d6e' + implementation 'com.github.AudricV:NewPipeExtractor:15e35a28df1205e8a0cf4680dd240554a818a120' implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0' /** Checkstyle **/ From b8e050f6c4d656995bc38ac021eb1bcf98ad36ea Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Fri, 31 Jan 2025 22:50:10 +0100 Subject: [PATCH 09/19] Adapt YoutubeHttpDataSource to extractor changes and improve requests Always use POST requests and the same body that official HTML5 clients use for a while. --- .../datasource/YoutubeHttpDataSource.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java b/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java index cf1f03b45..d4658d1de 100644 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java @@ -14,10 +14,12 @@ import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Util.castNonNull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTvHtml5UserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebEmbeddedPlayerStreamingUrl; import static java.lang.Math.min; import android.net.Uri; @@ -270,6 +272,7 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD private static final String RN_PARAMETER = "&rn="; private static final String YOUTUBE_BASE_URL = "https://www.youtube.com"; + private static final byte[] POST_BODY = new byte[] {0x78, 0}; private final boolean allowCrossProtocolRedirects; private final boolean rangeParameterEnabled; @@ -658,8 +661,11 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD } } + final boolean isTvHtml5StreamingUrl = isTvHtml5StreamingUrl(requestUrl); + if (isWebStreamingUrl(requestUrl) - || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(requestUrl)) { + || isTvHtml5StreamingUrl + || isWebEmbeddedPlayerStreamingUrl(requestUrl)) { httpURLConnection.setRequestProperty(HttpHeaders.ORIGIN, YOUTUBE_BASE_URL); httpURLConnection.setRequestProperty(HttpHeaders.REFERER, YOUTUBE_BASE_URL); httpURLConnection.setRequestProperty(HttpHeaders.SEC_FETCH_DEST, "empty"); @@ -679,6 +685,9 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD } else if (isIosStreamingUrl) { httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, getIosUserAgent(null)); + } else if (isTvHtml5StreamingUrl) { + httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, + getTvHtml5UserAgent()); } else { // non-mobile user agent httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, DownloaderImpl.USER_AGENT); @@ -687,22 +696,16 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD httpURLConnection.setRequestProperty(HttpHeaders.ACCEPT_ENCODING, allowGzip ? "gzip" : "identity"); httpURLConnection.setInstanceFollowRedirects(followRedirects); - httpURLConnection.setDoOutput(httpBody != null); + // Most clients use POST requests to fetch contents + httpURLConnection.setRequestMethod("POST"); + httpURLConnection.setDoOutput(true); + httpURLConnection.setFixedLengthStreamingMode(POST_BODY.length); + httpURLConnection.connect(); - // Mobile clients uses POST requests to fetch contents - httpURLConnection.setRequestMethod(isAndroidStreamingUrl || isIosStreamingUrl - ? "POST" - : DataSpec.getStringForHttpMethod(httpMethod)); + final OutputStream os = httpURLConnection.getOutputStream(); + os.write(POST_BODY); + os.close(); - if (httpBody != null) { - httpURLConnection.setFixedLengthStreamingMode(httpBody.length); - httpURLConnection.connect(); - final OutputStream os = httpURLConnection.getOutputStream(); - os.write(httpBody); - os.close(); - } else { - httpURLConnection.connect(); - } return httpURLConnection; } From 70ff47b81017a9ab79c617bfbbe49a825500638e Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 1 Feb 2025 15:39:07 +0100 Subject: [PATCH 10/19] [YouTube] Get visitorData from the service to get valid responses --- .../newpipe/util/potoken/PoTokenProviderImpl.kt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index 4cfcc6eb6..96c8b583e 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -6,6 +6,7 @@ import android.util.Log import org.schabi.newpipe.App import org.schabi.newpipe.BuildConfig import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider import org.schabi.newpipe.extractor.services.youtube.PoTokenResult import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper @@ -49,8 +50,20 @@ object PoTokenProviderImpl : PoTokenProvider { // create a new webPoTokenGenerator webPoTokenGenerator = PoTokenWebView .newPoTokenGenerator(App.getApp()).blockingGet() - webPoTokenVisitorData = YoutubeParsingHelper - .randomVisitorData(NewPipe.getPreferredContentCountry()) + + val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() + innertubeClientRequestInfo.clientInfo.clientVersion = + YoutubeParsingHelper.getClientVersion() + + webPoTokenVisitorData = YoutubeParsingHelper.getVisitorDataFromInnertube( + innertubeClientRequestInfo, + NewPipe.getPreferredLocalization(), + NewPipe.getPreferredContentCountry(), + YoutubeParsingHelper.getYouTubeHeaders(), + YoutubeParsingHelper.YOUTUBEI_V1_URL, + null, + false + ) // The streaming poToken needs to be generated exactly once before generating // any other (player) tokens. From ecd3f6c2ee36571c29f29a80811a1d12a59198eb Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Sat, 1 Feb 2025 15:40:16 +0100 Subject: [PATCH 11/19] [YouTube] Clarify BotGuard API key's origin and disable related Sonar warning --- .../java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 144b454c6..89cad89b0 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -324,7 +324,8 @@ class PoTokenWebView private constructor( companion object : PoTokenGenerator.Factory { private val TAG = PoTokenWebView::class.simpleName - private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" + // Public API key used by BotGuard, which has been got by looking at BotGuard requests + private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" // NOSONAR private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" From a60bb3e7afe96eab980bcebf42944fb46cede8fd Mon Sep 17 00:00:00 2001 From: AudricV <74829229+AudricV@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:05:39 +0100 Subject: [PATCH 12/19] [YouTube] Change BotGuard endpoint to youtube.com's one This prevents non-abilities to fetch BotGuard challenge and send its result with the jnn-pa.googleapis.com domain (domain block like done on Pi-hole lists or DNS servers). That's what the official website uses to send the challenge execution result, however it uses InnerTube to fetch the challenge. Embeds still use the jnn-pa.googleapis.com domain. Also rename the makeJnnPaGoogleapisRequest method appropriately. --- .../org/schabi/newpipe/util/potoken/PoTokenWebView.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 89cad89b0..2589e12c3 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -89,8 +89,8 @@ class PoTokenWebView private constructor( Log.d(TAG, "downloadAndRunBotguard() called") } - makeJnnPaGoogleapisRequest( - "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/Create", + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/Create", "[ \"$REQUEST_KEY\" ]", ) { responseBody -> webView.evaluateJavascript( @@ -129,8 +129,8 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.d(TAG, "botguardResponse: $botguardResponse") } - makeJnnPaGoogleapisRequest( - "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT", + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/GenerateIT", "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", ) { responseBody -> if (BuildConfig.DEBUG) { @@ -252,7 +252,7 @@ class PoTokenWebView private constructor( * successful. The request is performed in the background and a disposable is added to * [disposables]. */ - private fun makeJnnPaGoogleapisRequest( + private fun makeBotguardServiceRequest( url: String, data: String, handleResponseBody: (String) -> Unit, From 056809cb0d26e7763a65011624b19264fe1e265d Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 4 Feb 2025 10:22:10 +0100 Subject: [PATCH 13/19] Use "this" instead of "globalThis" as global scope globalThis was introduced only on newer versions of JS --- app/src/main/assets/po_token.html | 2 +- .../java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/main/assets/po_token.html b/app/src/main/assets/po_token.html index 572c3016e..3b9b18100 100644 --- a/app/src/main/assets/po_token.html +++ b/app/src/main/assets/po_token.html @@ -177,7 +177,7 @@ async function runBotGuard(rawChallengeData) { const botguard = await BotGuardClient.create({ globalName: challengeData.globalName, - globalObj: globalThis, + globalObj: this, program: challengeData.program }); diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 2589e12c3..76d2d9614 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -98,7 +98,7 @@ class PoTokenWebView private constructor( try { data = JSON.parse(String.raw`$responseBody`) result = await runBotGuard(data) - globalThis.webPoSignalOutput = result.webPoSignalOutput + this.webPoSignalOutput = result.webPoSignalOutput $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) } catch (error) { $JS_INTERFACE.onJsInitializationError(error.toString()) @@ -139,7 +139,7 @@ class PoTokenWebView private constructor( webView.evaluateJavascript( """(async function() { try { - globalThis.integrityToken = JSON.parse(String.raw`$responseBody`) + this.integrityToken = JSON.parse(String.raw`$responseBody`) $JS_INTERFACE.onInitializationFinished(integrityToken[1]) } catch (error) { $JS_INTERFACE.onJsInitializationError(error.toString()) From 3fc487310b1c70c1cbbbf58e696f873b9ae1c0ff Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 4 Feb 2025 10:23:45 +0100 Subject: [PATCH 14/19] Use Runnable instead of () -> Unit if converted to Runnable anyway --- .../main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 76d2d9614..e951bdd60 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -346,7 +346,7 @@ class PoTokenWebView private constructor( */ private fun runOnMainThread( emitterIfPostFails: SingleEmitter, - runnable: () -> Unit, + runnable: Runnable, ) { if (!Handler(Looper.getMainLooper()).post(runnable)) { emitterIfPostFails.onError(PoTokenException("Could not run on main thread")) From 21df24abfdd2d518445820ae6139ddd2a67f5672 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 4 Feb 2025 11:22:50 +0100 Subject: [PATCH 15/19] Detect when WebView is broken and return null poToken Some old Android devices have a broken WebView implementation, that can't execute the poToken code. This is now detected and the getWebClientPoToken return null instead of throwing an error in such a case, to allow the extractor to try to extract the video data even without a poToken. --- .../newpipe/util/potoken/PoTokenException.kt | 10 ++++++++ .../util/potoken/PoTokenProviderImpl.kt | 18 ++++++++++++-- .../newpipe/util/potoken/PoTokenWebView.kt | 24 +++++++++++++++++-- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt index 896c53a68..879f8f3e6 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt @@ -1,3 +1,13 @@ package org.schabi.newpipe.util.potoken class PoTokenException(message: String) : Exception(message) + +// to be thrown if the WebView provided by the system is broken +class BadWebViewException(message: String) : Exception(message) + +fun buildExceptionForJsError(error: String): Exception { + return if (error.contains("SyntaxError")) + BadWebViewException(error) + else + PoTokenException(error) +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index 96c8b583e..ac3a9f402 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -15,6 +15,7 @@ import org.schabi.newpipe.util.DeviceUtils object PoTokenProviderImpl : PoTokenProvider { val TAG = PoTokenProviderImpl::class.simpleName private val webViewSupported by lazy { DeviceUtils.supportsWebView() } + private var webViewBadImpl = false // whether the system has a bad WebView implementation private object WebPoTokenGenLock private var webPoTokenVisitorData: String? = null @@ -22,11 +23,24 @@ object PoTokenProviderImpl : PoTokenProvider { private var webPoTokenGenerator: PoTokenGenerator? = null override fun getWebClientPoToken(videoId: String): PoTokenResult? { - if (!webViewSupported) { + if (!webViewSupported || webViewBadImpl) { return null } - return getWebClientPoToken(videoId = videoId, forceRecreate = false) + try { + return getWebClientPoToken(videoId = videoId, forceRecreate = false) + } catch (e: RuntimeException) { + // RxJava's Single wraps exceptions into RuntimeErrors, so we need to unwrap them here + when (val cause = e.cause) { + is BadWebViewException -> { + Log.e(TAG, "Could not obtain poToken because WebView is broken", e) + webViewBadImpl = true + return null + } + null -> throw e + else -> throw cause // includes PoTokenException + } + } } /** diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index e951bdd60..504ab0b0b 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -5,7 +5,9 @@ import android.os.Build import android.os.Handler import android.os.Looper import android.util.Log +import android.webkit.ConsoleMessage import android.webkit.JavascriptInterface +import android.webkit.WebChromeClient import android.webkit.WebView import androidx.annotation.MainThread import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers @@ -40,6 +42,24 @@ class PoTokenWebView private constructor( // so that we can run async functions and get back the result webView.addJavascriptInterface(this, JS_INTERFACE) + + webView.webChromeClient = object : WebChromeClient() { + override fun onConsoleMessage(m: ConsoleMessage): Boolean { + if (m.message().contains("Uncaught")) { + // There should not be any uncaught errors while executing the code, because + // everything that can fail is guarded by try-catch. Therefore, this likely + // indicates that there was a syntax error in the code, i.e. the WebView only + // supports a really old version of JS. + + val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" + Log.e(TAG, "This WebView implementation is broken: $fmt") + + // This can only happen during initialization, where there is no try-catch + onInitializationErrorCloseAndCancel(BadWebViewException(fmt)) + } + return super.onConsoleMessage(m) + } + } } /** @@ -117,7 +137,7 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.e(TAG, "Initialization error from JavaScript: $error") } - onInitializationErrorCloseAndCancel(PoTokenException(error)) + onInitializationErrorCloseAndCancel(buildExceptionForJsError(error)) } /** @@ -223,7 +243,7 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.e(TAG, "obtainPoToken error from JavaScript: $error") } - popPoTokenEmitter(identifier)?.onError(PoTokenException(error)) + popPoTokenEmitter(identifier)?.onError(buildExceptionForJsError(error)) } /** From 53b599b042ff23cf2e74e3e8d90609b574a30fd1 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 4 Feb 2025 21:35:55 +0100 Subject: [PATCH 16/19] Make JavaScript code compatible with older WebViews --- app/src/main/assets/po_token.html | 254 ++++++------------ .../newpipe/util/potoken/JavaScriptUtil.kt | 113 ++++++++ .../util/potoken/PoTokenProviderImpl.kt | 12 +- .../newpipe/util/potoken/PoTokenWebView.kt | 116 ++++---- 4 files changed, 271 insertions(+), 224 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt diff --git a/app/src/main/assets/po_token.html b/app/src/main/assets/po_token.html index 3b9b18100..b55c13261 100644 --- a/app/src/main/assets/po_token.html +++ b/app/src/main/assets/po_token.html @@ -1,204 +1,120 @@ diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt new file mode 100644 index 000000000..a9169e2c6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt @@ -0,0 +1,113 @@ +package org.schabi.newpipe.util.potoken + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import com.grack.nanojson.JsonWriter +import okio.ByteString.Companion.decodeBase64 +import okio.ByteString.Companion.toByteString + +/** + * Parses the raw challenge data obtained from the Create endpoint and returns an object that can be + * embedded in a JavaScript snippet. + */ +fun parseChallengeData(rawChallengeData: String): String { + val scrambled = JsonParser.array().from(rawChallengeData) + + val challengeData = if (scrambled.size > 1 && scrambled.isString(1)) { + val descrambled = descramble(scrambled.getString(1)) + JsonParser.array().from(descrambled) + } else { + scrambled.getArray(1) + } + + val messageId = challengeData.getString(0) + val interpreterHash = challengeData.getString(3) + val program = challengeData.getString(4) + val globalName = challengeData.getString(5) + val clientExperimentsStateBlob = challengeData.getString(7) + + val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData.getArray(1, null)?.find { it is String } + val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData.getArray(2, null)?.find { it is String } + + return JsonWriter.string( + JsonObject.builder() + .value("messageId", messageId) + .`object`("interpreterJavascript") + .value("privateDoNotAccessOrElseSafeScriptWrappedValue", privateDoNotAccessOrElseSafeScriptWrappedValue) + .value("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", privateDoNotAccessOrElseTrustedResourceUrlWrappedValue) + .end() + .value("interpreterHash", interpreterHash) + .value("program", program) + .value("globalName", globalName) + .value("clientExperimentsStateBlob", clientExperimentsStateBlob) + .done() + ) +} + +/** + * Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the + * duration of this token in seconds. + */ +fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair { + val integrityTokenData = JsonParser.array().from(rawIntegrityTokenData) + return base64ToU8(integrityTokenData.getString(0)) to integrityTokenData.getLong(1) +} + +/** + * Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code. + */ +fun stringToU8(identifier: String): String { + return newUint8Array(identifier.toByteArray()) +} + +/** + * Takes a poToken encoded as a sequence of bytes represented as integers separated by commas + * (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript, + * and converts it to the specific base64 representation for poTokens. + */ +fun u8ToBase64(poToken: String): String { + return poToken.split(",") + .map { it.toUByte().toByte() } + .toByteArray() + .toByteString() + .base64() + .replace("+", "-") + .replace("/", "_") +} + +/** + * Takes the scrambled challenge, decodes it from base64, adds 97 to each byte. + */ +private fun descramble(scrambledChallenge: String): String { + return base64ToByteString(scrambledChallenge) + .map { (it + 97).toByte() } + .toByteArray() + .decodeToString() +} + +/** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube, and + * returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code. + */ +private fun base64ToU8(base64: String): String { + return newUint8Array(base64ToByteString(base64)) +} + +private fun newUint8Array(contents: ByteArray): String { + return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])" +} + +/** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube. + */ +private fun base64ToByteString(base64: String): ByteArray { + val base64Mod = base64 + .replace('-', '+') + .replace('_', '/') + .replace('.', '=') + + return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode")) + .toByteArray() +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index ac3a9f402..5383a613a 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -58,12 +58,6 @@ object PoTokenProviderImpl : PoTokenProvider { webPoTokenGenerator!!.isExpired() if (shouldRecreate) { - // close the current webPoTokenGenerator on the main thread - webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } - - // create a new webPoTokenGenerator - webPoTokenGenerator = PoTokenWebView - .newPoTokenGenerator(App.getApp()).blockingGet() val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() innertubeClientRequestInfo.clientInfo.clientVersion = @@ -78,6 +72,12 @@ object PoTokenProviderImpl : PoTokenProvider { null, false ) + // close the current webPoTokenGenerator on the main thread + webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } + + // create a new webPoTokenGenerator + webPoTokenGenerator = PoTokenWebView + .newPoTokenGenerator(App.getApp()).blockingGet() // The streaming poToken needs to be generated exactly once before generating // any other (player) tokens. diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 504ab0b0b..3ed5a0c75 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -52,10 +52,11 @@ class PoTokenWebView private constructor( // supports a really old version of JS. val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" + val exception = BadWebViewException(fmt) Log.e(TAG, "This WebView implementation is broken: $fmt") - // This can only happen during initialization, where there is no try-catch - onInitializationErrorCloseAndCancel(BadWebViewException(fmt)) + onInitializationErrorCloseAndCancel(exception) + popAllPoTokenEmitters().forEach { (_, emitter) -> emitter.onError(exception) } } return super.onConsoleMessage(m) } @@ -84,7 +85,7 @@ class PoTokenWebView private constructor( { html -> webView.loadDataWithBaseURL( "https://www.youtube.com", - html.replace( + html.replaceFirst( "", // calls downloadAndRunBotguard() when the page has finished loading "\n$JS_INTERFACE.downloadAndRunBotguard()" @@ -113,18 +114,21 @@ class PoTokenWebView private constructor( "https://www.youtube.com/api/jnn/v1/Create", "[ \"$REQUEST_KEY\" ]", ) { responseBody -> + val parsedChallengeData = parseChallengeData(responseBody) webView.evaluateJavascript( - """(async function() { - try { - data = JSON.parse(String.raw`$responseBody`) - result = await runBotGuard(data) + """try { + data = $parsedChallengeData + runBotGuard(data).then(function (result) { this.webPoSignalOutput = result.webPoSignalOutput $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) - } catch (error) { - $JS_INTERFACE.onJsInitializationError(error.toString()) - } - })();""", - ) {} + }, function (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }) + } catch (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }""", + null + ) } } @@ -156,38 +160,24 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.d(TAG, "GenerateIT response: $responseBody") } - webView.evaluateJavascript( - """(async function() { - try { - this.integrityToken = JSON.parse(String.raw`$responseBody`) - $JS_INTERFACE.onInitializationFinished(integrityToken[1]) - } catch (error) { - $JS_INTERFACE.onJsInitializationError(error.toString()) - } - })();""", - ) {} - } - } + val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) - /** - * Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the - * `integrityToken` has been received by JavaScript. - * - * @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found - * in `integrityToken[1]` - */ - @JavascriptInterface - fun onInitializationFinished(expirationTimeInSeconds: Long) { - if (BuildConfig.DEBUG) { - Log.d(TAG, "onInitializationFinished() called, expiration=${expirationTimeInSeconds}s") + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) + + webView.evaluateJavascript( + "this.integrityToken = $integrityToken" + ) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") + } + generatorEmitter.onSuccess(this) + } } - // leave 10 minutes of margin just to be sure - expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) - generatorEmitter.onSuccess(this) } //endregion - //region Obtaining poTokens + //region Handling multiple emitters /** * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that * multiple poToken requests can be generated invparallel, and the results will be notified to @@ -212,6 +202,20 @@ class PoTokenWebView private constructor( } } + /** + * Clears [poTokenEmitters] and returns its previous contents. The emitters are supposed to be + * used immediately after to either signal a success or an error. + */ + private fun popAllPoTokenEmitters(): List>> { + return synchronized(poTokenEmitters) { + val result = poTokenEmitters.toList() + poTokenEmitters.clear() + result + } + } + //endregion + + //region Obtaining poTokens override fun generatePoToken(identifier: String): Single = Single.create { emitter -> if (BuildConfig.DEBUG) { @@ -219,17 +223,21 @@ class PoTokenWebView private constructor( } runOnMainThread(emitter) { addPoTokenEmitter(identifier, emitter) + val u8Identifier = stringToU8(identifier) webView.evaluateJavascript( - """(async function() { - identifier = String.raw`$identifier` - try { - poToken = await obtainPoToken(webPoSignalOutput, integrityToken, - identifier) - $JS_INTERFACE.onObtainPoTokenResult(identifier, poToken) - } catch (error) { - $JS_INTERFACE.onObtainPoTokenError(identifier, error.toString()) + """try { + identifier = "$identifier" + u8Identifier = $u8Identifier + poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier) + poTokenU8String = "" + for (i = 0; i < poTokenU8.length; i++) { + if (i != 0) poTokenU8String += "," + poTokenU8String += poTokenU8[i] } - })();""", + $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String) + } catch (error) { + $JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack) + }""", ) {} } } @@ -251,7 +259,17 @@ class PoTokenWebView private constructor( * result of the JavaScript `obtainPoToken()` function. */ @JavascriptInterface - fun onObtainPoTokenResult(identifier: String, poToken: String) { + fun onObtainPoTokenResult(identifier: String, poTokenU8: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8") + } + val poToken = try { + u8ToBase64(poTokenU8) + } catch (t: Throwable) { + popPoTokenEmitter(identifier)?.onError(t) + return + } + if (BuildConfig.DEBUG) { Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") } From 87317c6faf65b414233e82c1eb78acf24478c6aa Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 4 Feb 2025 21:36:18 +0100 Subject: [PATCH 17/19] Reorder functions in PoTokenWebView --- .../newpipe/util/potoken/PoTokenWebView.kt | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 3ed5a0c75..37c2730b5 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -177,44 +177,6 @@ class PoTokenWebView private constructor( } //endregion - //region Handling multiple emitters - /** - * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that - * multiple poToken requests can be generated invparallel, and the results will be notified to - * the right emitters. - */ - private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { - synchronized(poTokenEmitters) { - poTokenEmitters.add(Pair(identifier, emitter)) - } - } - - /** - * Extracts and removes from the [poTokenEmitters] list a [SingleEmitter] based on its - * [identifier]. The emitter is supposed to be used immediately after to either signal a success - * or an error. - */ - private fun popPoTokenEmitter(identifier: String): SingleEmitter? { - return synchronized(poTokenEmitters) { - poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let { - poTokenEmitters.removeAt(it).second - } - } - } - - /** - * Clears [poTokenEmitters] and returns its previous contents. The emitters are supposed to be - * used immediately after to either signal a success or an error. - */ - private fun popAllPoTokenEmitters(): List>> { - return synchronized(poTokenEmitters) { - val result = poTokenEmitters.toList() - poTokenEmitters.clear() - result - } - } - //endregion - //region Obtaining poTokens override fun generatePoToken(identifier: String): Single = Single.create { emitter -> @@ -281,6 +243,44 @@ class PoTokenWebView private constructor( } //endregion + //region Handling multiple emitters + /** + * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that + * multiple poToken requests can be generated invparallel, and the results will be notified to + * the right emitters. + */ + private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { + synchronized(poTokenEmitters) { + poTokenEmitters.add(Pair(identifier, emitter)) + } + } + + /** + * Extracts and removes from the [poTokenEmitters] list a [SingleEmitter] based on its + * [identifier]. The emitter is supposed to be used immediately after to either signal a success + * or an error. + */ + private fun popPoTokenEmitter(identifier: String): SingleEmitter? { + return synchronized(poTokenEmitters) { + poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let { + poTokenEmitters.removeAt(it).second + } + } + } + + /** + * Clears [poTokenEmitters] and returns its previous contents. The emitters are supposed to be + * used immediately after to either signal a success or an error. + */ + private fun popAllPoTokenEmitters(): List>> { + return synchronized(poTokenEmitters) { + val result = poTokenEmitters.toList() + poTokenEmitters.clear() + result + } + } + //endregion + //region Utils /** * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls From b62a09b5b331edc0f66607591affd44ceda42518 Mon Sep 17 00:00:00 2001 From: Stypox Date: Tue, 4 Feb 2025 21:50:10 +0100 Subject: [PATCH 18/19] Use WebSettingsCompat.setSafeBrowsingEnabled --- app/build.gradle | 1 + .../schabi/newpipe/util/potoken/PoTokenWebView.kt | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 92983b4a0..dc16efe97 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -241,6 +241,7 @@ dependencies { implementation "androidx.work:work-runtime-ktx:${androidxWorkVersion}" implementation "androidx.work:work-rxjava3:${androidxWorkVersion}" implementation 'com.google.android.material:material:1.11.0' + implementation "androidx.webkit:webkit:1.9.0" /** Third-party libraries **/ // Instance state boilerplate elimination diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 37c2730b5..9b4b500f0 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -1,7 +1,6 @@ package org.schabi.newpipe.util.potoken import android.content.Context -import android.os.Build import android.os.Handler import android.os.Looper import android.util.Log @@ -10,6 +9,8 @@ import android.webkit.JavascriptInterface import android.webkit.WebChromeClient import android.webkit.WebView import androidx.annotation.MainThread +import androidx.webkit.WebSettingsCompat +import androidx.webkit.WebViewFeature import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.core.SingleEmitter @@ -31,14 +32,14 @@ class PoTokenWebView private constructor( //region Initialization init { - val webviewSettings = webView.settings + val webViewSettings = webView.settings //noinspection SetJavaScriptEnabled we want to use JavaScript! - webviewSettings.javaScriptEnabled = true - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - webviewSettings.safeBrowsingEnabled = false + webViewSettings.javaScriptEnabled = true + if (WebViewFeature.isFeatureSupported(WebViewFeature.SAFE_BROWSING_ENABLE)) { + WebSettingsCompat.setSafeBrowsingEnabled(webViewSettings, false) } - webviewSettings.userAgentString = USER_AGENT - webviewSettings.blockNetworkLoads = true // the WebView does not need internet access + webViewSettings.userAgentString = USER_AGENT + webViewSettings.blockNetworkLoads = true // the WebView does not need internet access // so that we can run async functions and get back the result webView.addJavascriptInterface(this, JS_INTERFACE) From dbee8d8128b173e1c5833c8d2e7c93cb5e55bb58 Mon Sep 17 00:00:00 2001 From: Stypox Date: Wed, 5 Feb 2025 10:24:34 +0100 Subject: [PATCH 19/19] Update NewPipeExtractor to v0.24.5 Using commit 9f83b385a since JitPack is buggy... --- app/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/build.gradle b/app/build.gradle index dc16efe97..85622ae98 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -208,7 +208,7 @@ dependencies { implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' // WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with // the corresponding commit hash, since JitPack is sometimes buggy - implementation 'com.github.AudricV:NewPipeExtractor:15e35a28df1205e8a0cf4680dd240554a818a120' + implementation 'com.github.TeamNewPipe:NewPipeExtractor:9f83b385a' implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0' /** Checkstyle **/