diff --git a/app/build.gradle b/app/build.gradle index 8d9981271..85622ae98 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -208,7 +208,7 @@ dependencies { implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751' // WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with // the corresponding commit hash, since JitPack is sometimes buggy - implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.24.4' + implementation 'com.github.TeamNewPipe:NewPipeExtractor:9f83b385a' implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0' /** Checkstyle **/ @@ -241,6 +241,7 @@ dependencies { implementation "androidx.work:work-runtime-ktx:${androidxWorkVersion}" implementation "androidx.work:work-rxjava3:${androidxWorkVersion}" implementation 'com.google.android.material:material:1.11.0' + implementation "androidx.webkit:webkit:1.9.0" /** Third-party libraries **/ // Instance state boilerplate elimination diff --git a/app/src/main/assets/po_token.html b/app/src/main/assets/po_token.html new file mode 100644 index 000000000..b55c13261 --- /dev/null +++ b/app/src/main/assets/po_token.html @@ -0,0 +1,127 @@ + + diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 9bc25d55d..8ce161eec 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -17,6 +17,7 @@ import org.acra.config.CoreConfigurationBuilder; import org.schabi.newpipe.error.ReCaptchaActivity; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.ktx.ExceptionUtils; import org.schabi.newpipe.settings.NewPipeSettings; import org.schabi.newpipe.util.BridgeStateSaverInitializer; @@ -26,6 +27,7 @@ import org.schabi.newpipe.util.StateSaver; import org.schabi.newpipe.util.image.ImageStrategy; import org.schabi.newpipe.util.image.PicassoHelper; import org.schabi.newpipe.util.image.PreferredImageQuality; +import org.schabi.newpipe.util.potoken.PoTokenProviderImpl; import java.io.IOException; import java.io.InterruptedIOException; @@ -118,6 +120,8 @@ public class App extends Application { && prefs.getBoolean(getString(R.string.show_image_indicators_key), false)); configureRxJavaErrorHandler(); + + YoutubeStreamExtractor.setPoTokenProvider(PoTokenProviderImpl.INSTANCE); } @Override diff --git a/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java b/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java index cf1f03b45..d4658d1de 100644 --- a/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java +++ b/app/src/main/java/org/schabi/newpipe/player/datasource/YoutubeHttpDataSource.java @@ -14,10 +14,12 @@ import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Util.castNonNull; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTvHtml5UserAgent; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isIosStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5StreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isTvHtml5SimplyEmbeddedPlayerStreamingUrl; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebEmbeddedPlayerStreamingUrl; import static java.lang.Math.min; import android.net.Uri; @@ -270,6 +272,7 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD private static final String RN_PARAMETER = "&rn="; private static final String YOUTUBE_BASE_URL = "https://www.youtube.com"; + private static final byte[] POST_BODY = new byte[] {0x78, 0}; private final boolean allowCrossProtocolRedirects; private final boolean rangeParameterEnabled; @@ -658,8 +661,11 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD } } + final boolean isTvHtml5StreamingUrl = isTvHtml5StreamingUrl(requestUrl); + if (isWebStreamingUrl(requestUrl) - || isTvHtml5SimplyEmbeddedPlayerStreamingUrl(requestUrl)) { + || isTvHtml5StreamingUrl + || isWebEmbeddedPlayerStreamingUrl(requestUrl)) { httpURLConnection.setRequestProperty(HttpHeaders.ORIGIN, YOUTUBE_BASE_URL); httpURLConnection.setRequestProperty(HttpHeaders.REFERER, YOUTUBE_BASE_URL); httpURLConnection.setRequestProperty(HttpHeaders.SEC_FETCH_DEST, "empty"); @@ -679,6 +685,9 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD } else if (isIosStreamingUrl) { httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, getIosUserAgent(null)); + } else if (isTvHtml5StreamingUrl) { + httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, + getTvHtml5UserAgent()); } else { // non-mobile user agent httpURLConnection.setRequestProperty(HttpHeaders.USER_AGENT, DownloaderImpl.USER_AGENT); @@ -687,22 +696,16 @@ public final class YoutubeHttpDataSource extends BaseDataSource implements HttpD httpURLConnection.setRequestProperty(HttpHeaders.ACCEPT_ENCODING, allowGzip ? "gzip" : "identity"); httpURLConnection.setInstanceFollowRedirects(followRedirects); - httpURLConnection.setDoOutput(httpBody != null); + // Most clients use POST requests to fetch contents + httpURLConnection.setRequestMethod("POST"); + httpURLConnection.setDoOutput(true); + httpURLConnection.setFixedLengthStreamingMode(POST_BODY.length); + httpURLConnection.connect(); - // Mobile clients uses POST requests to fetch contents - httpURLConnection.setRequestMethod(isAndroidStreamingUrl || isIosStreamingUrl - ? "POST" - : DataSpec.getStringForHttpMethod(httpMethod)); + final OutputStream os = httpURLConnection.getOutputStream(); + os.write(POST_BODY); + os.close(); - if (httpBody != null) { - httpURLConnection.setFixedLengthStreamingMode(httpBody.length); - httpURLConnection.connect(); - final OutputStream os = httpURLConnection.getOutputStream(); - os.write(httpBody); - os.close(); - } else { - httpURLConnection.connect(); - } return httpURLConnection; } diff --git a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java index e9678c2b0..9eb9fab37 100644 --- a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java +++ b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java @@ -17,6 +17,7 @@ import android.view.InputDevice; import android.view.KeyEvent; import android.view.WindowInsets; import android.view.WindowManager; +import android.webkit.CookieManager; import androidx.annotation.Dimension; import androidx.annotation.NonNull; @@ -335,4 +336,17 @@ public final class DeviceUtils { && !TX_50JXW834 && !HMB9213NW; } + + /** + * @return whether the device has support for WebView, see + * https://stackoverflow.com/a/69626735 + */ + public static boolean supportsWebView() { + try { + CookieManager.getInstance(); + return true; + } catch (final Throwable ignored) { + return false; + } + } } diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt new file mode 100644 index 000000000..a9169e2c6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt @@ -0,0 +1,113 @@ +package org.schabi.newpipe.util.potoken + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import com.grack.nanojson.JsonWriter +import okio.ByteString.Companion.decodeBase64 +import okio.ByteString.Companion.toByteString + +/** + * Parses the raw challenge data obtained from the Create endpoint and returns an object that can be + * embedded in a JavaScript snippet. + */ +fun parseChallengeData(rawChallengeData: String): String { + val scrambled = JsonParser.array().from(rawChallengeData) + + val challengeData = if (scrambled.size > 1 && scrambled.isString(1)) { + val descrambled = descramble(scrambled.getString(1)) + JsonParser.array().from(descrambled) + } else { + scrambled.getArray(1) + } + + val messageId = challengeData.getString(0) + val interpreterHash = challengeData.getString(3) + val program = challengeData.getString(4) + val globalName = challengeData.getString(5) + val clientExperimentsStateBlob = challengeData.getString(7) + + val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData.getArray(1, null)?.find { it is String } + val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData.getArray(2, null)?.find { it is String } + + return JsonWriter.string( + JsonObject.builder() + .value("messageId", messageId) + .`object`("interpreterJavascript") + .value("privateDoNotAccessOrElseSafeScriptWrappedValue", privateDoNotAccessOrElseSafeScriptWrappedValue) + .value("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", privateDoNotAccessOrElseTrustedResourceUrlWrappedValue) + .end() + .value("interpreterHash", interpreterHash) + .value("program", program) + .value("globalName", globalName) + .value("clientExperimentsStateBlob", clientExperimentsStateBlob) + .done() + ) +} + +/** + * Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the + * duration of this token in seconds. + */ +fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair { + val integrityTokenData = JsonParser.array().from(rawIntegrityTokenData) + return base64ToU8(integrityTokenData.getString(0)) to integrityTokenData.getLong(1) +} + +/** + * Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code. + */ +fun stringToU8(identifier: String): String { + return newUint8Array(identifier.toByteArray()) +} + +/** + * Takes a poToken encoded as a sequence of bytes represented as integers separated by commas + * (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript, + * and converts it to the specific base64 representation for poTokens. + */ +fun u8ToBase64(poToken: String): String { + return poToken.split(",") + .map { it.toUByte().toByte() } + .toByteArray() + .toByteString() + .base64() + .replace("+", "-") + .replace("/", "_") +} + +/** + * Takes the scrambled challenge, decodes it from base64, adds 97 to each byte. + */ +private fun descramble(scrambledChallenge: String): String { + return base64ToByteString(scrambledChallenge) + .map { (it + 97).toByte() } + .toByteArray() + .decodeToString() +} + +/** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube, and + * returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code. + */ +private fun base64ToU8(base64: String): String { + return newUint8Array(base64ToByteString(base64)) +} + +private fun newUint8Array(contents: ByteArray): String { + return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])" +} + +/** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube. + */ +private fun base64ToByteString(base64: String): ByteArray { + val base64Mod = base64 + .replace('-', '+') + .replace('_', '/') + .replace('.', '=') + + return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode")) + .toByteArray() +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt new file mode 100644 index 000000000..879f8f3e6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenException.kt @@ -0,0 +1,13 @@ +package org.schabi.newpipe.util.potoken + +class PoTokenException(message: String) : Exception(message) + +// to be thrown if the WebView provided by the system is broken +class BadWebViewException(message: String) : Exception(message) + +fun buildExceptionForJsError(error: String): Exception { + return if (error.contains("SyntaxError")) + BadWebViewException(error) + else + PoTokenException(error) +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt new file mode 100644 index 000000000..6446ecc72 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt @@ -0,0 +1,35 @@ +package org.schabi.newpipe.util.potoken + +import android.content.Context +import io.reactivex.rxjava3.core.Single +import java.io.Closeable + +/** + * This interface was created to allow for multiple methods to generate poTokens in the future (e.g. + * via WebView and via a local DOM implementation) + */ +interface PoTokenGenerator : Closeable { + /** + * Generates a poToken for the provided identifier, using the `integrityToken` and + * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be + * called multiple times. + */ + fun generatePoToken(identifier: String): Single + + /** + * @return whether the `integrityToken` is expired, in which case all tokens generated by + * [generatePoToken] will be invalid + */ + fun isExpired(): Boolean + + interface Factory { + /** + * Initializes a [PoTokenGenerator] by loading the BotGuard VM, running it, and obtaining + * an `integrityToken`. Can then be used multiple times to generate multiple poTokens with + * [generatePoToken]. + * + * @param context used e.g. to load the HTML asset or to instantiate a WebView + */ + fun newPoTokenGenerator(context: Context): Single + } +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt new file mode 100644 index 000000000..5383a613a --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -0,0 +1,131 @@ +package org.schabi.newpipe.util.potoken + +import android.os.Handler +import android.os.Looper +import android.util.Log +import org.schabi.newpipe.App +import org.schabi.newpipe.BuildConfig +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper +import org.schabi.newpipe.util.DeviceUtils + +object PoTokenProviderImpl : PoTokenProvider { + val TAG = PoTokenProviderImpl::class.simpleName + private val webViewSupported by lazy { DeviceUtils.supportsWebView() } + private var webViewBadImpl = false // whether the system has a bad WebView implementation + + private object WebPoTokenGenLock + private var webPoTokenVisitorData: String? = null + private var webPoTokenStreamingPot: String? = null + private var webPoTokenGenerator: PoTokenGenerator? = null + + override fun getWebClientPoToken(videoId: String): PoTokenResult? { + if (!webViewSupported || webViewBadImpl) { + return null + } + + try { + return getWebClientPoToken(videoId = videoId, forceRecreate = false) + } catch (e: RuntimeException) { + // RxJava's Single wraps exceptions into RuntimeErrors, so we need to unwrap them here + when (val cause = e.cause) { + is BadWebViewException -> { + Log.e(TAG, "Could not obtain poToken because WebView is broken", e) + webViewBadImpl = true + return null + } + null -> throw e + else -> throw cause // includes PoTokenException + } + } + } + + /** + * @param forceRecreate whether to force the recreation of [webPoTokenGenerator], to be used in + * case the current [webPoTokenGenerator] threw an error last time + * [PoTokenGenerator.generatePoToken] was called + */ + private fun getWebClientPoToken(videoId: String, forceRecreate: Boolean): PoTokenResult { + // just a helper class since Kotlin does not have builtin support for 4-tuples + data class Quadruple(val t1: T1, val t2: T2, val t3: T3, val t4: T4) + + val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) = + synchronized(WebPoTokenGenLock) { + val shouldRecreate = webPoTokenGenerator == null || forceRecreate || + webPoTokenGenerator!!.isExpired() + + if (shouldRecreate) { + + val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() + innertubeClientRequestInfo.clientInfo.clientVersion = + YoutubeParsingHelper.getClientVersion() + + webPoTokenVisitorData = YoutubeParsingHelper.getVisitorDataFromInnertube( + innertubeClientRequestInfo, + NewPipe.getPreferredLocalization(), + NewPipe.getPreferredContentCountry(), + YoutubeParsingHelper.getYouTubeHeaders(), + YoutubeParsingHelper.YOUTUBEI_V1_URL, + null, + false + ) + // close the current webPoTokenGenerator on the main thread + webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } + + // create a new webPoTokenGenerator + webPoTokenGenerator = PoTokenWebView + .newPoTokenGenerator(App.getApp()).blockingGet() + + // The streaming poToken needs to be generated exactly once before generating + // any other (player) tokens. + webPoTokenStreamingPot = webPoTokenGenerator!! + .generatePoToken(webPoTokenVisitorData!!).blockingGet() + } + + return@synchronized Quadruple( + webPoTokenGenerator!!, + webPoTokenVisitorData!!, + webPoTokenStreamingPot!!, + shouldRecreate + ) + } + + val playerPot = try { + // Not using synchronized here, since poTokenGenerator would be able to generate + // multiple poTokens in parallel if needed. The only important thing is for exactly one + // visitorData/streaming poToken to be generated before anything else. + poTokenGenerator.generatePoToken(videoId).blockingGet() + } catch (throwable: Throwable) { + if (hasBeenRecreated) { + // the poTokenGenerator has just been recreated (and possibly this is already the + // second time we try), so there is likely nothing we can do + throw throwable + } else { + // retry, this time recreating the [webPoTokenGenerator] from scratch; + // this might happen for example if NewPipe goes in the background and the WebView + // content is lost + Log.e(TAG, "Failed to obtain poToken, retrying", throwable) + return getWebClientPoToken(videoId = videoId, forceRecreate = true) + } + } + + if (BuildConfig.DEBUG) { + Log.d( + TAG, + "poToken for $videoId: playerPot=$playerPot, " + + "streamingPot=$streamingPot, visitor_data=$visitorData" + ) + } + + return PoTokenResult(visitorData, playerPot, streamingPot) + } + + override fun getWebEmbedClientPoToken(videoId: String): PoTokenResult? = null + + override fun getAndroidClientPoToken(videoId: String): PoTokenResult? = null + + override fun getIosClientPoToken(videoId: String): PoTokenResult? = null +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt new file mode 100644 index 000000000..9b4b500f0 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -0,0 +1,395 @@ +package org.schabi.newpipe.util.potoken + +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.webkit.ConsoleMessage +import android.webkit.JavascriptInterface +import android.webkit.WebChromeClient +import android.webkit.WebView +import androidx.annotation.MainThread +import androidx.webkit.WebSettingsCompat +import androidx.webkit.WebViewFeature +import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers +import io.reactivex.rxjava3.core.Single +import io.reactivex.rxjava3.core.SingleEmitter +import io.reactivex.rxjava3.disposables.CompositeDisposable +import io.reactivex.rxjava3.schedulers.Schedulers +import org.schabi.newpipe.BuildConfig +import org.schabi.newpipe.DownloaderImpl +import java.time.Instant + +class PoTokenWebView private constructor( + context: Context, + // to be used exactly once only during initialization! + private val generatorEmitter: SingleEmitter, +) : PoTokenGenerator { + private val webView = WebView(context) + private val disposables = CompositeDisposable() // used only during initialization + private val poTokenEmitters = mutableListOf>>() + private lateinit var expirationInstant: Instant + + //region Initialization + init { + val webViewSettings = webView.settings + //noinspection SetJavaScriptEnabled we want to use JavaScript! + webViewSettings.javaScriptEnabled = true + if (WebViewFeature.isFeatureSupported(WebViewFeature.SAFE_BROWSING_ENABLE)) { + WebSettingsCompat.setSafeBrowsingEnabled(webViewSettings, false) + } + webViewSettings.userAgentString = USER_AGENT + webViewSettings.blockNetworkLoads = true // the WebView does not need internet access + + // so that we can run async functions and get back the result + webView.addJavascriptInterface(this, JS_INTERFACE) + + webView.webChromeClient = object : WebChromeClient() { + override fun onConsoleMessage(m: ConsoleMessage): Boolean { + if (m.message().contains("Uncaught")) { + // There should not be any uncaught errors while executing the code, because + // everything that can fail is guarded by try-catch. Therefore, this likely + // indicates that there was a syntax error in the code, i.e. the WebView only + // supports a really old version of JS. + + val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" + val exception = BadWebViewException(fmt) + Log.e(TAG, "This WebView implementation is broken: $fmt") + + onInitializationErrorCloseAndCancel(exception) + popAllPoTokenEmitters().forEach { (_, emitter) -> emitter.onError(exception) } + } + return super.onConsoleMessage(m) + } + } + } + + /** + * Must be called right after instantiating [PoTokenWebView] to perform the actual + * initialization. This will asynchronously go through all the steps needed to load BotGuard, + * run it, and obtain an `integrityToken`. + */ + private fun loadHtmlAndObtainBotguard(context: Context) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "loadHtmlAndObtainBotguard() called") + } + + disposables.add( + Single.fromCallable { + val html = context.assets.open("po_token.html").bufferedReader() + .use { it.readText() } + return@fromCallable html + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe( + { html -> + webView.loadDataWithBaseURL( + "https://www.youtube.com", + html.replaceFirst( + "", + // calls downloadAndRunBotguard() when the page has finished loading + "\n$JS_INTERFACE.downloadAndRunBotguard()" + ), + "text/html", + "utf-8", + null, + ) + }, + this::onInitializationErrorCloseAndCancel + ) + ) + } + + /** + * Called during initialization by the JavaScript snippet appended to the HTML page content in + * [loadHtmlAndObtainBotguard] after the WebView content has been loaded. + */ + @JavascriptInterface + fun downloadAndRunBotguard() { + if (BuildConfig.DEBUG) { + Log.d(TAG, "downloadAndRunBotguard() called") + } + + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/Create", + "[ \"$REQUEST_KEY\" ]", + ) { responseBody -> + val parsedChallengeData = parseChallengeData(responseBody) + webView.evaluateJavascript( + """try { + data = $parsedChallengeData + runBotGuard(data).then(function (result) { + this.webPoSignalOutput = result.webPoSignalOutput + $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) + }, function (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }) + } catch (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }""", + null + ) + } + } + + /** + * Called during initialization by the JavaScript snippets from either + * [downloadAndRunBotguard] or [onRunBotguardResult]. + */ + @JavascriptInterface + fun onJsInitializationError(error: String) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "Initialization error from JavaScript: $error") + } + onInitializationErrorCloseAndCancel(buildExceptionForJsError(error)) + } + + /** + * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after + * obtaining the BotGuard execution output [botguardResponse]. + */ + @JavascriptInterface + fun onRunBotguardResult(botguardResponse: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "botguardResponse: $botguardResponse") + } + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/GenerateIT", + "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", + ) { responseBody -> + if (BuildConfig.DEBUG) { + Log.d(TAG, "GenerateIT response: $responseBody") + } + val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) + + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) + + webView.evaluateJavascript( + "this.integrityToken = $integrityToken" + ) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") + } + generatorEmitter.onSuccess(this) + } + } + } + //endregion + + //region Obtaining poTokens + override fun generatePoToken(identifier: String): Single = + Single.create { emitter -> + if (BuildConfig.DEBUG) { + Log.d(TAG, "generatePoToken() called with identifier $identifier") + } + runOnMainThread(emitter) { + addPoTokenEmitter(identifier, emitter) + val u8Identifier = stringToU8(identifier) + webView.evaluateJavascript( + """try { + identifier = "$identifier" + u8Identifier = $u8Identifier + poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier) + poTokenU8String = "" + for (i = 0; i < poTokenU8.length; i++) { + if (i != 0) poTokenU8String += "," + poTokenU8String += poTokenU8[i] + } + $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String) + } catch (error) { + $JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack) + }""", + ) {} + } + } + + /** + * Called by the JavaScript snippet from [generatePoToken] when an error occurs in calling the + * JavaScript `obtainPoToken()` function. + */ + @JavascriptInterface + fun onObtainPoTokenError(identifier: String, error: String) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "obtainPoToken error from JavaScript: $error") + } + popPoTokenEmitter(identifier)?.onError(buildExceptionForJsError(error)) + } + + /** + * Called by the JavaScript snippet from [generatePoToken] with the original identifier and the + * result of the JavaScript `obtainPoToken()` function. + */ + @JavascriptInterface + fun onObtainPoTokenResult(identifier: String, poTokenU8: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8") + } + val poToken = try { + u8ToBase64(poTokenU8) + } catch (t: Throwable) { + popPoTokenEmitter(identifier)?.onError(t) + return + } + + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") + } + popPoTokenEmitter(identifier)?.onSuccess(poToken) + } + + override fun isExpired(): Boolean { + return Instant.now().isAfter(expirationInstant) + } + //endregion + + //region Handling multiple emitters + /** + * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that + * multiple poToken requests can be generated invparallel, and the results will be notified to + * the right emitters. + */ + private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { + synchronized(poTokenEmitters) { + poTokenEmitters.add(Pair(identifier, emitter)) + } + } + + /** + * Extracts and removes from the [poTokenEmitters] list a [SingleEmitter] based on its + * [identifier]. The emitter is supposed to be used immediately after to either signal a success + * or an error. + */ + private fun popPoTokenEmitter(identifier: String): SingleEmitter? { + return synchronized(poTokenEmitters) { + poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let { + poTokenEmitters.removeAt(it).second + } + } + } + + /** + * Clears [poTokenEmitters] and returns its previous contents. The emitters are supposed to be + * used immediately after to either signal a success or an error. + */ + private fun popAllPoTokenEmitters(): List>> { + return synchronized(poTokenEmitters) { + val result = poTokenEmitters.toList() + poTokenEmitters.clear() + result + } + } + //endregion + + //region Utils + /** + * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls + * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response + * does not have HTTP code 200, therefore this is supposed to be used only during + * initialization. Calls [handleResponseBody] with the response body if the response is + * successful. The request is performed in the background and a disposable is added to + * [disposables]. + */ + private fun makeBotguardServiceRequest( + url: String, + data: String, + handleResponseBody: (String) -> Unit, + ) { + disposables.add( + Single.fromCallable { + return@fromCallable DownloaderImpl.getInstance().post( + url, + mapOf( + // replace the downloader user agent + "User-Agent" to listOf(USER_AGENT), + "Accept" to listOf("application/json"), + "Content-Type" to listOf("application/json+protobuf"), + "x-goog-api-key" to listOf(GOOGLE_API_KEY), + "x-user-agent" to listOf("grpc-web-javascript/0.1"), + ), + data.toByteArray() + ) + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe( + { response -> + val httpCode = response.responseCode() + if (httpCode != 200) { + onInitializationErrorCloseAndCancel( + PoTokenException("Invalid response code: $httpCode") + ) + return@subscribe + } + val responseBody = response.responseBody() + handleResponseBody(responseBody) + }, + this::onInitializationErrorCloseAndCancel + ) + ) + } + + /** + * Handles any error happening during initialization, releasing resources and sending the error + * to [generatorEmitter]. + */ + private fun onInitializationErrorCloseAndCancel(error: Throwable) { + runOnMainThread(generatorEmitter) { + close() + generatorEmitter.onError(error) + } + } + + /** + * Releases all [webView] and [disposables] resources. + */ + @MainThread + override fun close() { + disposables.dispose() + + webView.clearHistory() + // clears RAM cache and disk cache (globally for all WebViews) + webView.clearCache(true) + + // ensures that the WebView isn't doing anything when destroying it + webView.loadUrl("about:blank") + + webView.onPause() + webView.removeAllViews() + webView.destroy() + } + //endregion + + companion object : PoTokenGenerator.Factory { + private val TAG = PoTokenWebView::class.simpleName + // Public API key used by BotGuard, which has been got by looking at BotGuard requests + private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" // NOSONAR + private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" + private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + private const val JS_INTERFACE = "PoTokenWebView" + + override fun newPoTokenGenerator(context: Context): Single = + Single.create { emitter -> + runOnMainThread(emitter) { + val potWv = PoTokenWebView(context, emitter) + potWv.loadHtmlAndObtainBotguard(context) + emitter.setDisposable(potWv.disposables) + } + } + + /** + * Runs [runnable] on the main thread using `Handler(Looper.getMainLooper()).post()`, and + * if the `post` fails emits an error on [emitterIfPostFails]. + */ + private fun runOnMainThread( + emitterIfPostFails: SingleEmitter, + runnable: Runnable, + ) { + if (!Handler(Looper.getMainLooper()).post(runnable)) { + emitterIfPostFails.onError(PoTokenException("Could not run on main thread")) + } + } + } +}