diff --git a/app/src/main/assets/po_token.html b/app/src/main/assets/po_token.html index 3b9b18100..b55c13261 100644 --- a/app/src/main/assets/po_token.html +++ b/app/src/main/assets/po_token.html @@ -1,204 +1,120 @@ diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt new file mode 100644 index 000000000..a9169e2c6 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/JavaScriptUtil.kt @@ -0,0 +1,113 @@ +package org.schabi.newpipe.util.potoken + +import com.grack.nanojson.JsonObject +import com.grack.nanojson.JsonParser +import com.grack.nanojson.JsonWriter +import okio.ByteString.Companion.decodeBase64 +import okio.ByteString.Companion.toByteString + +/** + * Parses the raw challenge data obtained from the Create endpoint and returns an object that can be + * embedded in a JavaScript snippet. + */ +fun parseChallengeData(rawChallengeData: String): String { + val scrambled = JsonParser.array().from(rawChallengeData) + + val challengeData = if (scrambled.size > 1 && scrambled.isString(1)) { + val descrambled = descramble(scrambled.getString(1)) + JsonParser.array().from(descrambled) + } else { + scrambled.getArray(1) + } + + val messageId = challengeData.getString(0) + val interpreterHash = challengeData.getString(3) + val program = challengeData.getString(4) + val globalName = challengeData.getString(5) + val clientExperimentsStateBlob = challengeData.getString(7) + + val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData.getArray(1, null)?.find { it is String } + val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData.getArray(2, null)?.find { it is String } + + return JsonWriter.string( + JsonObject.builder() + .value("messageId", messageId) + .`object`("interpreterJavascript") + .value("privateDoNotAccessOrElseSafeScriptWrappedValue", privateDoNotAccessOrElseSafeScriptWrappedValue) + .value("privateDoNotAccessOrElseTrustedResourceUrlWrappedValue", privateDoNotAccessOrElseTrustedResourceUrlWrappedValue) + .end() + .value("interpreterHash", interpreterHash) + .value("program", program) + .value("globalName", globalName) + .value("clientExperimentsStateBlob", clientExperimentsStateBlob) + .done() + ) +} + +/** + * Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the + * duration of this token in seconds. + */ +fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair { + val integrityTokenData = JsonParser.array().from(rawIntegrityTokenData) + return base64ToU8(integrityTokenData.getString(0)) to integrityTokenData.getLong(1) +} + +/** + * Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code. + */ +fun stringToU8(identifier: String): String { + return newUint8Array(identifier.toByteArray()) +} + +/** + * Takes a poToken encoded as a sequence of bytes represented as integers separated by commas + * (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript, + * and converts it to the specific base64 representation for poTokens. + */ +fun u8ToBase64(poToken: String): String { + return poToken.split(",") + .map { it.toUByte().toByte() } + .toByteArray() + .toByteString() + .base64() + .replace("+", "-") + .replace("/", "_") +} + +/** + * Takes the scrambled challenge, decodes it from base64, adds 97 to each byte. + */ +private fun descramble(scrambledChallenge: String): String { + return base64ToByteString(scrambledChallenge) + .map { (it + 97).toByte() } + .toByteArray() + .decodeToString() +} + +/** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube, and + * returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code. + */ +private fun base64ToU8(base64: String): String { + return newUint8Array(base64ToByteString(base64)) +} + +private fun newUint8Array(contents: ByteArray): String { + return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])" +} + +/** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube. + */ +private fun base64ToByteString(base64: String): ByteArray { + val base64Mod = base64 + .replace('-', '+') + .replace('_', '/') + .replace('.', '=') + + return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode")) + .toByteArray() +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt index ac3a9f402..5383a613a 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -58,12 +58,6 @@ object PoTokenProviderImpl : PoTokenProvider { webPoTokenGenerator!!.isExpired() if (shouldRecreate) { - // close the current webPoTokenGenerator on the main thread - webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } - - // create a new webPoTokenGenerator - webPoTokenGenerator = PoTokenWebView - .newPoTokenGenerator(App.getApp()).blockingGet() val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() innertubeClientRequestInfo.clientInfo.clientVersion = @@ -78,6 +72,12 @@ object PoTokenProviderImpl : PoTokenProvider { null, false ) + // close the current webPoTokenGenerator on the main thread + webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } + + // create a new webPoTokenGenerator + webPoTokenGenerator = PoTokenWebView + .newPoTokenGenerator(App.getApp()).blockingGet() // The streaming poToken needs to be generated exactly once before generating // any other (player) tokens. diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 504ab0b0b..3ed5a0c75 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -52,10 +52,11 @@ class PoTokenWebView private constructor( // supports a really old version of JS. val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" + val exception = BadWebViewException(fmt) Log.e(TAG, "This WebView implementation is broken: $fmt") - // This can only happen during initialization, where there is no try-catch - onInitializationErrorCloseAndCancel(BadWebViewException(fmt)) + onInitializationErrorCloseAndCancel(exception) + popAllPoTokenEmitters().forEach { (_, emitter) -> emitter.onError(exception) } } return super.onConsoleMessage(m) } @@ -84,7 +85,7 @@ class PoTokenWebView private constructor( { html -> webView.loadDataWithBaseURL( "https://www.youtube.com", - html.replace( + html.replaceFirst( "", // calls downloadAndRunBotguard() when the page has finished loading "\n$JS_INTERFACE.downloadAndRunBotguard()" @@ -113,18 +114,21 @@ class PoTokenWebView private constructor( "https://www.youtube.com/api/jnn/v1/Create", "[ \"$REQUEST_KEY\" ]", ) { responseBody -> + val parsedChallengeData = parseChallengeData(responseBody) webView.evaluateJavascript( - """(async function() { - try { - data = JSON.parse(String.raw`$responseBody`) - result = await runBotGuard(data) + """try { + data = $parsedChallengeData + runBotGuard(data).then(function (result) { this.webPoSignalOutput = result.webPoSignalOutput $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) - } catch (error) { - $JS_INTERFACE.onJsInitializationError(error.toString()) - } - })();""", - ) {} + }, function (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }) + } catch (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }""", + null + ) } } @@ -156,38 +160,24 @@ class PoTokenWebView private constructor( if (BuildConfig.DEBUG) { Log.d(TAG, "GenerateIT response: $responseBody") } - webView.evaluateJavascript( - """(async function() { - try { - this.integrityToken = JSON.parse(String.raw`$responseBody`) - $JS_INTERFACE.onInitializationFinished(integrityToken[1]) - } catch (error) { - $JS_INTERFACE.onJsInitializationError(error.toString()) - } - })();""", - ) {} - } - } + val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) - /** - * Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the - * `integrityToken` has been received by JavaScript. - * - * @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found - * in `integrityToken[1]` - */ - @JavascriptInterface - fun onInitializationFinished(expirationTimeInSeconds: Long) { - if (BuildConfig.DEBUG) { - Log.d(TAG, "onInitializationFinished() called, expiration=${expirationTimeInSeconds}s") + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) + + webView.evaluateJavascript( + "this.integrityToken = $integrityToken" + ) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") + } + generatorEmitter.onSuccess(this) + } } - // leave 10 minutes of margin just to be sure - expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) - generatorEmitter.onSuccess(this) } //endregion - //region Obtaining poTokens + //region Handling multiple emitters /** * Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that * multiple poToken requests can be generated invparallel, and the results will be notified to @@ -212,6 +202,20 @@ class PoTokenWebView private constructor( } } + /** + * Clears [poTokenEmitters] and returns its previous contents. The emitters are supposed to be + * used immediately after to either signal a success or an error. + */ + private fun popAllPoTokenEmitters(): List>> { + return synchronized(poTokenEmitters) { + val result = poTokenEmitters.toList() + poTokenEmitters.clear() + result + } + } + //endregion + + //region Obtaining poTokens override fun generatePoToken(identifier: String): Single = Single.create { emitter -> if (BuildConfig.DEBUG) { @@ -219,17 +223,21 @@ class PoTokenWebView private constructor( } runOnMainThread(emitter) { addPoTokenEmitter(identifier, emitter) + val u8Identifier = stringToU8(identifier) webView.evaluateJavascript( - """(async function() { - identifier = String.raw`$identifier` - try { - poToken = await obtainPoToken(webPoSignalOutput, integrityToken, - identifier) - $JS_INTERFACE.onObtainPoTokenResult(identifier, poToken) - } catch (error) { - $JS_INTERFACE.onObtainPoTokenError(identifier, error.toString()) + """try { + identifier = "$identifier" + u8Identifier = $u8Identifier + poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier) + poTokenU8String = "" + for (i = 0; i < poTokenU8.length; i++) { + if (i != 0) poTokenU8String += "," + poTokenU8String += poTokenU8[i] } - })();""", + $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String) + } catch (error) { + $JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack) + }""", ) {} } } @@ -251,7 +259,17 @@ class PoTokenWebView private constructor( * result of the JavaScript `obtainPoToken()` function. */ @JavascriptInterface - fun onObtainPoTokenResult(identifier: String, poToken: String) { + fun onObtainPoTokenResult(identifier: String, poTokenU8: String) { + if (BuildConfig.DEBUG) { + Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8") + } + val poToken = try { + u8ToBase64(poTokenU8) + } catch (t: Throwable) { + popPoTokenEmitter(identifier)?.onError(t) + return + } + if (BuildConfig.DEBUG) { Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") }