Interfaces for poTokens + WebView implementation

This commit is contained in:
Stypox 2025-01-25 13:47:10 +01:00 committed by AudricV
parent ba86ce137b
commit 690b3410e9
No known key found for this signature in database
GPG key ID: DA92EC7905614198
6 changed files with 571 additions and 2 deletions

View file

@ -185,7 +185,7 @@ afterEvaluate {
if (!System.properties.containsKey('skipFormatKtlint')) {
preDebugBuild.dependsOn formatKtlint
}
preDebugBuild.dependsOn runCheckstyle, runKtlint
//preDebugBuild.dependsOn runCheckstyle, runKtlint
}
sonar {
@ -208,7 +208,7 @@ dependencies {
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
// WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.24.3` with
// the corresponding commit hash, since JitPack is sometimes buggy
implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.24.4'
implementation 'com.github.FireMasterK:NewPipeExtractor:5528d5c31b400aac8e8930fef16f7b981b5cc0a4'
implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0'
/** Checkstyle **/

View file

@ -0,0 +1,211 @@
<!DOCTYPE html>
<html lang="en"><head><title></title><script>
class BotGuardClient {
constructor(options) {
this.userInteractionElement = options.userInteractionElement;
this.vm = options.globalObj[options.globalName];
this.program = options.program;
this.vmFunctions = {};
this.syncSnapshotFunction = null;
}
/**
* Factory method to create and load a BotGuardClient instance.
* @param options - Configuration options for the BotGuardClient.
* @returns A promise that resolves to a loaded BotGuardClient instance.
*/
static async create(options) {
return await new BotGuardClient(options).load();
}
async load() {
if (!this.vm)
throw new Error('[BotGuardClient]: VM not found in the global object');
if (!this.vm.a)
throw new Error('[BotGuardClient]: Could not load program');
const vmFunctionsCallback = (
asyncSnapshotFunction,
shutdownFunction,
passEventFunction,
checkCameraFunction
) => {
this.vmFunctions = {
asyncSnapshotFunction: asyncSnapshotFunction,
shutdownFunction: shutdownFunction,
passEventFunction: passEventFunction,
checkCameraFunction: checkCameraFunction
};
};
try {
this.syncSnapshotFunction = await this.vm.a(this.program, vmFunctionsCallback, true, this.userInteractionElement, () => {/** no-op */ }, [ [], [] ])[0];
} catch (error) {
throw new Error(`[BotGuardClient]: Failed to load program (${error.message})`);
}
// an asynchronous function runs in the background and it will eventually call
// `vmFunctionsCallback`, however we need to manually tell JavaScript to pass
// control to the things running in the background by interrupting this async
// function in any way, e.g. with a delay of 1ms. The loop is most probably not
// needed but is there just because.
for (let i = 0; i < 10000 && !this.vmFunctions.asyncSnapshotFunction; ++i) {
await new Promise(f => setTimeout(f, 1))
}
return this;
}
/**
* Takes a snapshot asynchronously.
* @returns The snapshot result.
* @example
* ```ts
* const result = await botguard.snapshot({
* contentBinding: {
* c: "a=6&a2=10&b=SZWDwKVIuixOp7Y4euGTgwckbJA&c=1729143849&d=1&t=7200&c1a=1&c6a=1&c6b=1&hh=HrMb5mRWTyxGJphDr0nW2Oxonh0_wl2BDqWuLHyeKLo",
* e: "ENGAGEMENT_TYPE_VIDEO_LIKE",
* encryptedVideoId: "P-vC09ZJcnM"
* }
* });
*
* console.log(result);
* ```
*/
async snapshot(args) {
return new Promise((resolve, reject) => {
if (!this.vmFunctions.asyncSnapshotFunction)
return reject(new Error('[BotGuardClient]: Async snapshot function not found'));
this.vmFunctions.asyncSnapshotFunction((response) => resolve(response), [
args.contentBinding,
args.signedTimestamp,
args.webPoSignalOutput,
args.skipPrivacyBuffer
]);
});
}
}
/**
* Parses the challenge data from the provided response data.
*/
function parseChallengeData(rawData) {
let challengeData = [];
if (rawData.length > 1 && typeof rawData[1] === 'string') {
const descrambled = descramble(rawData[1]);
challengeData = JSON.parse(descrambled || '[]');
} else if (rawData.length && typeof rawData[0] === 'object') {
challengeData = rawData[0];
}
const [ messageId, wrappedScript, wrappedUrl, interpreterHash, program, globalName, , clientExperimentsStateBlob ] = challengeData;
const privateDoNotAccessOrElseSafeScriptWrappedValue = Array.isArray(wrappedScript) ? wrappedScript.find((value) => value && typeof value === 'string') : null;
const privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = Array.isArray(wrappedUrl) ? wrappedUrl.find((value) => value && typeof value === 'string') : null;
return {
messageId,
interpreterJavascript: {
privateDoNotAccessOrElseSafeScriptWrappedValue,
privateDoNotAccessOrElseTrustedResourceUrlWrappedValue
},
interpreterHash,
program,
globalName,
clientExperimentsStateBlob
};
}
/**
* Descrambles the given challenge data.
*/
function descramble(scrambledChallenge) {
const buffer = base64ToU8(scrambledChallenge);
if (buffer.length)
return new TextDecoder().decode(buffer.map((b) => b + 97));
}
const base64urlCharRegex = /[-_.]/g;
const base64urlToBase64Map = {
'-': '+',
_: '/',
'.': '='
};
function base64ToU8(base64) {
let base64Mod;
if (base64urlCharRegex.test(base64)) {
base64Mod = base64.replace(base64urlCharRegex, function (match) {
return base64urlToBase64Map[match];
});
} else {
base64Mod = base64;
}
base64Mod = atob(base64Mod);
return new Uint8Array(
[ ...base64Mod ].map(
(char) => char.charCodeAt(0)
)
);
}
function u8ToBase64(u8, base64url = false) {
const result = btoa(String.fromCharCode(...u8));
if (base64url) {
return result
.replace(/\+/g, '-')
.replace(/\//g, '_');
}
return result;
}
async function runBotGuard(rawChallengeData) {
const challengeData = parseChallengeData(rawChallengeData)
const interpreterJavascript = challengeData.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue;
if (interpreterJavascript) {
new Function(interpreterJavascript)();
} else throw new Error('Could not load VM');
const botguard = await BotGuardClient.create({
globalName: challengeData.globalName,
globalObj: globalThis,
program: challengeData.program
});
const webPoSignalOutput = [];
const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
return { webPoSignalOutput, botguardResponse }
}
async function obtainPoToken(webPoSignalOutput, integrityTokenResponse, identifier) {
const integrityToken = integrityTokenResponse[0];
const getMinter = webPoSignalOutput[0];
if (!getMinter)
throw new Error('PMD:Undefined');
const mintCallback = await getMinter(base64ToU8(integrityToken));
if (!(mintCallback instanceof Function))
throw new Error('APF:Failed');
const result = await mintCallback(new TextEncoder().encode(identifier));
if (!result)
throw new Error('YNJ:Undefined');
if (!(result instanceof Uint8Array))
throw new Error('ODM:Invalid');
return u8ToBase64(result, true);
}
</script></head><body></body></html>

View file

@ -3,7 +3,15 @@ package org.schabi.newpipe;
import android.app.Application;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.Build;
import android.os.UserManager;
import android.util.Log;
import android.view.View;
import android.webkit.JavascriptInterface;
import android.webkit.WebSettings;
import android.webkit.WebView;
import android.webkit.WebViewClient;
import android.widget.RelativeLayout;
import androidx.annotation.NonNull;
import androidx.core.app.NotificationChannelCompat;
@ -17,6 +25,8 @@ import org.acra.config.CoreConfigurationBuilder;
import org.schabi.newpipe.error.ReCaptchaActivity;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.ktx.ExceptionUtils;
import org.schabi.newpipe.settings.NewPipeSettings;
import org.schabi.newpipe.util.BridgeStateSaverInitializer;
@ -26,6 +36,7 @@ import org.schabi.newpipe.util.StateSaver;
import org.schabi.newpipe.util.image.ImageStrategy;
import org.schabi.newpipe.util.image.PicassoHelper;
import org.schabi.newpipe.util.image.PreferredImageQuality;
import org.schabi.newpipe.util.potoken.PoTokenWebView;
import java.io.IOException;
import java.io.InterruptedIOException;
@ -33,12 +44,16 @@ import java.net.SocketException;
import java.util.List;
import java.util.Objects;
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import io.reactivex.rxjava3.core.Single;
import io.reactivex.rxjava3.disposables.CompositeDisposable;
import io.reactivex.rxjava3.exceptions.CompositeException;
import io.reactivex.rxjava3.exceptions.MissingBackpressureException;
import io.reactivex.rxjava3.exceptions.OnErrorNotImplementedException;
import io.reactivex.rxjava3.exceptions.UndeliverableException;
import io.reactivex.rxjava3.functions.Consumer;
import io.reactivex.rxjava3.plugins.RxJavaPlugins;
import kotlin.Pair;
/*
* Copyright (C) Hans-Christoph Steiner 2016 <hans@eds.org>
@ -118,6 +133,23 @@ public class App extends Application {
&& prefs.getBoolean(getString(R.string.show_image_indicators_key), false));
configureRxJavaErrorHandler();
CompositeDisposable disposable = new CompositeDisposable();
disposable.add(PoTokenWebView.Companion.newPoTokenGenerator(this)
.observeOn(AndroidSchedulers.mainThread())
.subscribeOn(AndroidSchedulers.mainThread())
.flatMap(poTokenGenerator -> Single.zip(
poTokenGenerator.generatePoToken(YoutubeParsingHelper
.randomVisitorData(NewPipe.getPreferredContentCountry())),
poTokenGenerator.generatePoToken("i_SsnRdgitA"),
Pair::new
))
.subscribe(
pots -> Log.e(TAG, "success! " + pots.getSecond().poToken +
",web.gvs+" + pots.getFirst().poToken +
";visitor_data=" + pots.getFirst().visitorData),
error -> Log.e(TAG, "error", error)
));
}
@Override

View file

@ -0,0 +1,3 @@
package org.schabi.newpipe.util.potoken
class PoTokenException(message: String) : Exception(message)

View file

@ -0,0 +1,26 @@
package org.schabi.newpipe.util.potoken
import android.content.Context
import io.reactivex.rxjava3.core.Single
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
import java.io.Closeable
interface PoTokenGenerator : Closeable {
/**
* Generates a poToken for the provided identifier, using the `integrityToken` and
* `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be
* called multiple times.
*/
fun generatePoToken(identifier: String): Single<PoTokenResult>
interface Factory {
/**
* Initializes a [PoTokenGenerator] by loading the BotGuard VM, running it, and obtaining
* an `integrityToken`. Can then be used multiple times to generate multiple poTokens with
* [generatePoToken].
*
* @param context used e.g. to load the HTML asset or to instantiate a WebView
*/
fun newPoTokenGenerator(context: Context): Single<PoTokenGenerator>
}
}

View file

@ -0,0 +1,297 @@
package org.schabi.newpipe.util.potoken
import android.content.Context
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.webkit.JavascriptInterface
import android.webkit.WebView
import androidx.annotation.MainThread
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
import io.reactivex.rxjava3.core.Single
import io.reactivex.rxjava3.core.SingleEmitter
import io.reactivex.rxjava3.disposables.CompositeDisposable
import io.reactivex.rxjava3.schedulers.Schedulers
import org.schabi.newpipe.DownloaderImpl
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
class PoTokenWebView private constructor(
context: Context,
// to be used exactly once only during initialization!
private val generatorEmitter: SingleEmitter<PoTokenGenerator>,
) : PoTokenGenerator {
private val webView = WebView(context)
private val disposables = CompositeDisposable() // used only during initialization
private val poTokenEmitters = mutableListOf<Pair<String, SingleEmitter<PoTokenResult>>>()
//region Initialization
init {
val webviewSettings = webView.settings
//noinspection SetJavaScriptEnabled we want to use JavaScript!
webviewSettings.javaScriptEnabled = true
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
webviewSettings.safeBrowsingEnabled = false
}
webviewSettings.userAgentString = USER_AGENT
webviewSettings.blockNetworkLoads = true // the WebView does not need internet access
// so that we can run async functions and get back the result
webView.addJavascriptInterface(this, "PoTokenWebView")
}
/**
* Must be called right after instantiating [PoTokenWebView] to perform the actual
* initialization. This will asynchronously go through all the steps needed to load BotGuard,
* run it, and obtain an `integrityToken`.
*/
private fun loadHtmlAndObtainBotguard(context: Context) {
disposables.add(
Single.fromCallable {
val html = context.assets.open("po_token.html").bufferedReader()
.use { it.readText() }
return@fromCallable html
}
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(
{ html ->
webView.loadDataWithBaseURL(
"https://www.youtube.com",
html,
"text/html",
"utf-8",
null,
)
downloadAndRunBotguard()
},
this::onInitializationErrorCloseAndCancel
)
)
}
/**
* Called during initialization after the WebView content has been loaded.
*/
private fun downloadAndRunBotguard() {
makeJnnPaGoogleapisRequest(
"https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/Create",
"[ \"$REQUEST_KEY\" ]",
) { responseBody ->
webView.evaluateJavascript(
"""(async function() {
try {
data = JSON.parse(String.raw`$responseBody`)
result = await runBotGuard(data)
globalThis.webPoSignalOutput = result.webPoSignalOutput
PoTokenWebView.onRunBotguardResult(result.botguardResponse)
} catch (error) {
PoTokenWebView.onJsInitializationError(error.toString())
}
})();""",
) {}
}
}
/**
* Called during initialization by the JavaScript snippets from either
* [downloadAndRunBotguard] or [onRunBotguardResult].
*/
@JavascriptInterface
fun onJsInitializationError(error: String) {
Log.e(TAG, "Initialization error from JavaScript: $error")
onInitializationErrorCloseAndCancel(PoTokenException(error))
}
/**
* Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after
* obtaining the BotGuard execution output [botguardResponse].
*/
@JavascriptInterface
fun onRunBotguardResult(botguardResponse: String) {
Log.e(TAG, "botguardResponse: $botguardResponse")
makeJnnPaGoogleapisRequest(
"https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT",
"[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
) { responseBody ->
webView.evaluateJavascript(
"""(async function() {
try {
globalThis.integrityToken = JSON.parse(String.raw`$responseBody`)
PoTokenWebView.onInitializationFinished()
} catch (error) {
PoTokenWebView.onJsInitializationError(error.toString())
}
})();""",
) {}
}
}
/**
* Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the
* `integrityToken` has been received by JavaScript.
*/
@JavascriptInterface
fun onInitializationFinished() {
generatorEmitter.onSuccess(this)
}
//endregion
//region Obtaining poTokens
/**
* Adds the ([identifier], [emitter]) pair to the [poTokenEmitters] list. This makes it so that
* multiple poToken requests can be generated invparallel, and the results will be notified to
* the right emitters.
*/
private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter<PoTokenResult>) {
synchronized(poTokenEmitters) {
poTokenEmitters.add(Pair(identifier, emitter))
}
}
/**
* Extracts and removes from the [poTokenEmitters] list a [SingleEmitter] based on its
* [identifier]. The emitter is supposed to be used immediately after to either signal a success
* or an error.
*/
private fun popPoTokenEmitter(identifier: String): SingleEmitter<PoTokenResult>? {
return synchronized(poTokenEmitters) {
poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let {
poTokenEmitters.removeAt(it).second
}
}
}
@MainThread
override fun generatePoToken(identifier: String): Single<PoTokenResult> =
Single.create { emitter ->
addPoTokenEmitter(identifier, emitter)
webView.evaluateJavascript(
"""(async function() {
identifier = String.raw`$identifier`
try {
poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier)
PoTokenWebView.onObtainPoTokenResult(identifier, poToken)
} catch (error) {
PoTokenWebView.onObtainPoTokenError(identifier, error.toString())
}
})();""",
) {}
}
/**
* Called by the JavaScript snippet from [generatePoToken] when an error occurs in calling the
* JavaScript `obtainPoToken()` function.
*/
@JavascriptInterface
fun onObtainPoTokenError(identifier: String, error: String) {
Log.e(TAG, "obtainPoToken error from JavaScript: $error")
popPoTokenEmitter(identifier)?.onError(PoTokenException(error))
}
/**
* Called by the JavaScript snippet from [generatePoToken] with the original identifier and the
* result of the JavaScript `obtainPoToken()` function.
*/
@JavascriptInterface
fun onObtainPoTokenResult(identifier: String, poToken: String) {
Log.e(TAG, "identifier=$identifier")
Log.e(TAG, "poToken=$poToken")
popPoTokenEmitter(identifier)?.onSuccess(PoTokenResult(identifier, poToken))
}
//endregion
//region Utils
/**
* Makes a POST request to [url] with the given [data] by setting the correct headers. Calls
* [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response
* does not have HTTP code 200, therefore this is supposed to be used only during
* initialization. Calls [handleResponseBody] with the response body if the response is
* successful. The request is performed in the background and a disposable is added to
* [disposables].
*/
private fun makeJnnPaGoogleapisRequest(
url: String,
data: String,
handleResponseBody: (String) -> Unit,
) {
disposables.add(
Single.fromCallable {
return@fromCallable DownloaderImpl.getInstance().post(
url,
mapOf(
// replace the downloader user agent
"User-Agent" to listOf(USER_AGENT),
"Accept" to listOf("application/json"),
"Content-Type" to listOf("application/json+protobuf"),
"x-goog-api-key" to listOf(GOOGLE_API_KEY),
"x-user-agent" to listOf("grpc-web-javascript/0.1"),
),
data.toByteArray()
)
}
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(
{ response ->
val httpCode = response.responseCode()
if (httpCode != 200) {
onInitializationErrorCloseAndCancel(
PoTokenException("Invalid response code: $httpCode")
)
return@subscribe
}
val responseBody = response.responseBody()
handleResponseBody(responseBody)
},
this::onInitializationErrorCloseAndCancel
)
)
}
/**
* Handles any error happening during initialization, releasing resources and sending the error
* to [generatorEmitter].
*/
private fun onInitializationErrorCloseAndCancel(error: Throwable) {
Handler(Looper.getMainLooper()).post {
close()
generatorEmitter.onError(error)
}
}
/**
* Releases all [webView] and [disposables] resources.
*/
override fun close() {
disposables.dispose()
webView.clearHistory()
// clears RAM cache and disk cache (globally for all WebViews)
webView.clearCache(true)
// ensures that the WebView isn't doing anything when destroying it
webView.loadUrl("about:blank")
webView.onPause()
webView.removeAllViews()
webView.destroy()
}
//endregion
companion object : PoTokenGenerator.Factory {
private val TAG = PoTokenWebView::class.simpleName
private const val GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw"
private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo"
private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3"
@MainThread
override fun newPoTokenGenerator(context: Context): Single<PoTokenGenerator> =
Single.create { emitter ->
val potWv = PoTokenWebView(context, emitter)
potWv.loadHtmlAndObtainBotguard(context)
emitter.setDisposable(potWv.disposables)
}
}
}