Connect poToken generation to extractor

app/src/main/java/org/schabi/newpipe/App.java

@@ -3,15 +3,7 @@ package org.schabi.newpipe;
 import android.app.Application;
 import android.content.Context;
 import android.content.SharedPreferences;
-import android.os.Build;
-import android.os.UserManager;
 import android.util.Log;
-import android.view.View;
-import android.webkit.JavascriptInterface;
-import android.webkit.WebSettings;
-import android.webkit.WebView;
-import android.webkit.WebViewClient;
-import android.widget.RelativeLayout;
 
 import androidx.annotation.NonNull;
 import androidx.core.app.NotificationChannelCompat;
@@ -25,8 +17,7 @@ import org.acra.config.CoreConfigurationBuilder;
 import org.schabi.newpipe.error.ReCaptchaActivity;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.services.youtube.PoTokenResult;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
 import org.schabi.newpipe.ktx.ExceptionUtils;
 import org.schabi.newpipe.settings.NewPipeSettings;
 import org.schabi.newpipe.util.BridgeStateSaverInitializer;
@@ -36,7 +27,7 @@ import org.schabi.newpipe.util.StateSaver;
 import org.schabi.newpipe.util.image.ImageStrategy;
 import org.schabi.newpipe.util.image.PicassoHelper;
 import org.schabi.newpipe.util.image.PreferredImageQuality;
-import org.schabi.newpipe.util.potoken.PoTokenWebView;
+import org.schabi.newpipe.util.potoken.PoTokenProviderImpl;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
@@ -44,16 +35,12 @@ import java.net.SocketException;
 import java.util.List;
 import java.util.Objects;
 
-import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
-import io.reactivex.rxjava3.core.Single;
-import io.reactivex.rxjava3.disposables.CompositeDisposable;
 import io.reactivex.rxjava3.exceptions.CompositeException;
 import io.reactivex.rxjava3.exceptions.MissingBackpressureException;
 import io.reactivex.rxjava3.exceptions.OnErrorNotImplementedException;
 import io.reactivex.rxjava3.exceptions.UndeliverableException;
 import io.reactivex.rxjava3.functions.Consumer;
 import io.reactivex.rxjava3.plugins.RxJavaPlugins;
-import kotlin.Pair;
 
 /*
  * Copyright (C) Hans-Christoph Steiner 2016 <hans@eds.org>
@@ -134,22 +121,7 @@ public class App extends Application {
 
         configureRxJavaErrorHandler();
 
-        CompositeDisposable disposable = new CompositeDisposable();
+        YoutubeStreamExtractor.setPoTokenProvider(PoTokenProviderImpl.INSTANCE);
-        disposable.add(PoTokenWebView.Companion.newPoTokenGenerator(this)
-                .observeOn(AndroidSchedulers.mainThread())
-                .subscribeOn(AndroidSchedulers.mainThread())
-                .flatMap(poTokenGenerator -> Single.zip(
-                        poTokenGenerator.generatePoToken(YoutubeParsingHelper
-                                .randomVisitorData(NewPipe.getPreferredContentCountry())),
-                        poTokenGenerator.generatePoToken("i_SsnRdgitA"),
-                        Pair::new
-                ))
-                .subscribe(
-                        pots -> Log.e(TAG, "success! " + pots.getSecond().poToken +
-                                ",web.gvs+" + pots.getFirst().poToken +
-                                ";visitor_data=" + pots.getFirst().visitorData),
-                        error -> Log.e(TAG, "error", error)
-                ));
     }
 
     @Override

app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java

@@ -17,6 +17,7 @@ import android.view.InputDevice;
 import android.view.KeyEvent;
 import android.view.WindowInsets;
 import android.view.WindowManager;
+import android.webkit.CookieManager;
 
 import androidx.annotation.Dimension;
 import androidx.annotation.NonNull;
@@ -335,4 +336,17 @@ public final class DeviceUtils {
                 && !TX_50JXW834
                 && !HMB9213NW;
     }
+
+    /**
+     * @return whether the device has support for WebView, see
+     * <a href="https://stackoverflow.com/a/69626735">https://stackoverflow.com/a/69626735</a>
+     */
+    public static boolean supportsWebView() {
+        try {
+            CookieManager.getInstance();
+            return true;
+        } catch (Throwable ignored) {
+            return false;
+        }
+    }
 }

app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt

@@ -2,16 +2,25 @@ package org.schabi.newpipe.util.potoken
 
 import android.content.Context
 import io.reactivex.rxjava3.core.Single
-import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
 import java.io.Closeable
 
+/**
+ * This interface was created to allow for multiple methods to generate poTokens in the future (e.g.
+ * via WebView and via a local DOM implementation)
+ */
 interface PoTokenGenerator : Closeable {
     /**
      * Generates a poToken for the provided identifier, using the `integrityToken` and
      * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be
      * called multiple times.
      */
-    fun generatePoToken(identifier: String): Single<PoTokenResult>
+    fun generatePoToken(identifier: String): Single<String>
+
+    /**
+     * @return whether the `integrityToken` is expired, in which case all tokens generated by
+     * [generatePoToken] will be invalid
+     */
+    fun isExpired(): Boolean
 
     interface Factory {
         /**

app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt

@@ -0,0 +1,59 @@
+package org.schabi.newpipe.util.potoken
+
+import android.util.Log
+import org.schabi.newpipe.App
+import org.schabi.newpipe.extractor.NewPipe
+import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider
+import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
+import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
+import org.schabi.newpipe.util.DeviceUtils
+
+object PoTokenProviderImpl : PoTokenProvider {
+    val TAG = PoTokenProviderImpl::class.simpleName
+    private val webViewSupported by lazy { DeviceUtils.supportsWebView() }
+
+    private object WebPoTokenGenLock
+    private var webPoTokenVisitorData: String? = null
+    private var webPoTokenStreamingPot: String? = null
+    private var webPoTokenGenerator: PoTokenGenerator? = null
+
+    override fun getWebClientPoToken(videoId: String): PoTokenResult? {
+        if (!webViewSupported) {
+            return null
+        }
+
+        val (poTokenGenerator, visitorData, streamingPot) = synchronized(WebPoTokenGenLock) {
+            if (webPoTokenGenerator == null || webPoTokenGenerator!!.isExpired()) {
+                webPoTokenGenerator = PoTokenWebView.newPoTokenGenerator(App.getApp()).blockingGet()
+                webPoTokenVisitorData = YoutubeParsingHelper
+                    .randomVisitorData(NewPipe.getPreferredContentCountry())
+
+                // The streaming poToken needs to be generated exactly once before generating any
+                // other (player) tokens.
+                webPoTokenStreamingPot = webPoTokenGenerator!!
+                    .generatePoToken(webPoTokenVisitorData!!).blockingGet()
+            }
+            return@synchronized Triple(
+                webPoTokenGenerator!!, webPoTokenVisitorData!!, webPoTokenStreamingPot!!
+            )
+        }
+
+        // Not using synchronized here, since poTokenGenerator would be able to generate multiple
+        // poTokens in parallel if needed. The only important thing is for exactly one
+        // visitorData/streaming poToken to be generated before anything else.
+        val playerPot = poTokenGenerator.generatePoToken(videoId).blockingGet()
+        Log.e(TAG, "success($videoId) $playerPot,web.gvs+$streamingPot;visitor_data=$visitorData")
+
+        return PoTokenResult(
+            webPoTokenVisitorData!!,
+            playerPot,
+            webPoTokenStreamingPot!!,
+        )
+    }
+
+    override fun getWebEmbedClientPoToken(videoId: String): PoTokenResult? = null
+
+    override fun getAndroidClientPoToken(videoId: String): PoTokenResult? = null
+
+    override fun getIosClientPoToken(videoId: String): PoTokenResult? = null
+}

app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt

@@ -7,14 +7,13 @@ import android.os.Looper
 import android.util.Log
 import android.webkit.JavascriptInterface
 import android.webkit.WebView
-import androidx.annotation.MainThread
 import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
 import io.reactivex.rxjava3.core.Single
 import io.reactivex.rxjava3.core.SingleEmitter
 import io.reactivex.rxjava3.disposables.CompositeDisposable
 import io.reactivex.rxjava3.schedulers.Schedulers
 import org.schabi.newpipe.DownloaderImpl
-import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
+import java.time.Instant
 
 class PoTokenWebView private constructor(
     context: Context,
@@ -23,7 +22,8 @@ class PoTokenWebView private constructor(
 ) : PoTokenGenerator {
     private val webView = WebView(context)
     private val disposables = CompositeDisposable() // used only during initialization
-    private val poTokenEmitters = mutableListOf<Pair<String, SingleEmitter<PoTokenResult>>>()
+    private val poTokenEmitters = mutableListOf<Pair<String, SingleEmitter<String>>>()
+    private lateinit var expirationInstant: Instant
 
     //region Initialization
     init {
@@ -114,11 +114,12 @@ class PoTokenWebView private constructor(
             "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT",
             "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
         ) { responseBody ->
+            Log.e(TAG, "GenerateIT response: $responseBody")
             webView.evaluateJavascript(
                 """(async function() {
                     try {
                         globalThis.integrityToken = JSON.parse(String.raw`$responseBody`)
-                        PoTokenWebView.onInitializationFinished()
+                        PoTokenWebView.onInitializationFinished(integrityToken[1])
                     } catch (error) {
                         PoTokenWebView.onJsInitializationError(error.toString())
                     }
@@ -130,9 +131,14 @@ class PoTokenWebView private constructor(
     /**
      * Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the
      * `integrityToken` has been received by JavaScript.
+     *
+     * @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found
+     * in `integrityToken[1]`
      */
     @JavascriptInterface
-    fun onInitializationFinished() {
+    fun onInitializationFinished(expirationTimeInSeconds: Long) {
+        // leave 10 minutes of margin just to be sure
+        expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600)
         generatorEmitter.onSuccess(this)
     }
     //endregion
@@ -143,7 +149,7 @@ class PoTokenWebView private constructor(
      * multiple poToken requests can be generated invparallel, and the results will be notified to
      * the right emitters.
      */
-    private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter<PoTokenResult>) {
+    private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter<String>) {
         synchronized(poTokenEmitters) {
             poTokenEmitters.add(Pair(identifier, emitter))
         }
@@ -154,7 +160,7 @@ class PoTokenWebView private constructor(
      * [identifier]. The emitter is supposed to be used immediately after to either signal a success
      * or an error.
      */
-    private fun popPoTokenEmitter(identifier: String): SingleEmitter<PoTokenResult>? {
+    private fun popPoTokenEmitter(identifier: String): SingleEmitter<String>? {
         return synchronized(poTokenEmitters) {
             poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let {
                 poTokenEmitters.removeAt(it).second
@@ -162,22 +168,23 @@ class PoTokenWebView private constructor(
         }
     }
 
-    @MainThread
+    override fun generatePoToken(identifier: String): Single<String> =
-    override fun generatePoToken(identifier: String): Single<PoTokenResult> =
         Single.create { emitter ->
             addPoTokenEmitter(identifier, emitter)
-
+            Handler(Looper.getMainLooper()).post {
-            webView.evaluateJavascript(
+                webView.evaluateJavascript(
-                """(async function() {
+                    """(async function() {
-                    identifier = String.raw`$identifier`
+                        identifier = String.raw`$identifier`
-                    try {
+                        try {
-                        poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier)
+                            poToken = await obtainPoToken(webPoSignalOutput, integrityToken,
-                        PoTokenWebView.onObtainPoTokenResult(identifier, poToken)
+                                identifier)
-                    } catch (error) {
+                            PoTokenWebView.onObtainPoTokenResult(identifier, poToken)
-                        PoTokenWebView.onObtainPoTokenError(identifier, error.toString())
+                        } catch (error) {
-                    }
+                            PoTokenWebView.onObtainPoTokenError(identifier, error.toString())
-                })();""",
+                        }
-            ) {}
+                    })();""",
+                ) {}
+            }
         }
 
     /**
@@ -198,7 +205,11 @@ class PoTokenWebView private constructor(
     fun onObtainPoTokenResult(identifier: String, poToken: String) {
         Log.e(TAG, "identifier=$identifier")
         Log.e(TAG, "poToken=$poToken")
-        popPoTokenEmitter(identifier)?.onSuccess(PoTokenResult(identifier, poToken))
+        popPoTokenEmitter(identifier)?.onSuccess(poToken)
+    }
+
+    override fun isExpired(): Boolean {
+        return Instant.now().isAfter(expirationInstant)
     }
     //endregion
 
@@ -286,12 +297,13 @@ class PoTokenWebView private constructor(
         private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo"
         private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3"
 
-        @MainThread
         override fun newPoTokenGenerator(context: Context): Single<PoTokenGenerator> =
             Single.create { emitter ->
-                val potWv = PoTokenWebView(context, emitter)
+                Handler(Looper.getMainLooper()).post {
-                potWv.loadHtmlAndObtainBotguard(context)
+                    val potWv = PoTokenWebView(context, emitter)
-                emitter.setDisposable(potWv.disposables)
+                    potWv.loadHtmlAndObtainBotguard(context)
+                    emitter.setDisposable(potWv.disposables)
+                }
             }
     }
 }