diff --git a/app/src/main/java/org/schabi/newpipe/App.java b/app/src/main/java/org/schabi/newpipe/App.java index 74f2c687b..8ce161eec 100644 --- a/app/src/main/java/org/schabi/newpipe/App.java +++ b/app/src/main/java/org/schabi/newpipe/App.java @@ -3,15 +3,7 @@ package org.schabi.newpipe; import android.app.Application; import android.content.Context; import android.content.SharedPreferences; -import android.os.Build; -import android.os.UserManager; import android.util.Log; -import android.view.View; -import android.webkit.JavascriptInterface; -import android.webkit.WebSettings; -import android.webkit.WebView; -import android.webkit.WebViewClient; -import android.widget.RelativeLayout; import androidx.annotation.NonNull; import androidx.core.app.NotificationChannelCompat; @@ -25,8 +17,7 @@ import org.acra.config.CoreConfigurationBuilder; import org.schabi.newpipe.error.ReCaptchaActivity; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.downloader.Downloader; -import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; -import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.ktx.ExceptionUtils; import org.schabi.newpipe.settings.NewPipeSettings; import org.schabi.newpipe.util.BridgeStateSaverInitializer; @@ -36,7 +27,7 @@ import org.schabi.newpipe.util.StateSaver; import org.schabi.newpipe.util.image.ImageStrategy; import org.schabi.newpipe.util.image.PicassoHelper; import org.schabi.newpipe.util.image.PreferredImageQuality; -import org.schabi.newpipe.util.potoken.PoTokenWebView; +import org.schabi.newpipe.util.potoken.PoTokenProviderImpl; import java.io.IOException; import java.io.InterruptedIOException; @@ -44,16 +35,12 @@ import java.net.SocketException; import java.util.List; import java.util.Objects; -import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; -import io.reactivex.rxjava3.core.Single; -import io.reactivex.rxjava3.disposables.CompositeDisposable; import io.reactivex.rxjava3.exceptions.CompositeException; import io.reactivex.rxjava3.exceptions.MissingBackpressureException; import io.reactivex.rxjava3.exceptions.OnErrorNotImplementedException; import io.reactivex.rxjava3.exceptions.UndeliverableException; import io.reactivex.rxjava3.functions.Consumer; import io.reactivex.rxjava3.plugins.RxJavaPlugins; -import kotlin.Pair; /* * Copyright (C) Hans-Christoph Steiner 2016 @@ -134,22 +121,7 @@ public class App extends Application { configureRxJavaErrorHandler(); - CompositeDisposable disposable = new CompositeDisposable(); - disposable.add(PoTokenWebView.Companion.newPoTokenGenerator(this) - .observeOn(AndroidSchedulers.mainThread()) - .subscribeOn(AndroidSchedulers.mainThread()) - .flatMap(poTokenGenerator -> Single.zip( - poTokenGenerator.generatePoToken(YoutubeParsingHelper - .randomVisitorData(NewPipe.getPreferredContentCountry())), - poTokenGenerator.generatePoToken("i_SsnRdgitA"), - Pair::new - )) - .subscribe( - pots -> Log.e(TAG, "success! " + pots.getSecond().poToken + - ",web.gvs+" + pots.getFirst().poToken + - ";visitor_data=" + pots.getFirst().visitorData), - error -> Log.e(TAG, "error", error) - )); + YoutubeStreamExtractor.setPoTokenProvider(PoTokenProviderImpl.INSTANCE); } @Override diff --git a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java index e9678c2b0..a037e48d3 100644 --- a/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java +++ b/app/src/main/java/org/schabi/newpipe/util/DeviceUtils.java @@ -17,6 +17,7 @@ import android.view.InputDevice; import android.view.KeyEvent; import android.view.WindowInsets; import android.view.WindowManager; +import android.webkit.CookieManager; import androidx.annotation.Dimension; import androidx.annotation.NonNull; @@ -335,4 +336,17 @@ public final class DeviceUtils { && !TX_50JXW834 && !HMB9213NW; } + + /** + * @return whether the device has support for WebView, see + * https://stackoverflow.com/a/69626735 + */ + public static boolean supportsWebView() { + try { + CookieManager.getInstance(); + return true; + } catch (Throwable ignored) { + return false; + } + } } diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt index c48afc5c7..6446ecc72 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenGenerator.kt @@ -2,16 +2,25 @@ package org.schabi.newpipe.util.potoken import android.content.Context import io.reactivex.rxjava3.core.Single -import org.schabi.newpipe.extractor.services.youtube.PoTokenResult import java.io.Closeable +/** + * This interface was created to allow for multiple methods to generate poTokens in the future (e.g. + * via WebView and via a local DOM implementation) + */ interface PoTokenGenerator : Closeable { /** * Generates a poToken for the provided identifier, using the `integrityToken` and * `webPoSignalOutput` previously obtained in the initialization of [PoTokenWebView]. Can be * called multiple times. */ - fun generatePoToken(identifier: String): Single + fun generatePoToken(identifier: String): Single + + /** + * @return whether the `integrityToken` is expired, in which case all tokens generated by + * [generatePoToken] will be invalid + */ + fun isExpired(): Boolean interface Factory { /** diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt new file mode 100644 index 000000000..a00e19f2d --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenProviderImpl.kt @@ -0,0 +1,59 @@ +package org.schabi.newpipe.util.potoken + +import android.util.Log +import org.schabi.newpipe.App +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper +import org.schabi.newpipe.util.DeviceUtils + +object PoTokenProviderImpl : PoTokenProvider { + val TAG = PoTokenProviderImpl::class.simpleName + private val webViewSupported by lazy { DeviceUtils.supportsWebView() } + + private object WebPoTokenGenLock + private var webPoTokenVisitorData: String? = null + private var webPoTokenStreamingPot: String? = null + private var webPoTokenGenerator: PoTokenGenerator? = null + + override fun getWebClientPoToken(videoId: String): PoTokenResult? { + if (!webViewSupported) { + return null + } + + val (poTokenGenerator, visitorData, streamingPot) = synchronized(WebPoTokenGenLock) { + if (webPoTokenGenerator == null || webPoTokenGenerator!!.isExpired()) { + webPoTokenGenerator = PoTokenWebView.newPoTokenGenerator(App.getApp()).blockingGet() + webPoTokenVisitorData = YoutubeParsingHelper + .randomVisitorData(NewPipe.getPreferredContentCountry()) + + // The streaming poToken needs to be generated exactly once before generating any + // other (player) tokens. + webPoTokenStreamingPot = webPoTokenGenerator!! + .generatePoToken(webPoTokenVisitorData!!).blockingGet() + } + return@synchronized Triple( + webPoTokenGenerator!!, webPoTokenVisitorData!!, webPoTokenStreamingPot!! + ) + } + + // Not using synchronized here, since poTokenGenerator would be able to generate multiple + // poTokens in parallel if needed. The only important thing is for exactly one + // visitorData/streaming poToken to be generated before anything else. + val playerPot = poTokenGenerator.generatePoToken(videoId).blockingGet() + Log.e(TAG, "success($videoId) $playerPot,web.gvs+$streamingPot;visitor_data=$visitorData") + + return PoTokenResult( + webPoTokenVisitorData!!, + playerPot, + webPoTokenStreamingPot!!, + ) + } + + override fun getWebEmbedClientPoToken(videoId: String): PoTokenResult? = null + + override fun getAndroidClientPoToken(videoId: String): PoTokenResult? = null + + override fun getIosClientPoToken(videoId: String): PoTokenResult? = null +} diff --git a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt index 626ebea45..008477790 100644 --- a/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt +++ b/app/src/main/java/org/schabi/newpipe/util/potoken/PoTokenWebView.kt @@ -7,14 +7,13 @@ import android.os.Looper import android.util.Log import android.webkit.JavascriptInterface import android.webkit.WebView -import androidx.annotation.MainThread import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Single import io.reactivex.rxjava3.core.SingleEmitter import io.reactivex.rxjava3.disposables.CompositeDisposable import io.reactivex.rxjava3.schedulers.Schedulers import org.schabi.newpipe.DownloaderImpl -import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import java.time.Instant class PoTokenWebView private constructor( context: Context, @@ -23,7 +22,8 @@ class PoTokenWebView private constructor( ) : PoTokenGenerator { private val webView = WebView(context) private val disposables = CompositeDisposable() // used only during initialization - private val poTokenEmitters = mutableListOf>>() + private val poTokenEmitters = mutableListOf>>() + private lateinit var expirationInstant: Instant //region Initialization init { @@ -114,11 +114,12 @@ class PoTokenWebView private constructor( "https://jnn-pa.googleapis.com/\$rpc/google.internal.waa.v1.Waa/GenerateIT", "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", ) { responseBody -> + Log.e(TAG, "GenerateIT response: $responseBody") webView.evaluateJavascript( """(async function() { try { globalThis.integrityToken = JSON.parse(String.raw`$responseBody`) - PoTokenWebView.onInitializationFinished() + PoTokenWebView.onInitializationFinished(integrityToken[1]) } catch (error) { PoTokenWebView.onJsInitializationError(error.toString()) } @@ -130,9 +131,14 @@ class PoTokenWebView private constructor( /** * Called during initialization by the JavaScript snippet from [onRunBotguardResult] when the * `integrityToken` has been received by JavaScript. + * + * @param expirationTimeInSeconds in how many seconds the integrity token expires, can be found + * in `integrityToken[1]` */ @JavascriptInterface - fun onInitializationFinished() { + fun onInitializationFinished(expirationTimeInSeconds: Long) { + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds - 600) generatorEmitter.onSuccess(this) } //endregion @@ -143,7 +149,7 @@ class PoTokenWebView private constructor( * multiple poToken requests can be generated invparallel, and the results will be notified to * the right emitters. */ - private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { + private fun addPoTokenEmitter(identifier: String, emitter: SingleEmitter) { synchronized(poTokenEmitters) { poTokenEmitters.add(Pair(identifier, emitter)) } @@ -154,7 +160,7 @@ class PoTokenWebView private constructor( * [identifier]. The emitter is supposed to be used immediately after to either signal a success * or an error. */ - private fun popPoTokenEmitter(identifier: String): SingleEmitter? { + private fun popPoTokenEmitter(identifier: String): SingleEmitter? { return synchronized(poTokenEmitters) { poTokenEmitters.indexOfFirst { it.first == identifier }.takeIf { it >= 0 }?.let { poTokenEmitters.removeAt(it).second @@ -162,22 +168,23 @@ class PoTokenWebView private constructor( } } - @MainThread - override fun generatePoToken(identifier: String): Single = + override fun generatePoToken(identifier: String): Single = Single.create { emitter -> addPoTokenEmitter(identifier, emitter) - - webView.evaluateJavascript( - """(async function() { - identifier = String.raw`$identifier` - try { - poToken = await obtainPoToken(webPoSignalOutput, integrityToken, identifier) - PoTokenWebView.onObtainPoTokenResult(identifier, poToken) - } catch (error) { - PoTokenWebView.onObtainPoTokenError(identifier, error.toString()) - } - })();""", - ) {} + Handler(Looper.getMainLooper()).post { + webView.evaluateJavascript( + """(async function() { + identifier = String.raw`$identifier` + try { + poToken = await obtainPoToken(webPoSignalOutput, integrityToken, + identifier) + PoTokenWebView.onObtainPoTokenResult(identifier, poToken) + } catch (error) { + PoTokenWebView.onObtainPoTokenError(identifier, error.toString()) + } + })();""", + ) {} + } } /** @@ -198,7 +205,11 @@ class PoTokenWebView private constructor( fun onObtainPoTokenResult(identifier: String, poToken: String) { Log.e(TAG, "identifier=$identifier") Log.e(TAG, "poToken=$poToken") - popPoTokenEmitter(identifier)?.onSuccess(PoTokenResult(identifier, poToken)) + popPoTokenEmitter(identifier)?.onSuccess(poToken) + } + + override fun isExpired(): Boolean { + return Instant.now().isAfter(expirationInstant) } //endregion @@ -286,12 +297,13 @@ class PoTokenWebView private constructor( private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" - @MainThread override fun newPoTokenGenerator(context: Context): Single = Single.create { emitter -> - val potWv = PoTokenWebView(context, emitter) - potWv.loadHtmlAndObtainBotguard(context) - emitter.setDisposable(potWv.disposables) + Handler(Looper.getMainLooper()).post { + val potWv = PoTokenWebView(context, emitter) + potWv.loadHtmlAndObtainBotguard(context) + emitter.setDisposable(potWv.disposables) + } } } }