merge latest changes from upstream

build.gradle.kts

@@ -10,7 +10,7 @@ plugins {
 allprojects {
     apply(plugin = "java-library")
 
-    version = "v0.26.0"
+    version = "v0.26.2"
 
     tasks.withType<JavaCompile> {
         options.encoding = Charsets.UTF_8.toString()

extractor/build.gradle.kts

@@ -191,10 +191,7 @@ publishing {
 }
 
 signing {
+    setRequired(shouldSignCIRelease)
     useInMemoryPgpKeys(ciSigningKey, ciSigningPassword)
     sign(publishing.publications["snapshot"])
 }
-
-tasks.withType<Sign> {
-    onlyIf("Signing credentials are present (only used for maven central)") { shouldSignCIRelease }
-}

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -7,6 +7,7 @@
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -15,6 +16,8 @@
 import java.util.Objects;
 
 public abstract class Extractor {
+    private final String TAG = getClass().getSimpleName() + "@" + hashCode();
+
     /**
      * {@link StreamingService} currently related to this extractor.<br>
      * Useful for getting other things from a service (like the url handlers for
@@ -54,7 +57,9 @@ public LinkHandler getLinkHandler() {
      * @throws ExtractionException if the pages content is not understood
      */
     public void fetchPage() throws IOException, ExtractionException {
+        ExtractorLogger.d(TAG, "base fetchPage called");
         if (pageFetched) {
+            ExtractorLogger.d(TAG, "Page already fetched; returning");
             return;
         }
         onFetchPage(downloader);
@@ -151,4 +156,9 @@ public ContentCountry getExtractorContentCountry() {
     public TimeAgoParser getTimeAgoParser() {
         return getService().getTimeAgoParser(getExtractorLocalization());
     }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/Info.java

@@ -2,6 +2,7 @@
 
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -10,6 +11,7 @@
 
 public abstract class Info implements Serializable {
 
+    private static final String TAG = "Info";
     private final int serviceId;
     /**
      * Id of this Info object <br>
@@ -52,6 +54,7 @@ public Info(final int serviceId,
         this.url = url;
         this.originalUrl = originalUrl;
         this.name = name;
+        ExtractorLogger.d(TAG, "Base Created {}", this);
     }
 
     public Info(final int serviceId, final LinkHandler linkHandler, final String name) {

extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java

@@ -24,6 +24,7 @@
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 
 import java.util.List;
 
@@ -34,6 +35,7 @@
  * Provides access to streaming services supported by NewPipe.
  */
 public final class NewPipe {
+    private static final String TAG = NewPipe.class.getSimpleName();
     private static Downloader downloader;
     private static Localization preferredLocalization;
     private static ContentCountry preferredContentCountry;
@@ -42,15 +44,19 @@ private NewPipe() {
     }
 
     public static void init(final Downloader d) {
+        ExtractorLogger.d(TAG, "Default init called");
         init(d, Localization.DEFAULT);
     }
 
     public static void init(final Downloader d, final Localization l) {
+        ExtractorLogger.d(TAG, "Default init called with localization={}");
         init(d, l, l.getCountryCode().isEmpty()
                 ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode()));
     }
 
     public static void init(final Downloader d, final Localization l, final ContentCountry c) {
+        ExtractorLogger.d(TAG, "Initializing with downloader={}, localization={}, country={}",
+                          d, l, c);
         downloader = d;
         preferredLocalization = l;
         preferredContentCountry = c;

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java

@@ -243,4 +243,9 @@ public Response postWithContentTypeJson(final String url,
      */
     public abstract Response execute(@Nonnull Request request)
             throws IOException, ReCaptchaException;
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampCommentsExtractor.java

@@ -114,9 +114,7 @@ private JsonObject fetchReviewsData(final String trackId, final String token)
     }
 
     private String getNextPageToken(final JsonArray reviews) throws ParsingException {
-        return reviews.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+        return reviews.streamAsJsonObjects()
                 .map(review -> review.getString("token"))
                 .reduce((a, b) -> b) // keep only the last element
                 .orElseThrow(() -> new ParsingException("Could not get token"));

extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampSuggestionExtractor.java

@@ -48,9 +48,8 @@ public List<String> suggestionList(final String query) throws IOException, Extra
                                     .done()
                                     .getBytes(StandardCharsets.UTF_8)).responseBody());
 
-            return fuzzyResults.getObject("auto").getArray("results").stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
+            return fuzzyResults.getObject("auto").getArray("results")
+                    .streamAsJsonObjects()
                     .map(jsonObject -> jsonObject.getString("name"))
                     .distinct()
                     .collect(Collectors.toList());

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCChannelTabExtractor.java

@@ -55,9 +55,7 @@ public ListExtractor.InfoItemsPage<InfoItem> getInitialPage() {
                 new MultiInfoItemsCollector(getServiceId());
         Objects.requireNonNull(conferenceData) // will surely be != null after onFetchPage
                 .getArray("events")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+                .streamAsJsonObjects()
                 .forEach(event -> collector.commit(new MediaCCCStreamInfoItemExtractor(event)));
         return new ListExtractor.InfoItemsPage<>(collector, null);
     }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCLiveStreamExtractor.java

@@ -145,9 +145,7 @@ public String getHlsUrl() {
 
     @Nonnull
     private String getManifestOfDeliveryMethodWanted(@Nonnull final String deliveryMethod) {
-        return room.getArray(STREAMS).stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+        return room.getArray(STREAMS).streamAsJsonObjects()
                 .map(streamObject -> streamObject.getObject(URLS))
                 .filter(urls -> urls.has(deliveryMethod))
                 .map(urls -> urls.getObject(deliveryMethod).getString(URL, ""))

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCRecentKiosk.java

@@ -57,9 +57,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
                 .reversed();
         final var collector = new StreamInfoItemsCollector(getServiceId(), comparator);
 
-        events.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+        events.streamAsJsonObjects()
                 .map(MediaCCCRecentKioskExtractor::new)
                 // #813 / voc/voctoweb#609 -> returns faulty data -> filter it out
                 .filter(extractor -> extractor.getDuration() > 0)

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/PeertubeParsingHelper.java

@@ -293,9 +293,7 @@ private static List<Image> getImagesFromAvatarsOrBanners(
     private static List<Image> getImagesFromAvatarOrBannerArray(
             @Nonnull final String baseUrl,
             @Nonnull final JsonArray avatarsOrBannersArray) {
-        return avatarsOrBannersArray.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+        return avatarsOrBannersArray.streamAsJsonObjects()
                 .filter(image -> !isNullOrEmpty(image.getString("path")))
                 .map(image -> new Image(baseUrl + image.getString("path"), HEIGHT_UNKNOWN,
                         image.getInt("width", WIDTH_UNKNOWN), ResolutionLevel.UNKNOWN))

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeStreamExtractor.java

@@ -480,10 +480,7 @@ private void loadSubtitles() {
 
     private void extractLiveVideoStreams() throws ParsingException {
         try {
-            final JsonArray streamingPlaylists = json.getArray(STREAMING_PLAYLISTS);
-            streamingPlaylists.stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
+            json.getArray(STREAMING_PLAYLISTS).streamAsJsonObjects()
                     .map(stream -> new VideoStream.Builder()
                             .setId(String.valueOf(stream.getInt("id", -1)))
                             .setContent(stream.getString(PLAYLIST_URL, ""), true)
@@ -507,10 +504,10 @@ private void getStreams() throws ParsingException {
 
         // HLS streams
         try {
-            for (final JsonObject playlist : json.getArray(STREAMING_PLAYLISTS).stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
-                    .collect(Collectors.toList())) {
+            final var playlistStream = json.getArray(STREAMING_PLAYLISTS).streamAsJsonObjects();
+            final var it = playlistStream.iterator();
+            while (it.hasNext()) {
+                final var playlist = it.next();
                 getStreamsFromArray(playlist.getArray(FILES), playlist.getString(PLAYLIST_URL));
             }
         } catch (final Exception e) {
@@ -530,11 +527,10 @@ private void getStreamsFromArray(@Nonnull final JsonArray streams,
             */
             final boolean isInstanceUsingRandomUuidsForHlsStreams = !isNullOrEmpty(playlistUrl)
                     && playlistUrl.endsWith("-master.m3u8");
-
-            for (final JsonObject stream : streams.stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
-                    .collect(Collectors.toList())) {
+            final var streamOfStreams = streams.streamAsJsonObjects();
+            final var it = streamOfStreams.iterator();
+            while (it.hasNext()) {
+                final var stream = it.next();
 
                 // Extract stream version of streams first
                 final String url = JsonUtils.getString(stream,

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -383,9 +383,7 @@ public static String getInfoItemsFromApi(final MultiInfoItemsCollector collector
         }
 
         responseObject.getArray("collection")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+                .streamAsJsonObjects()
                 .forEach(searchResult -> {
                     final String kind = searchResult.getString("kind", "");
                     switch (kind) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java

@@ -41,7 +41,7 @@ public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionExcepti
     public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException,
             IOException {
         if (page == null || isNullOrEmpty(page.getUrl())) {
-            throw new IllegalArgumentException("Page doesn't contain an URL");
+            return InfoItemsPage.emptyPage();
         }
         return getPage(page.getUrl());
     }
@@ -63,7 +63,8 @@ private InfoItemsPage<CommentsInfoItem> getPage(@Nonnull final String url)
                 getServiceId());
 
         collectStreamsFrom(collector, json.getArray("collection"));
-        return new InfoItemsPage<>(collector, new Page(json.getString("next_href", null)));
+        final String nextHref = json.getString("next_href");
+        return new InfoItemsPage<>(collector, isNullOrEmpty(nextHref) ? null : new Page(nextHref));
     }
 
     @Override

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudPlaylistExtractor.java

@@ -140,9 +140,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() {
         final List<String> ids = new ArrayList<>();
 
         playlist.getArray("tracks")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+                .streamAsJsonObjects()
                 .forEachOrdered(track -> {
                     // i.e. if full info is available
                     if (track.has("title")) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java

@@ -204,9 +204,7 @@ private String getTranscodingUrl(final String endpointUrl)
 
     private void extractAudioStreams(@Nonnull final JsonArray transcodings,
                                      final List<AudioStream> audioStreams) {
-        transcodings.stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+        transcodings.streamAsJsonObjects()
                 .forEachOrdered(transcoding -> {
                     final String url = transcoding.getString("url");
                     if (isNullOrEmpty(url)) {

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java

@@ -344,11 +344,8 @@ public static ChannelHeader getChannelHeader(
                     .map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED))
                     .orElse(null);
         } else if (header.has(CAROUSEL_HEADER_RENDERER)) {
-            return header.getObject(CAROUSEL_HEADER_RENDERER)
-                    .getArray(CONTENTS)
-                    .stream()
-                    .filter(JsonObject.class::isInstance)
-                    .map(JsonObject.class::cast)
+            return header.getObject(CAROUSEL_HEADER_RENDERER).getArray(CONTENTS)
+                    .streamAsJsonObjects()
                     .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
                     .findFirst()
                     .map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
@@ -465,9 +462,7 @@ public static String getChannelId(
                     final String navigationCarouselChannelId = channelHeader.json.getObject(HEADER)
                             .getObject(CAROUSEL_HEADER_RENDERER)
                             .getArray(CONTENTS)
-                            .stream()
-                            .filter(JsonObject.class::isInstance)
-                            .map(JsonObject.class::cast)
+                            .streamAsJsonObjects()
                             .filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
                             .findFirst()
                             .orElse(new JsonObject())
@@ -551,16 +546,12 @@ public static JsonObject getChannelAgeGateRenderer(@Nonnull final JsonObject jso
         return jsonResponse.getObject(CONTENTS)
                 .getObject("twoColumnBrowseResultsRenderer")
                 .getArray("tabs")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+                .streamAsJsonObjects()
                 .flatMap(tab -> tab.getObject(TAB_RENDERER)
                         .getObject(CONTENT)
                         .getObject("sectionListRenderer")
                         .getArray(CONTENTS)
-                        .stream()
-                        .filter(JsonObject.class::isInstance)
-                        .map(JsonObject.class::cast))
+                        .streamAsJsonObjects())
                 .filter(content -> content.has("channelAgeGateRenderer"))
                 .map(content -> content.getObject("channelAgeGateRenderer"))
                 .findFirst()

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeDescriptionHelper.java

@@ -228,9 +228,7 @@ private static void addAllCommandRuns(
             @Nonnull final List<Run> closers
     ) {
         attributedDescription.getArray("commandRuns")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+                .streamAsJsonObjects()
                 .forEach(run -> {
                     final JsonObject navigationEndpoint = run.getObject("onTap")
                             .getObject("innertubeCommand");
@@ -301,9 +299,7 @@ private static void addAllStyleRuns(
             @Nonnull final List<Run> closers
     ) {
         attributedDescription.getArray("styleRuns")
-                .stream()
-                .filter(JsonObject.class::isInstance)
-                .map(JsonObject.class::cast)
+                .streamAsJsonObjects()
                 .forEach(run -> {
                     final int start = run.getInt("startIndex", -1);
                     final int length = run.getInt("length", 0);