Skip to content

Commit

Permalink
[YouTube] Fix serialization of Videos channel tab when already fetched
Browse files Browse the repository at this point in the history 
Also remove usage of Optional as fields as it is not a good practice. This
simplifies in some places channel info extraction code.
  • Loading branch information
@AudricV
AudricV committed Sep 15, 2024
1 parent 6e3a4a6 commit 422df70
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 96 deletions.
48 changes: 25 additions & 23 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.Optional;

Expand Down Expand Up @@ -233,7 +234,7 @@ private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject json
* properties.
* </p>
*/
public static final class ChannelHeader {
public static final class ChannelHeader implements Serializable {

/**
* Types of supported YouTube channel headers.
Expand Down Expand Up @@ -294,27 +295,27 @@ public enum HeaderType {
*/
public final HeaderType headerType;

private ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
public ChannelHeader(@Nonnull final JsonObject json, final HeaderType headerType) {
this.json = json;
this.headerType = headerType;
}
}

/**
* Get a channel header as an {@link Optional} it if exists.
* Get a channel header it if exists.
*
* @param channelResponse a full channel JSON response
* @return an {@link Optional} containing a {@link ChannelHeader} or an empty {@link Optional}
* if no supported header has been found
* @return a {@link ChannelHeader} or {@code null} if no supported header has been found
*/
@Nonnull
public static Optional<ChannelHeader> getChannelHeader(
@Nullable
public static ChannelHeader getChannelHeader(
@Nonnull final JsonObject channelResponse) {
final JsonObject header = channelResponse.getObject(HEADER);

if (header.has(C4_TABBED_HEADER_RENDERER)) {
return Optional.of(header.getObject(C4_TABBED_HEADER_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.C4_TABBED))
.orElse(null);
} else if (header.has(CAROUSEL_HEADER_RENDERER)) {
return header.getObject(CAROUSEL_HEADER_RENDERER)
.getArray(CONTENTS)
Expand All @@ -324,17 +325,20 @@ public static Optional<ChannelHeader> getChannelHeader(
.filter(item -> item.has(TOPIC_CHANNEL_DETAILS_RENDERER))
.findFirst()
.map(item -> item.getObject(TOPIC_CHANNEL_DETAILS_RENDERER))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.CAROUSEL))
.orElse(null);
} else if (header.has("pageHeaderRenderer")) {
return Optional.of(header.getObject("pageHeaderRenderer"))
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE));
.map(json -> new ChannelHeader(json, ChannelHeader.HeaderType.PAGE))
.orElse(null);
} else if (header.has("interactiveTabbedHeaderRenderer")) {
return Optional.of(header.getObject("interactiveTabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json,
ChannelHeader.HeaderType.INTERACTIVE_TABBED));
} else {
return Optional.empty();
ChannelHeader.HeaderType.INTERACTIVE_TABBED))
.orElse(null);
}

return null;
}

/**
Expand Down Expand Up @@ -418,20 +422,18 @@ public static boolean isChannelVerified(@Nonnull final ChannelHeader channelHead
* If the ID cannot still be get, the fallback channel ID, if provided, will be used.
* </p>
*
* @param header the channel header
* @param channelHeader the channel header
* @param fallbackChannelId the fallback channel ID, which can be null
* @return the ID of the channel
* @throws ParsingException if the channel ID cannot be got from the channel header, the
* channel response and the fallback channel ID
*/
@Nonnull
public static String getChannelId(
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> header,
@Nullable final ChannelHeader channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final String fallbackChannelId) throws ParsingException {
if (header.isPresent()) {
final ChannelHeader channelHeader = header.get();
if (channelHeader != null) {
switch (channelHeader.headerType) {
case C4_TABBED:
final String channelId = channelHeader.json.getObject(HEADER)
Expand Down Expand Up @@ -486,10 +488,9 @@ public static String getChannelId(
}

@Nonnull
public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull final Optional<ChannelHeader> channelHeader,
@Nonnull final JsonObject jsonResponse,
@Nullable final JsonObject channelAgeGateRenderer)
public static String getChannelName(@Nullable final ChannelHeader channelHeader,
@Nullable final JsonObject channelAgeGateRenderer,
@Nonnull final JsonObject jsonResponse)
throws ParsingException {
if (channelAgeGateRenderer != null) {
final String title = channelAgeGateRenderer.getString("channelTitle");
Expand All @@ -506,7 +507,8 @@ public static String getChannelName(@SuppressWarnings("OptionalUsedAsFieldOrPara
return metadataRendererTitle;
}

return channelHeader.map(header -> {
return Optional.ofNullable(channelHeader)
.map(header -> {
final JsonObject channelJson = header.json;
switch (header.headerType) {
case PAGE:
Expand Down
134 changes: 73 additions & 61 deletions ...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {

private JsonObject jsonResponse;

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private Optional<ChannelHeader> channelHeader;
@Nullable
private ChannelHeader channelHeader;

private String channelId;

Expand Down Expand Up @@ -132,7 +132,7 @@ public String getId() throws ParsingException {
public String getName() throws ParsingException {
assertPageFetched();
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse, channelAgeGateRenderer);
channelHeader, channelAgeGateRenderer, jsonResponse);
}

@Nonnull
Expand All @@ -146,40 +146,40 @@ public List<Image> getAvatars() throws ParsingException {
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}

return channelHeader.map(header -> {
switch (header.headerType) {
case PAGE:
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);

if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
}

if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
return Optional.ofNullable(channelHeader)
.map(header -> {
switch (header.headerType) {
case PAGE:
final JsonObject imageObj = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL)
.getObject(IMAGE);

if (imageObj.has(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)) {
return imageObj.getObject(CONTENT_PREVIEW_IMAGE_VIEW_MODEL)
.getObject(IMAGE)
.getArray(SOURCES);
}

if (imageObj.has("decoratedAvatarViewModel")) {
return imageObj.getObject("decoratedAvatarViewModel")
.getObject(AVATAR)
.getObject("avatarViewModel")
.getObject(IMAGE)
.getArray(SOURCES);
}

// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);
case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}

// Return an empty avatar array as a fallback
return new JsonArray();
case INTERACTIVE_TABBED:
return header.json.getObject("boxArt")
.getArray(THUMBNAILS);

case C4_TABBED:
case CAROUSEL:
default:
return header.json.getObject(AVATAR)
.getArray(THUMBNAILS);
}
})
})
.map(YoutubeParsingHelper::getImagesFromThumbnailsArray)
.orElseThrow(() -> new ParsingException("Could not get avatars"));
}
Expand All @@ -192,7 +192,8 @@ public List<Image> getBanners() {
return List.of();
}

return channelHeader.map(header -> {
return Optional.ofNullable(channelHeader)
.map(header -> {
if (header.headerType == HeaderType.PAGE) {
final JsonObject pageHeaderViewModel = header.json.getObject(CONTENT)
.getObject(PAGE_HEADER_VIEW_MODEL);
Expand Down Expand Up @@ -235,16 +236,14 @@ public long getSubscriberCount() throws ParsingException {
return UNKNOWN_SUBSCRIBER_COUNT;
}

if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();

if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
if (channelHeader != null) {
if (channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
// No subscriber count is available on interactiveTabbedHeaderRenderer header
return UNKNOWN_SUBSCRIBER_COUNT;
}

final JsonObject headerJson = header.json;
if (header.headerType == HeaderType.PAGE) {
final JsonObject headerJson = channelHeader.json;
if (channelHeader.headerType == HeaderType.PAGE) {
return getSubscriberCountFromPageChannelHeader(headerJson);
}

Expand Down Expand Up @@ -321,19 +320,17 @@ public String getDescription() throws ParsingException {
}

try {
if (channelHeader.isPresent()) {
final ChannelHeader header = channelHeader.get();
if (header.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(header.json.getObject("description"));
}
if (channelHeader != null
&& channelHeader.headerType == HeaderType.INTERACTIVE_TABBED) {
/*
In an interactiveTabbedHeaderRenderer, the real description, is only available
in its header
The other one returned in non-About tabs accessible in the
microformatDataRenderer object of the response may be completely different
The description extracted is incomplete and the original one can be only
accessed from the About tab
*/
return getTextFromObject(channelHeader.json.getObject("description"));
}

return jsonResponse.getObject(METADATA)
Expand Down Expand Up @@ -368,8 +365,11 @@ public boolean isVerified() throws ParsingException {
return false;
}

return YoutubeChannelHelper.isChannelVerified(channelHeader.orElseThrow(() ->
new ParsingException("Could not get verified status")));
if (channelHeader == null) {
throw new ParsingException("Could not get channel verified status, no channel header has been extracted");
}

return YoutubeChannelHelper.isChannelVerified(channelHeader);
}

@Nonnull
Expand Down Expand Up @@ -421,6 +421,19 @@ private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws Parsin

final String urlSuffix = urlParts[urlParts.length - 1];

/*
Make a copy of the channelHeader member to avoid keeping a reference to
this YoutubeChannelExtractor instance which would prevent serialization of
the ReadyChannelTabListLinkHandler instance created above
*/
final ChannelHeader channelHeaderCopy;
if (channelHeader == null) {
channelHeaderCopy = null;
} else {
channelHeaderCopy = new ChannelHeader(channelHeader.json,
channelHeader.headerType);
}

switch (urlSuffix) {
case "videos":
// Since the Videos tab has already its contents fetched, make
Expand All @@ -431,9 +444,8 @@ private List<ListLinkHandler> getTabsForNonAgeRestrictedChannels() throws Parsin
channelId,
ChannelTabs.VIDEOS,
(service, linkHandler) -> new VideosTabExtractor(
service, linkHandler, tabRenderer, channelHeader,
name, id, url)));

service, linkHandler, tabRenderer,
channelHeaderCopy, name, id, url)));
break;
case "shorts":
addNonVideosTab.accept(ChannelTabs.SHORTS);
Expand Down
27 changes: 15 additions & 12 deletions .../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@
*/
public class YoutubeChannelTabExtractor extends ChannelTabExtractor {

@Nullable
protected YoutubeChannelHelper.ChannelHeader channelHeader;

/**
* Whether the visitor data extracted from the initial channel response is required to be used
* for continuations.
Expand All @@ -59,8 +62,6 @@ public class YoutubeChannelTabExtractor extends ChannelTabExtractor {
private String channelId;
@Nullable
private String visitorData;
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
protected Optional<YoutubeChannelHelper.ChannelHeader> channelHeader;

public YoutubeChannelTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler) {
Expand Down Expand Up @@ -123,9 +124,9 @@ public String getId() throws ParsingException {
}

protected String getChannelName() throws ParsingException {
return YoutubeChannelHelper.getChannelName(
channelHeader, jsonResponse,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse));
return YoutubeChannelHelper.getChannelName(channelHeader,
YoutubeChannelHelper.getChannelAgeGateRenderer(jsonResponse),
jsonResponse);
}

@Nonnull
Expand Down Expand Up @@ -159,11 +160,14 @@ public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionEx
}
}

final VerifiedStatus verifiedStatus = channelHeader.flatMap(header ->
YoutubeChannelHelper.isChannelVerified(header)
? Optional.of(VerifiedStatus.VERIFIED)
: Optional.of(VerifiedStatus.UNVERIFIED))
.orElse(VerifiedStatus.UNKNOWN);
final VerifiedStatus verifiedStatus;
if (channelHeader == null) {
verifiedStatus = VerifiedStatus.UNKNOWN;
} else {
verifiedStatus = YoutubeChannelHelper.isChannelVerified(channelHeader)
? VerifiedStatus.VERIFIED
: VerifiedStatus.UNVERIFIED;
}

// If a channel tab is fetched, the next page requires channel ID and name, as channel
// streams don't have their channel specified.
Expand Down Expand Up @@ -457,8 +461,7 @@ public static final class VideosTabExtractor extends YoutubeChannelTabExtractor
VideosTabExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final JsonObject tabRenderer,
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
final Optional<YoutubeChannelHelper.ChannelHeader> channelHeader,
@Nullable final YoutubeChannelHelper.ChannelHeader channelHeader,
final String channelName,
final String channelId,
final String channelUrl) {
Expand Down

0 comments on commit 422df70

Please sign in to comment.