From 2241473b6bb6dcabd56ab566c983282a3d45955d Mon Sep 17 00:00:00 2001 From: dec05eba Date: Mon, 30 Oct 2023 23:01:47 +0100 Subject: Youtube: do proper playlist pagination, show playlist videos views and uploaded date --- src/plugins/Page.cpp | 4 ++ src/plugins/Youtube.cpp | 174 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 147 insertions(+), 31 deletions(-) (limited to 'src/plugins') diff --git a/src/plugins/Page.cpp b/src/plugins/Page.cpp index 991f651..0e52ca4 100644 --- a/src/plugins/Page.cpp +++ b/src/plugins/Page.cpp @@ -19,6 +19,10 @@ namespace QuickMedia { return DownloadResult::OK; } + //FILE *f = fopen("data.json", "wb"); + //fwrite(server_response.data(), 1, server_response.size(), f); + //fclose(f); + Json::CharReaderBuilder json_builder; std::unique_ptr json_reader(json_builder.newCharReader()); std::string json_errors; diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index 2e59a0b..d4211b3 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -192,6 +192,20 @@ namespace QuickMedia { return false; } + static bool youtube_url_extract_playlist_id(const std::string &youtube_url, std::string &playlist_id) { + size_t start_index = youtube_url.find("&list="); + if(start_index == std::string::npos) + return false; + + start_index += 6; + size_t end_index = youtube_url.find("&", start_index); + if(end_index == std::string::npos) + end_index = youtube_url.size(); + + playlist_id = youtube_url.substr(start_index, end_index - start_index); + return true; + } + static std::mutex cookies_mutex; static std::string cookies_filepath; static std::string api_key; @@ -557,6 +571,7 @@ namespace QuickMedia { if(!title) title = std::move(headline); + std::optional video_info = yt_json_get_text(video_item_json, "videoInfo"); std::optional date = yt_json_get_text(video_item_json, "publishedTimeText"); std::optional view_count_text = yt_json_get_text(video_item_json, "viewCountText"); std::optional owner_text = yt_json_get_text(video_item_json, "shortBylineText"); @@ -596,33 +611,43 @@ namespace QuickMedia { auto body_item = BodyItem::create(title.value()); std::string desc; - if(view_count_text) - desc += view_count_text.value(); - if(date) { - if(!desc.empty()) - desc += " • "; - desc += date.value(); + if(video_info) { + desc += video_info.value(); + } else { + if(view_count_text) + desc += view_count_text.value(); + + if(date) { + if(!desc.empty()) + desc += " • "; + desc += date.value(); + } } + if(!scheduled_text.empty()) { if(!desc.empty()) desc += " • "; desc += scheduled_text; } + if(length) { if(!desc.empty()) desc += '\n'; desc += length.value(); } + if(video_is_live(video_item_json)) { if(!desc.empty()) desc += '\n'; desc += "Live now"; } + if(owner_text) { if(!desc.empty()) desc += '\n'; desc += owner_text.value(); } + /*if(description_snippet) { if(!desc.empty()) desc += '\n'; @@ -781,8 +806,15 @@ namespace QuickMedia { if(long_byline_text.empty()) long_byline_text = yt_json_get_text(playlist_renderer_json, "shortBylineText").value_or(""); + std::optional published_time_text = yt_json_get_text(playlist_renderer_json, "publishedTimeText"); + std::string video_id; std::string description = std::move(long_byline_text); + if(published_time_text) { + if(!description.empty()) + description += " • "; + description += published_time_text.value(); + } if(!description.empty()) description += '\n'; description += video_count_text + " video" + (strcmp(video_count_text.c_str(), "1") == 0 ? "" : "s"); @@ -842,7 +874,7 @@ namespace QuickMedia { if(body_item) result_items.push_back(std::move(body_item)); - // TODO: Mix + // TODO: youtube mix //body_item = parse_playlist_renderer(item_json["radioRenderer"]); //if(body_item) // result_items.push_back(std::move(body_item)); @@ -870,7 +902,7 @@ namespace QuickMedia { if(body_item) result_items.push_back(std::move(body_item)); - // TODO: Mix + // TODO: youtube mix //body_item = parse_playlist_renderer(content_item_json["radioRenderer"]); //if(body_item) // result_items.push_back(std::move(body_item)); @@ -912,6 +944,27 @@ namespace QuickMedia { return ""; } + static void parse_playlist_video_list(const Json::Value &playlist_video_list_json, const char *list_name, std::string &continuation_token, std::unordered_set &added_videos, BodyItems &body_items) { + if(!playlist_video_list_json.isObject()) + return; + + const Json::Value &contents_json = playlist_video_list_json[list_name]; + if(!contents_json.isArray()) + return; + + for(const Json::Value &content_json : contents_json) { + if(!content_json.isObject()) + continue; + + if(continuation_token.empty()) + continuation_token = item_section_renderer_get_continuation_token(content_json); + + auto body_item = parse_common_video_item(content_json["playlistVideoRenderer"], added_videos); + if(body_item) + body_items.push_back(std::move(body_item)); + } + } + static void parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set &added_videos, std::string &browse_id, BodyItems &body_items) { if(!json_root.isObject()) return; @@ -1014,6 +1067,8 @@ namespace QuickMedia { if(!content_json.isObject()) continue; + parse_playlist_video_list(content_json["playlistVideoListRenderer"], "contents", continuation_token, added_videos, body_items); + const Json::Value &grid_renderer_json = content_json["gridRenderer"]; if(!grid_renderer_json.isObject()) continue; @@ -1214,7 +1269,7 @@ namespace QuickMedia { // TODO: Make all pages (for all services) lazy fetch in a similar manner! YoutubeChannelPage::create_each_type(program, args.url, "", args.title, result_tabs); } else if(strstr(args.url.c_str(), "&list=")) { - result_tabs.push_back(Tab{create_body(false, true), std::make_unique(program, args.url, args.title), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); + result_tabs.push_back(Tab{create_body(false, true), std::make_unique(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)}); } else { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url), nullptr}); } @@ -1998,7 +2053,7 @@ namespace QuickMedia { return PluginResult::OK; if(strstr(args.url.c_str(), "&list=")) { - result_tabs.push_back(Tab{create_body(false, true), std::make_unique(program, args.url, args.title), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); + result_tabs.push_back(Tab{create_body(false, true), std::make_unique(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)}); } else { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url), nullptr}); } @@ -2257,20 +2312,15 @@ namespace QuickMedia { return PluginResult::OK; } - PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector &result_tabs) { - result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url, true, true), nullptr}); - return PluginResult::OK; - } - static void playlist_get_items(const Json::Value &json_root, std::unordered_set &added_videos, BodyItems &result_items) { if(!json_root.isObject()) return; - const Json::Value &response_json = json_root["response"]; - if(!response_json.isObject()) - return; + const Json::Value *response_json = &json_root["response"]; + if(!response_json->isObject()) + response_json = &json_root; - const Json::Value &contents_json = response_json["contents"]; + const Json::Value &contents_json = (*response_json)["contents"]; if(!contents_json.isObject()) return; @@ -2301,35 +2351,97 @@ namespace QuickMedia { } } - PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) { + YoutubePlaylistPage::YoutubePlaylistPage(Program *program, const std::string &url, std::string title) : + LazyFetchPage(program), title(std::move(title)) + { + if(!youtube_url_extract_playlist_id(url, playlist_id)) + fprintf(stderr, "Error: failed to extract playlist id from url: %s\n", url.c_str()); + } + + PluginResult YoutubePlaylistPage::get_page(const std::string&, int, BodyItems &result_items) { + if(reached_end) + return PluginResult::OK; + + std::vector cookies = get_cookies(); + std::string next_url = "https://www.youtube.com/youtubei/v1/browse?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false"; + + Json::Value client_json(Json::objectValue); + client_json["hl"] = "en"; + client_json["gl"] = "US"; + client_json["deviceMake"] = ""; + client_json["deviceModel"] = ""; + client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; + client_json["clientName"] = "WEB"; + client_json["clientVersion"] = "2.20210622.10.00"; + client_json["osName"] = "X11"; + client_json["osVersion"] = ""; + client_json["originalUrl"] = "https://www.youtube.com/playlist?list=" + playlist_id; + + Json::Value context_json(Json::objectValue); + context_json["client"] = std::move(client_json); + + Json::Value request_json(Json::objectValue); + request_json["context"] = std::move(context_json); + if(continuation_token.empty()) + request_json["browseId"] = "VL" + playlist_id; + else + request_json["continuation"] = continuation_token; + + Json::StreamWriterBuilder json_builder; + json_builder["commentStyle"] = "None"; + json_builder["indentation"] = ""; + std::vector additional_args = { + { "-H", "content-type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, + { "--data-raw", Json::writeString(json_builder, request_json) } }; - std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; - DownloadResult download_result = download_json(json_root, url + "&pbj=1&gl=US&hl=en", additional_args, true); - if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); + DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); + if(result != DownloadResult::OK) return download_result_to_plugin_result(result); + + if(!json_root.isObject()) + return PluginResult::ERR; std::unordered_set added_videos; - if(json_root.isObject()) { - playlist_get_items(json_root, added_videos, result_items); - return PluginResult::OK; - } + std::string browse_id; + continuation_token.clear(); - if(!json_root.isArray()) - return PluginResult::OK; + const Json::Value &on_response_received_actions_json = json_root["onResponseReceivedActions"]; + if(on_response_received_actions_json.isArray()) { + for(const Json::Value &json_item : on_response_received_actions_json) { + if(!json_item.isObject()) + continue; - for(const Json::Value &json_item : json_root) { - playlist_get_items(json_item, added_videos, result_items); + const Json::Value &append_continuation_items_action_json = json_item["appendContinuationItemsAction"]; + if(!append_continuation_items_action_json.isObject()) + continue; + + parse_playlist_video_list(append_continuation_items_action_json, "continuationItems", continuation_token, added_videos, result_items); + } + } else { + parse_channel_videos(json_root, continuation_token, added_videos, browse_id, result_items); } + if(continuation_token.empty()) + reached_end = true; + return PluginResult::OK; } + PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector &result_tabs) { + result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url, true, true), nullptr}); + return PluginResult::OK; + } + + PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) { + return get_page("", 0, result_items); + } + static std::string two_column_watch_next_results_get_comments_continuation_token(const Json::Value &tcwnr_json) { const Json::Value &results_json = tcwnr_json["results"]; if(!results_json.isObject()) -- cgit v1.2.3