#include "../../plugins/Youtube.hpp" #include "../../include/Storage.hpp" #include "../../include/NetUtils.hpp" #include "../../include/Scale.hpp" #include namespace QuickMedia { // This is a common setup of text in the youtube json static const char* yt_json_get_text(const Json::Value &json, const char *root_name) { if(!json.isObject()) return nullptr; const Json::Value &text_json = json[root_name]; if(!text_json.isObject()) return nullptr; const Json::Value &simple_text_json = text_json["simpleText"]; if(simple_text_json.isString()) { return simple_text_json.asCString(); } else { const Json::Value &runs_json = text_json["runs"]; if(runs_json.isArray() && !runs_json.empty()) { const Json::Value &first_runs_json = runs_json[0]; if(first_runs_json.isObject()) { const Json::Value &text_json = first_runs_json["text"]; if(text_json.isString()) return text_json.asCString(); } } } return nullptr; } static std::shared_ptr parse_common_video_item(const Json::Value &video_item_json, std::unordered_set &added_videos) { const Json::Value &video_id_json = video_item_json["videoId"]; if(!video_id_json.isString()) return nullptr; std::string video_id_str = video_id_json.asString(); if(added_videos.find(video_id_str) != added_videos.end()) return nullptr; const char *title = yt_json_get_text(video_item_json, "title"); if(!title) return nullptr; const char *date = yt_json_get_text(video_item_json, "publishedTimeText"); const char *view_count_text = yt_json_get_text(video_item_json, "viewCountText"); const char *owner_text = yt_json_get_text(video_item_json, "shortBylineText"); const char *length = yt_json_get_text(video_item_json, "lengthText"); if(!length) { const Json::Value &thumbnail_overlays_json = video_item_json["thumbnailOverlays"]; if(thumbnail_overlays_json.isArray() && !thumbnail_overlays_json.empty()) { const Json::Value &thumbnail_overlay_json = thumbnail_overlays_json[0]; if(thumbnail_overlay_json.isObject()) length = yt_json_get_text(thumbnail_overlay_json["thumbnailOverlayTimeStatusRenderer"], "text"); } } auto body_item = BodyItem::create(title); std::string desc; if(view_count_text) desc += view_count_text; if(date) { if(!desc.empty()) desc += " • "; desc += date; } if(length) { if(!desc.empty()) desc += '\n'; desc += length; } if(owner_text) { if(!desc.empty()) desc += '\n'; desc += owner_text; } body_item->set_description(std::move(desc)); body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; body_item->thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg"; body_item->thumbnail_size = sf::Vector2i(175, 131); added_videos.insert(video_id_str); return body_item; } static std::shared_ptr parse_content_video_renderer(const Json::Value &content_item_json, std::unordered_set &added_videos) { if(!content_item_json.isObject()) return nullptr; const Json::Value &video_renderer_json = content_item_json["videoRenderer"]; if(!video_renderer_json.isObject()) return nullptr; return parse_common_video_item(video_renderer_json, added_videos); } struct Thumbnail { const char *url; int width; int height; }; static std::optional yt_json_get_largest_thumbnail(const Json::Value &thumbnail_json) { if(!thumbnail_json.isObject()) return std::nullopt; const Json::Value &thumbnails_json = thumbnail_json["thumbnails"]; if(!thumbnails_json.isArray()) return std::nullopt; std::vector thumbnails; for(const Json::Value &thumbnail_data_json : thumbnails_json) { if(!thumbnail_data_json.isObject()) continue; const Json::Value &url_json = thumbnail_data_json["url"]; if(!url_json.isString()) continue; const Json::Value &width_json = thumbnail_data_json["width"]; if(!width_json.isInt()) continue; const Json::Value &height_json = thumbnail_data_json["height"]; if(!height_json.isInt()) continue; thumbnails.push_back({ url_json.asCString(), width_json.asInt(), height_json.asInt() }); } return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { int size1 = thumbnail1.width * thumbnail1.height; int size2 = thumbnail2.width * thumbnail2.height; return size1 < size2; }); } static std::shared_ptr parse_channel_renderer(const Json::Value &channel_renderer_json) { if(!channel_renderer_json.isObject()) return nullptr; const Json::Value &channel_id_json = channel_renderer_json["channelId"]; if(!channel_id_json.isString()) return nullptr; const char *title = yt_json_get_text(channel_renderer_json, "title"); if(!title) return nullptr; const char *description = yt_json_get_text(channel_renderer_json, "descriptionSnippet"); const char *video_count = yt_json_get_text(channel_renderer_json, "videoCountText"); const char *subscribers = yt_json_get_text(channel_renderer_json, "subscriberCountText"); const Json::Value &thumbnail_json = channel_renderer_json["thumbnail"]; std::optional thumbnail = yt_json_get_largest_thumbnail(thumbnail_json); auto body_item = BodyItem::create(title); std::string desc; if(subscribers) desc += subscribers; if(video_count) { if(!desc.empty()) desc += " • "; desc += video_count; if(strcmp(video_count, "1") == 0) desc += " video"; else desc += " videos"; } if(description) { if(!desc.empty()) desc += '\n'; desc += description; } body_item->set_description(std::move(desc)); body_item->url = "https://www.youtube.com/channel/" + channel_id_json.asString(); if(thumbnail) { body_item->thumbnail_url = std::string("https:") + thumbnail->url; body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE; body_item->thumbnail_size.x = thumbnail->width; body_item->thumbnail_size.y = thumbnail->height; body_item->thumbnail_size = clamp_to_size(body_item->thumbnail_size, sf::Vector2i(136, 136)); } return body_item; } // Returns empty string if continuation token can't be found static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) { const Json::Value &continuation_item_renderer_json = item_section_renderer_json["continuationItemRenderer"]; if(!continuation_item_renderer_json.isObject()) return ""; const Json::Value &continuation_endpoint_json = continuation_item_renderer_json["continuationEndpoint"]; if(!continuation_endpoint_json.isObject()) return ""; const Json::Value &continuation_command_json = continuation_endpoint_json["continuationCommand"]; if(!continuation_command_json.isObject()) return ""; const Json::Value &token_json = continuation_command_json["token"]; if(!token_json.isString()) return ""; return token_json.asString(); } static std::string grid_renderer_get_continuation_token(const Json::Value &grid_renderer_json) { const Json::Value &continuations_json = grid_renderer_json["continuations"]; if(!continuations_json.isArray()) return ""; for(const Json::Value &continuation_json : continuations_json) { if(!continuation_json.isObject()) continue; const Json::Value &next_continuation_data_json = continuation_json["nextContinuationData"]; if(!next_continuation_data_json.isObject()) continue; const Json::Value &continuation_item_json = next_continuation_data_json["continuation"]; if(continuation_item_json.isString()) return continuation_item_json.asString(); } return ""; } static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::unordered_set &added_videos, BodyItems &result_items) { const Json::Value &item_contents_json = item_section_renderer_json["contents"]; if(!item_contents_json.isArray()) return; for(const Json::Value &content_item_json : item_contents_json) { if(!content_item_json.isObject()) continue; for(Json::Value::const_iterator it = content_item_json.begin(); it != content_item_json.end(); ++it) { Json::Value key = it.key(); if(key.isString() && strcmp(key.asCString(), "shelfRenderer") == 0) { const Json::Value &shelf_renderer_json = *it; if(!shelf_renderer_json.isObject()) continue; const Json::Value &item_content_json = shelf_renderer_json["content"]; if(!item_content_json.isObject()) continue; const Json::Value &vertical_list_renderer_json = item_content_json["verticalListRenderer"]; if(!vertical_list_renderer_json.isObject()) continue; const Json::Value &items_json = vertical_list_renderer_json["items"]; if(!items_json.isArray()) continue; for(const Json::Value &item_json : items_json) { std::shared_ptr body_item = parse_content_video_renderer(item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } else if(key.isString() && strcmp(key.asCString(), "channelRenderer") == 0) { std::shared_ptr body_item = parse_channel_renderer(*it); if(body_item) result_items.push_back(std::move(body_item)); } else { std::shared_ptr body_item = parse_content_video_renderer(content_item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } } } static std::string remove_index_from_playlist_url(const std::string &url) { std::string result = url; size_t index = result.rfind("&index="); if(index == std::string::npos) return result; return result.substr(0, index); } static std::shared_ptr parse_compact_video_renderer_json(const Json::Value &item_json, std::unordered_set &added_videos) { const Json::Value &compact_video_renderer_json = item_json["compactVideoRenderer"]; if(!compact_video_renderer_json.isObject()) return nullptr; return parse_common_video_item(compact_video_renderer_json, added_videos); } static BodyItems parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set &added_videos) { BodyItems body_items; if(!json_root.isArray()) return body_items; std::string new_continuation_token; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) continue; const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"]; if(!tcbrr_json.isObject()) continue; const Json::Value &tabs_json = tcbrr_json["tabs"]; if(!tabs_json.isArray()) continue; for(const Json::Value &tab_json : tabs_json) { if(!tab_json.isObject()) continue; const Json::Value &tab_renderer_json = tab_json["tabRenderer"]; if(!tab_renderer_json.isObject()) continue; const Json::Value &content_json = tab_renderer_json["content"]; if(!content_json.isObject()) continue; const Json::Value §ion_list_renderer = content_json["sectionListRenderer"]; if(!section_list_renderer.isObject()) continue; const Json::Value &contents2_json = section_list_renderer["contents"]; if(!contents2_json.isArray()) continue; for(const Json::Value &content_item_json : contents2_json) { if(!content_item_json.isObject()) continue; const Json::Value &item_section_renderer_json = content_item_json["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; const Json::Value &item_contents_json = item_section_renderer_json["contents"]; if(!item_contents_json.isArray()) continue; for(const Json::Value &content_json : item_contents_json) { if(!content_json.isObject()) continue; const Json::Value &grid_renderer_json = content_json["gridRenderer"]; if(!grid_renderer_json.isObject()) continue; if(new_continuation_token.empty()) new_continuation_token = grid_renderer_get_continuation_token(grid_renderer_json); const Json::Value &items_json = grid_renderer_json["items"]; if(!items_json.isArray()) continue; for(const Json::Value &item_json : items_json) { if(!item_json.isObject()) continue; const Json::Value &grid_video_renderer = item_json["gridVideoRenderer"]; if(!grid_video_renderer.isObject()) continue; auto body_item = parse_common_video_item(grid_video_renderer, added_videos); if(body_item) body_items.push_back(std::move(body_item)); } } } } } if(!new_continuation_token.empty()) continuation_token = std::move(new_continuation_token); return body_items; } SearchResult YoutubeSearchPage::search(const std::string &str, BodyItems &result_items) { continuation_token.clear(); current_page = 0; added_videos.clear(); search_url = "https://youtube.com/results?search_query="; search_url += url_param_encode(str); std::vector additional_args = { { "-H", "x-spf-referer: " + search_url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + search_url } }; //std::vector cookies = get_cookies(); //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, search_url + "&pbj=1", std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_search_result(result); if(!json_root.isArray()) return SearchResult::ERR; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) continue; const Json::Value &tcsrr_json = contents_json["twoColumnSearchResultsRenderer"]; if(!tcsrr_json.isObject()) continue; const Json::Value &primary_contents_json = tcsrr_json["primaryContents"]; if(!primary_contents_json.isObject()) continue; const Json::Value §ion_list_renderer_json = primary_contents_json["sectionListRenderer"]; if(!section_list_renderer_json.isObject()) continue; const Json::Value &contents2_json = section_list_renderer_json["contents"]; if(!contents2_json.isArray()) continue; for(const Json::Value &item_json : contents2_json) { if(!item_json.isObject()) continue; if(continuation_token.empty()) continuation_token = item_section_renderer_get_continuation_token(item_json); const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); } } return SearchResult::OK; } PluginResult YoutubeSearchPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = search_get_continuation(search_url, continuation_token, result_items); if(plugin_result != PluginResult::OK) return plugin_result; ++current_page; } return PluginResult::OK; } PluginResult YoutubeSearchPage::submit(const std::string &title, const std::string &url, std::vector &result_tabs) { if(strncmp(url.c_str(), "https://www.youtube.com/channel/", 32) == 0) { // TODO: Make all pages (for all services) lazy fetch in a similar manner! result_tabs.push_back(Tab{create_body(), std::make_unique(program, url, "", title), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); } else { result_tabs.push_back(Tab{nullptr, std::make_unique(program), nullptr}); } return PluginResult::OK; } PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { std::string next_url = url + "&pbj=1&ctoken=" + current_continuation_token; std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + url } }; //std::vector cookies = get_cookies(); //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(!json_root.isArray()) return PluginResult::ERR; std::string new_continuation_token; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &on_response_received_commands_json = response_json["onResponseReceivedCommands"]; if(!on_response_received_commands_json.isArray()) continue; for(const Json::Value &response_received_command : on_response_received_commands_json) { if(!response_received_command.isObject()) continue; const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"]; if(!append_continuation_items_action_json.isObject()) continue; const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"]; if(!continuation_items_json.isArray()) continue; for(const Json::Value &continuation_item : continuation_items_json) { if(!continuation_item.isObject()) continue; if(new_continuation_token.empty()) { // Note: item_section_renderer is compatible with continuation_item new_continuation_token = item_section_renderer_get_continuation_token(continuation_item); } const Json::Value &item_section_renderer_json = continuation_item["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); } } } if(!new_continuation_token.empty()) continuation_token = std::move(new_continuation_token); return PluginResult::OK; } PluginResult YoutubeChannelPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = search_get_continuation(url, continuation_token, result_items); if(plugin_result != PluginResult::OK) return plugin_result; ++current_page; } return PluginResult::OK; } PluginResult YoutubeChannelPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { std::string next_url = "https://www.youtube.com/browse_ajax?ctoken=" + current_continuation_token + "&continuation=" + current_continuation_token; std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + url } }; //std::vector cookies = get_cookies(); //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(!json_root.isArray()) return PluginResult::ERR; std::string new_continuation_token; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &continuation_contents_json = response_json["continuationContents"]; if(!continuation_contents_json.isObject()) continue; const Json::Value &grid_continuation_json = continuation_contents_json["gridContinuation"]; if(!grid_continuation_json.isObject()) continue; if(new_continuation_token.empty()) { // grid_continuation_json is compatible with grid_renderer new_continuation_token = grid_renderer_get_continuation_token(grid_continuation_json); } const Json::Value &items_json = grid_continuation_json["items"]; if(!items_json.isArray()) continue; for(const Json::Value &item_json : items_json) { if(!item_json.isObject()) continue; const Json::Value &grid_video_renderer = item_json["gridVideoRenderer"]; if(!grid_video_renderer.isObject()) continue; auto body_item = parse_common_video_item(grid_video_renderer, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } if(!new_continuation_token.empty()) continuation_token = std::move(new_continuation_token); return PluginResult::OK; } PluginResult YoutubeChannelPage::submit(const std::string&, const std::string&, std::vector &result_tabs) { result_tabs.push_back(Tab{nullptr, std::make_unique(program), nullptr}); return PluginResult::OK; } PluginResult YoutubeChannelPage::lazy_fetch(BodyItems &result_items) { std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + url } }; //std::vector cookies = get_cookies(); //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, url + "/videos?pbj=1", std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); result_items = parse_channel_videos(json_root, continuation_token, added_videos); return PluginResult::OK; } PluginResult YoutubeRelatedVideosPage::submit(const std::string&, const std::string&, std::vector &result_tabs) { result_tabs.push_back(Tab{nullptr, std::make_unique(program), nullptr}); return PluginResult::OK; } // TODO: Make this faster by using string search instead of parsing html. // TODO: If the result is a play BodyItems YoutubeVideoPage::get_related_media(const std::string &url, std::string &channel_url) { BodyItems result_items; std::string modified_url = remove_index_from_playlist_url(url); std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + url } }; //std::vector cookies = get_cookies(); //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, modified_url + "&pbj=1", std::move(additional_args), true); if(result != DownloadResult::OK) return result_items; if(!json_root.isArray()) return result_items; std::unordered_set added_videos; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; if(channel_url.empty()) { const Json::Value &player_response_json = json_item["playerResponse"]; if(player_response_json.isObject()) { const Json::Value &video_details_json = player_response_json["videoDetails"]; if(video_details_json.isObject()) { const Json::Value &channel_id_json = video_details_json["channelId"]; if(channel_id_json.isString()) channel_url = "https://www.youtube.com/channel/" + channel_id_json.asString(); } } } const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) return result_items; const Json::Value &tcwnr_json = contents_json["twoColumnWatchNextResults"]; if(!tcwnr_json.isObject()) return result_items; const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"]; if(!secondary_results_json.isObject()) return result_items; const Json::Value &secondary_results2_json = secondary_results_json["secondaryResults"]; if(!secondary_results2_json.isObject()) return result_items; const Json::Value &results_json = secondary_results2_json["results"]; if(!results_json.isArray()) return result_items; for(const Json::Value &item_json : results_json) { if(!item_json.isObject()) continue; auto body_item = parse_compact_video_renderer_json(item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); const Json::Value &compact_autoplay_renderer_json = item_json["compactAutoplayRenderer"]; if(!compact_autoplay_renderer_json.isObject()) continue; const Json::Value &item_contents_json = compact_autoplay_renderer_json["contents"]; if(!item_contents_json.isArray()) continue; for(const Json::Value &content_item_json : item_contents_json) { if(!content_item_json.isObject()) continue; auto body_item = parse_compact_video_renderer_json(content_item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } } return result_items; } std::unique_ptr YoutubeVideoPage::create_search_page(Program *program, int &search_delay) { search_delay = 350; return std::make_unique(program); } std::unique_ptr YoutubeVideoPage::create_related_videos_page(Program *program, const std::string&, const std::string&) { return std::make_unique(program); } std::unique_ptr YoutubeVideoPage::create_channels_page(Program *program, const std::string &channel_url) { return std::make_unique(program, channel_url, "", "Channel videos"); } }