From 5d29f22093fd602bc4d8863208e7812c0746e62e Mon Sep 17 00:00:00 2001 From: dec05eba Date: Wed, 10 Mar 2021 23:25:09 +0100 Subject: Youtube: add youtube comments to ctrl+r --- src/plugins/Youtube.cpp | 412 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 389 insertions(+), 23 deletions(-) (limited to 'src/plugins/Youtube.cpp') diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index b3c9a60..ced6c45 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -5,6 +5,7 @@ #include "../../include/Scale.hpp" #include #include +#include namespace QuickMedia { // This is a common setup of text in the youtube json @@ -18,7 +19,7 @@ namespace QuickMedia { const Json::Value &simple_text_json = text_json["simpleText"]; if(simple_text_json.isString()) { - return simple_text_json.asCString(); + return simple_text_json.asString(); } else { const Json::Value &runs_json = text_json["runs"]; if(!runs_json.isArray() || runs_json.empty()) @@ -46,8 +47,13 @@ namespace QuickMedia { int height; }; + enum class ThumbnailSize { + SMALLEST, + LARGEST + }; + // TODO: Use this in |parse_common_video_item| when QuickMedia supports webp - static std::optional yt_json_get_largest_thumbnail(const Json::Value &thumbnail_json) { + static std::optional yt_json_get_thumbnail(const Json::Value &thumbnail_json, ThumbnailSize thumbnail_size) { if(!thumbnail_json.isObject()) return std::nullopt; @@ -75,11 +81,22 @@ namespace QuickMedia { thumbnails.push_back({ url_json.asCString(), width_json.asInt(), height_json.asInt() }); } - return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { - int size1 = thumbnail1.width * thumbnail1.height; - int size2 = thumbnail2.width * thumbnail2.height; - return size1 < size2; - }); + switch(thumbnail_size) { + case ThumbnailSize::SMALLEST: + return *std::min_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { + int size1 = thumbnail1.width * thumbnail1.height; + int size2 = thumbnail2.width * thumbnail2.height; + return size1 < size2; + }); + case ThumbnailSize::LARGEST: + return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { + int size1 = thumbnail1.width * thumbnail1.height; + int size2 = thumbnail2.width * thumbnail2.height; + return size1 < size2; + }); + } + + return std::nullopt; } static std::shared_ptr parse_common_video_item(const Json::Value &video_item_json, std::unordered_set &added_videos) { @@ -196,7 +213,7 @@ namespace QuickMedia { std::optional subscribers = yt_json_get_text(channel_renderer_json, "subscriberCountText"); const Json::Value &thumbnail_json = channel_renderer_json["thumbnail"]; - std::optional thumbnail = yt_json_get_largest_thumbnail(thumbnail_json); + std::optional thumbnail = yt_json_get_thumbnail(thumbnail_json, ThumbnailSize::LARGEST); auto body_item = BodyItem::create(title.value()); std::string desc; @@ -300,6 +317,37 @@ namespace QuickMedia { } } + static std::mutex cookies_mutex; + static std::string cookies_filepath; + + static void remove_cookies_file_at_exit() { + std::lock_guard lock(cookies_mutex); + if(!cookies_filepath.empty()) + remove(cookies_filepath.c_str()); + } + + static std::vector get_cookies() { + std::lock_guard lock(cookies_mutex); + if(cookies_filepath.empty()) { + char filename[] = "quickmedia.youtube.cookie.XXXXXX"; + int fd = mkstemp(filename); + if(fd == -1) + return {}; + close(fd); + + cookies_filepath = "/tmp/"; + cookies_filepath += filename; + cookies_filepath += ".txt"; + + atexit(remove_cookies_file_at_exit); + } + + return { + CommandArg{ "-b", cookies_filepath }, + CommandArg{ "-c", cookies_filepath } + }; + } + static std::string remove_index_from_playlist_url(const std::string &url) { std::string result = url; size_t index = result.rfind("&index="); @@ -316,6 +364,30 @@ namespace QuickMedia { return parse_common_video_item(compact_video_renderer_json, added_videos); } + static std::string item_section_renderer_get_continuation(const Json::Value &item_section_renderer) { + if(!item_section_renderer.isObject()) + return ""; + + const Json::Value &continuations_json = item_section_renderer["continuations"]; + if(!continuations_json.isArray()) + return ""; + + for(const Json::Value &json_item : continuations_json) { + if(!json_item.isObject()) + continue; + + const Json::Value &next_continuation_data_json = json_item["nextContinuationData"]; + if(!next_continuation_data_json.isObject()) + continue; + + const Json::Value &continuation_json = next_continuation_data_json["continuation"]; + if(continuation_json.isString()) + return continuation_json.asString(); + } + + return ""; + } + static BodyItems parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set &added_videos) { BodyItems body_items; if(!json_root.isArray()) @@ -450,8 +522,8 @@ namespace QuickMedia { { "-H", "referer: " + search_url } }; - //std::vector cookies = get_cookies(); - //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, search_url + "&pbj=1", std::move(additional_args), true); @@ -519,8 +591,8 @@ namespace QuickMedia { { "-H", "referer: " + url } }; - //std::vector cookies = get_cookies(); - //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); @@ -578,6 +650,263 @@ namespace QuickMedia { return PluginResult::OK; } + PluginResult YoutubeCommentsPage::get_page(const std::string&, int page, BodyItems &result_items) { + while(current_page < page) { + PluginResult plugin_result = lazy_fetch(result_items); + if(plugin_result != PluginResult::OK) return plugin_result; + ++current_page; + } + return PluginResult::OK; + } + + PluginResult YoutubeCommentsPage::submit(const std::string&, const std::string &url, std::vector &result_tabs) { + if(url.empty()) + return PluginResult::OK; + result_tabs.push_back(Tab{create_body(), std::make_unique(program, xsrf_token, url), nullptr}); + return PluginResult::OK; + } + + static std::string comment_thread_renderer_get_replies_continuation(const Json::Value &comment_thread_renderer_json) { + if(!comment_thread_renderer_json.isObject()) + return ""; + + const Json::Value &replies_json = comment_thread_renderer_json["replies"]; + if(!replies_json.isObject()) + return ""; + + const Json::Value &comment_replies_renderer = replies_json["commentRepliesRenderer"]; + if(!comment_replies_renderer.isObject()) + return ""; + + // item_section_renderer_get_continuation is compatible with commentRepliesRenderer + return item_section_renderer_get_continuation(comment_replies_renderer); + } + + // Returns empty string if comment is not hearted + static std::string comment_renderer_get_hearted_tooltip(const Json::Value &comment_renderer_json) { + const Json::Value &action_buttons_json = comment_renderer_json["actionButtons"]; + if(!action_buttons_json.isObject()) + return ""; + + const Json::Value &comment_action_buttons_renderer_json = action_buttons_json["commentActionButtonsRenderer"]; + if(!comment_action_buttons_renderer_json.isObject()) + return ""; + + const Json::Value &creator_heart_json = comment_action_buttons_renderer_json["creatorHeart"]; + if(!creator_heart_json.isObject()) + return ""; + + const Json::Value &creator_heart_renderer_json = creator_heart_json["creatorHeartRenderer"]; + if(!creator_heart_renderer_json.isObject()) + return ""; + + const Json::Value &hearted_tooltip_json = creator_heart_renderer_json["heartedTooltip"]; + if(!hearted_tooltip_json.isString()) + return ""; + + return hearted_tooltip_json.asString(); + } + + static std::shared_ptr comment_renderer_to_body_item(const Json::Value &comment_renderer_json) { + if(!comment_renderer_json.isObject()) + return nullptr; + + std::optional author_text = yt_json_get_text(comment_renderer_json, "authorText"); + if(!author_text) + return nullptr; + + std::string title = author_text.value(); + std::optional published_time_text = yt_json_get_text(comment_renderer_json, "publishedTimeText"); + if(published_time_text) + title += " - " + published_time_text.value(); + + auto body_item = BodyItem::create(std::move(title)); + std::string description; + + const Json::Value &author_is_channel_owner_json = comment_renderer_json["authorIsChannelOwner"]; + if(author_is_channel_owner_json.isBool() && author_is_channel_owner_json.asBool()) + body_item->set_title_color(sf::Color(150, 255, 150)); + + std::optional comment = yt_json_get_text(comment_renderer_json, "contentText"); + if(comment) + description = comment.value(); + + std::optional thumbnail = yt_json_get_thumbnail(comment_renderer_json["authorThumbnail"], ThumbnailSize::SMALLEST); + if(thumbnail) { + body_item->thumbnail_url = thumbnail->url; + body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE; + body_item->thumbnail_size.x = thumbnail->width; + body_item->thumbnail_size.y = thumbnail->height; + body_item->thumbnail_size = body_item->thumbnail_size; + } + + const Json::Value &like_count_json = comment_renderer_json["likeCount"]; + if(like_count_json.isInt64()) { + if(!description.empty()) + description += '\n'; + description += "👍 " + std::to_string(like_count_json.asInt64()); + } + + const Json::Value &reply_count_json = comment_renderer_json["replyCount"]; + if(reply_count_json.isInt64()) { + if(!description.empty()) + description += '\n'; + description += std::to_string(reply_count_json.asInt64()) + " replies"; + } + + std::string hearted_tooltip = comment_renderer_get_hearted_tooltip(comment_renderer_json); + if(!hearted_tooltip.empty()) { + if(!description.empty()) + description += ' '; + description += std::move(hearted_tooltip); + } + + body_item->set_description(std::move(description)); + return body_item; + } + + PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) { + std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comments=1&pbj=1&ctoken="; + next_url += url_param_encode(continuation_token); + //next_url += "&continuation="; + //next_url += url_param_encode(comments_continuation_token); + next_url += "&type=next"; + + std::vector additional_args = { + { "-H", "x-youtube-client-name: 1" }, + { "-H", "x-youtube-client-version: 2.20210308.08.00" }, + { "-F", "session_token=" + xsrf_token } + }; + + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + + Json::Value json_root; + DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); + if(result != DownloadResult::OK) return download_result_to_plugin_result(result); + + if(!json_root.isObject()) + return PluginResult::ERR; + + const Json::Value &xsrf_token_json = json_root["xsrf_token"]; + if(xsrf_token_json.isString()) + xsrf_token = xsrf_token_json.asString(); + + const Json::Value &response_json = json_root["response"]; + if(!response_json.isObject()) + return PluginResult::ERR; + + const Json::Value &continuation_contents_json = response_json["continuationContents"]; + if(!continuation_contents_json.isObject()) + return PluginResult::ERR; + + const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"]; + if(!item_section_continuation_json.isObject()) + return PluginResult::ERR; + + const Json::Value &contents_json = item_section_continuation_json["contents"]; + if(contents_json.isArray()) { + for(const Json::Value &json_item : contents_json) { + if(!json_item.isObject()) + continue; + + const Json::Value &comment_thread_renderer = json_item["commentThreadRenderer"]; + if(!comment_thread_renderer.isObject()) + continue; + + const Json::Value &comment_json = comment_thread_renderer["comment"]; + if(!comment_json.isObject()) + continue; + + auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]); + if(body_item) { + body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer); + result_items.push_back(std::move(body_item)); + } + } + } + + continuation_token = item_section_renderer_get_continuation(item_section_continuation_json); + + return PluginResult::OK; + } + + PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) { + while(current_page < page) { + PluginResult plugin_result = lazy_fetch(result_items); + if(plugin_result != PluginResult::OK) return plugin_result; + ++current_page; + } + return PluginResult::OK; + } + + PluginResult YoutubeCommentRepliesPage::submit(const std::string&, const std::string&, std::vector&) { + return PluginResult::OK; + } + + PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) { + std::string next_url = "https://www.youtube.com/comment_service_ajax?action_get_comment_replies=1&pbj=1&ctoken="; + next_url += url_param_encode(continuation_token); + //next_url += "&continuation="; + //next_url += url_param_encode(comments_continuation_token); + next_url += "&type=next"; + + std::vector additional_args = { + { "-H", "x-youtube-client-name: 1" }, + { "-H", "x-youtube-client-version: 2.20210308.08.00" }, + { "-F", "session_token=" + xsrf_token } + }; + + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + + Json::Value json_root; + DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); + if(result != DownloadResult::OK) return download_result_to_plugin_result(result); + + if(!json_root.isArray()) + return PluginResult::ERR; + + for(const Json::Value &json_item : json_root) { + if(!json_item.isObject()) + continue; + + const Json::Value &xsrf_token_json = json_item["xsrf_token"]; + if(xsrf_token_json.isString()) + xsrf_token = xsrf_token_json.asString(); + + const Json::Value &response_json = json_item["response"]; + if(!response_json.isObject()) + continue; + + const Json::Value &continuation_contents_json = response_json["continuationContents"]; + if(!continuation_contents_json.isObject()) + continue; + + const Json::Value &comment_replies_continuation_json = continuation_contents_json["commentRepliesContinuation"]; + if(!comment_replies_continuation_json.isObject()) + continue; + + const Json::Value &contents_json = comment_replies_continuation_json["contents"]; + if(contents_json.isArray()) { + for(const Json::Value &content_item_json : contents_json) { + if(!content_item_json.isObject()) + continue; + + auto body_item = comment_renderer_to_body_item(content_item_json["commentRenderer"]); + if(body_item) + result_items.push_back(std::move(body_item)); + } + } + + // item_section_renderer_get_continuation is compatible with commentRepliesContinuation + continuation_token = item_section_renderer_get_continuation(comment_replies_continuation_json); + return PluginResult::OK; + } + + return PluginResult::ERR; + } + static std::string channel_url_extract_id(const std::string &channel_url) { size_t index = channel_url.find("channel/"); if(index == std::string::npos) @@ -634,8 +963,8 @@ namespace QuickMedia { { "--data-raw", Json::writeString(json_builder, request_json) } }; - //std::vector cookies = get_cookies(); - //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); @@ -717,8 +1046,8 @@ namespace QuickMedia { { "--data-raw", Json::writeString(json_builder, request_json) } }; - //std::vector cookies = get_cookies(); - //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); @@ -768,7 +1097,7 @@ namespace QuickMedia { return PluginResult::OK; } - PluginResult YoutubeChannelPage::submit(const std::string&, const std::string &url, std::vector &result_tabs) { + PluginResult YoutubeChannelPage::submit(const std::string&, const std::string&, std::vector &result_tabs) { if(url.empty()) return PluginResult::OK; result_tabs.push_back(Tab{nullptr, std::make_unique(program), nullptr}); @@ -785,8 +1114,8 @@ namespace QuickMedia { { "-H", "referer: " + url } }; - //std::vector cookies = get_cookies(); - //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, url + "/videos?pbj=1", std::move(additional_args), true); @@ -800,8 +1129,32 @@ namespace QuickMedia { return PluginResult::OK; } - // TODO: Make this faster by using string search instead of parsing html. - // TODO: If the result is a play + static std::string two_column_watch_next_results_get_comments_continuation_token(const Json::Value &tcwnr_json) { + const Json::Value &results_json = tcwnr_json["results"]; + if(!results_json.isObject()) + return ""; + + const Json::Value &results2_json = results_json["results"]; + if(!results2_json.isObject()) + return ""; + + const Json::Value &contents_json = results2_json["contents"]; + if(!contents_json.isArray()) + return ""; + + std::string comments_continuation_token; + for(const Json::Value &content_item_json : contents_json) { + if(!content_item_json.isObject()) + continue; + + comments_continuation_token = item_section_renderer_get_continuation(content_item_json["itemSectionRenderer"]); + if(!comments_continuation_token.empty()) + return comments_continuation_token; + } + + return ""; + } + BodyItems YoutubeVideoPage::get_related_media(const std::string &url, std::string &channel_url) { BodyItems result_items; @@ -814,8 +1167,8 @@ namespace QuickMedia { { "-H", "referer: " + url } }; - //std::vector cookies = get_cookies(); - //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, modified_url + "&pbj=1", std::move(additional_args), true); @@ -842,6 +1195,12 @@ namespace QuickMedia { } } + if(xsrf_token.empty()) { + const Json::Value &xsrf_token_json = json_item["xsrf_token"]; + if(xsrf_token_json.isString()) + xsrf_token = xsrf_token_json.asString(); + } + const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; @@ -854,6 +1213,9 @@ namespace QuickMedia { if(!tcwnr_json.isObject()) return result_items; + if(comments_continuation_token.empty()) + comments_continuation_token = two_column_watch_next_results_get_comments_continuation_token(tcwnr_json); + const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"]; if(!secondary_results_json.isObject()) return result_items; @@ -901,6 +1263,10 @@ namespace QuickMedia { return std::make_unique(program); } + std::unique_ptr YoutubeVideoPage::create_comments_page(Program *program) { + return std::make_unique(program, xsrf_token, comments_continuation_token); + } + std::unique_ptr YoutubeVideoPage::create_related_videos_page(Program *program, const std::string&, const std::string&) { return std::make_unique(program); } -- cgit v1.2.3