#include "../../plugins/Youtube.hpp" #include #include namespace QuickMedia { static size_t find_end_of_json(const std::string &website_data, size_t data_start) { int brace_count = 0; char string_char = '\0'; bool escape = false; for(size_t i = data_start; i < website_data.size(); ++i) { char c = website_data[i]; if(c == '{' && string_char == '\0') { escape = false; ++brace_count; } else if(c == '}' && string_char == '\0') { escape = false; --brace_count; if(brace_count == 0) { return i + 1; } } else if(c == '"' || c == '\'') { if(string_char == '\0') { string_char = c; } else if(c == string_char && !escape) { string_char = '\0'; } escape = false; } else if(c == '\\' && string_char) { escape = !escape; } else if(string_char) { escape = false; } } return std::string::npos; } // Returns empty string if continuation token can't be found static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) { const Json::Value &continuations_json = item_section_renderer_json["continuations"]; if(!continuations_json.isArray() || continuations_json.empty()) return ""; const Json::Value &first_continuation_json = continuations_json[0]; if(!first_continuation_json.isObject()) return ""; const Json::Value &next_continuation_data_json = first_continuation_json["nextContinuationData"]; if(!next_continuation_data_json.isObject()) return ""; const Json::Value &continuation_json = next_continuation_data_json["continuation"]; if(!continuation_json.isString()) return ""; return continuation_json.asString(); } static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::string &continuation_token, BodyItems &result_items) { if(continuation_token.empty()) continuation_token = item_section_renderer_get_continuation_token(item_section_renderer_json); const Json::Value &item_contents_json = item_section_renderer_json["contents"]; if(!item_contents_json.isArray()) return; for(const Json::Value &content_item_json : item_contents_json) { if(!content_item_json.isObject()) continue; const Json::Value &video_renderer_json = content_item_json["videoRenderer"]; if(!video_renderer_json.isObject()) continue; const Json::Value &video_id_json = video_renderer_json["videoId"]; if(!video_id_json.isString()) continue; std::string video_id_str = video_id_json.asString(); std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg"; const char *title = nullptr; const Json::Value &title_json = video_renderer_json["title"]; if(title_json.isObject()) { const Json::Value &runs_json = title_json["runs"]; if(runs_json.isArray() && !runs_json.empty()) { const Json::Value &first_runs_json = runs_json[0]; if(first_runs_json.isObject()) { const Json::Value &text_json = first_runs_json["text"]; if(text_json.isString()) title = text_json.asCString(); } } } if(!title) continue; auto body_item = std::make_unique(title); body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; body_item->thumbnail_url = std::move(thumbnail_url); result_items.push_back(std::move(body_item)); } } SuggestionResult Youtube::update_search_suggestions(const std::string &text, BodyItems &result_items) { std::string url = "https://youtube.com/results?search_query="; url += url_param_encode(text); std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + url } }; std::string website_data; if(download_to_string(url + "&pbj=1", website_data, additional_args, use_tor, true) != DownloadResult::OK) return SuggestionResult::NET_ERR; Json::Value json_root; Json::CharReaderBuilder json_builder; std::unique_ptr json_reader(json_builder.newCharReader()); std::string json_errors; if(!json_reader->parse(&website_data[0], &website_data[website_data.size()], &json_root, &json_errors)) { fprintf(stderr, "Youtube search json error: %s\n", json_errors.c_str()); return SuggestionResult::ERR; } if(!json_root.isArray()) return SuggestionResult::ERR; std::string continuation_token; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) return SuggestionResult::ERR; const Json::Value &tcsrr_json = contents_json["twoColumnSearchResultsRenderer"]; if(!tcsrr_json.isObject()) return SuggestionResult::ERR; const Json::Value &primary_contents_json = tcsrr_json["primaryContents"]; if(!primary_contents_json.isObject()) return SuggestionResult::ERR; const Json::Value §ion_list_renderer_json = primary_contents_json["sectionListRenderer"]; if(!section_list_renderer_json.isObject()) return SuggestionResult::ERR; const Json::Value &contents2_json = section_list_renderer_json["contents"]; if(!contents2_json.isArray()) return SuggestionResult::ERR; for(const Json::Value &item_json : contents2_json) { if(!item_json.isObject()) continue; const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; parse_item_section_renderer(item_section_renderer_json, continuation_token, result_items); } } // The continuation data can also contain continuation, but we ignore that for now. Only get the first continuation data if(!continuation_token.empty()) search_suggestions_get_continuation(url, continuation_token, result_items); return SuggestionResult::OK; } void Youtube::search_suggestions_get_continuation(const std::string &url, const std::string &continuation_token, BodyItems &result_items) { std::string next_url = url + "&pbj=1&ctoken=" + continuation_token; std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", "x-youtube-client-version: 2.20200626.03.00" }, { "-H", "referer: " + url } }; std::string website_data; if(download_to_string(next_url, website_data, additional_args, use_tor, true) != DownloadResult::OK) return; Json::Value json_root; Json::CharReaderBuilder json_builder; std::unique_ptr json_reader(json_builder.newCharReader()); std::string json_errors; if(!json_reader->parse(&website_data[0], &website_data[website_data.size()], &json_root, &json_errors)) { fprintf(stderr, "Youtube search continuation json error: %s\n", json_errors.c_str()); return; } if(!json_root.isArray()) return; std::string next_continuation_token; for(const Json::Value &json_item : json_root) { if(!json_item.isObject()) continue; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) continue; const Json::Value &continuation_contents_json = response_json["continuationContents"]; if(!continuation_contents_json.isObject()) continue; const Json::Value &item_section_continuation_json = continuation_contents_json["itemSectionContinuation"]; if(!item_section_continuation_json.isObject()) continue; // Note: item_section_continuation json object is compatible with item_section_renderer json object parse_item_section_renderer(item_section_continuation_json, next_continuation_token, result_items); } } static std::string get_playlist_id_from_url(const std::string &url) { std::string playlist_id = url; size_t list_index = playlist_id.find("&list="); if(list_index == std::string::npos) return playlist_id; return playlist_id.substr(list_index); } static std::string remove_index_from_playlist_url(const std::string &url) { std::string result = url; size_t index = result.rfind("&index="); if(index == std::string::npos) return result; return result.substr(0, index); } static std::unique_ptr parse_compact_video_renderer_json(const Json::Value &item_json) { const Json::Value &compact_video_renderer_json = item_json["compactVideoRenderer"]; if(!compact_video_renderer_json.isObject()) return nullptr; const Json::Value &video_id_json = compact_video_renderer_json["videoId"]; if(!video_id_json.isString()) return nullptr; std::string video_id_str = video_id_json.asString(); std::string thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg"; const char *title = nullptr; const Json::Value &title_json = compact_video_renderer_json["title"]; if(title_json.isObject()) { const Json::Value &simple_text_json = title_json["simpleText"]; if(simple_text_json.isString()) { title = simple_text_json.asCString(); } } if(!title) return nullptr; auto body_item = std::make_unique(title); body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; body_item->thumbnail_url = std::move(thumbnail_url); return body_item; } // TODO: Make this faster by using string search instead of parsing html. // TODO: If the result is a play BodyItems Youtube::get_related_media(const std::string &url) { BodyItems result_items; std::string modified_url = remove_index_from_playlist_url(url); std::string playlist_id = get_playlist_id_from_url(modified_url); if(playlist_id == last_related_media_playlist_id) { result_items.reserve(last_playlist_data.size()); for(auto &data : last_playlist_data) { result_items.push_back(std::make_unique(*data)); } return result_items; } std::string website_data; if(download_to_string(modified_url, website_data, {}, use_tor, true) != DownloadResult::OK) return result_items; size_t data_start = website_data.find("window[\"ytInitialData\"] = {"); if(data_start == std::string::npos) return result_items; data_start = data_start + 26; size_t data_end = find_end_of_json(website_data, data_start); if(data_end == std::string::npos) return result_items; Json::Value json_root; Json::CharReaderBuilder json_builder; std::unique_ptr json_reader(json_builder.newCharReader()); std::string json_errors; if(!json_reader->parse(&website_data[data_start], &website_data[data_end], &json_root, &json_errors)) { fprintf(stderr, "Youtube search json error: %s\n", json_errors.c_str()); return result_items; } const Json::Value &contents_json = json_root["contents"]; if(!contents_json.isObject()) return result_items; const Json::Value &tcwnr_json = contents_json["twoColumnWatchNextResults"]; if(!tcwnr_json.isObject()) return result_items; const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"]; if(!secondary_results_json.isObject()) return result_items; const Json::Value &secondary_results2_json = secondary_results_json["secondaryResults"]; if(!secondary_results2_json.isObject()) return result_items; const Json::Value &results_json = secondary_results2_json["results"]; if(!results_json.isArray()) return result_items; for(const Json::Value &item_json : results_json) { if(!item_json.isObject()) continue; auto body_item = parse_compact_video_renderer_json(item_json); if(body_item) result_items.push_back(std::move(body_item)); const Json::Value &compact_autoplay_renderer_json = item_json["compactAutoplayRenderer"]; if(!compact_autoplay_renderer_json.isObject()) continue; const Json::Value &item_contents_json = compact_autoplay_renderer_json["contents"]; if(!item_contents_json.isArray()) continue; for(const Json::Value &content_item_json : item_contents_json) { if(!content_item_json.isObject()) continue; const Json::Value &compact_video_renderer_json = content_item_json["compactVideoRenderer"]; if(!compact_video_renderer_json.isObject()) continue; auto body_item = parse_compact_video_renderer_json(content_item_json); result_items.push_back(std::move(body_item)); } } return result_items; } }