#include "../../plugins/Youtube.hpp" #include "../../include/Storage.hpp" #include "../../include/NetUtils.hpp" #include "../../include/StringUtils.hpp" #include "../../include/Scale.hpp" #include "../../include/Notification.hpp" #include "../../include/VideoPlayer.hpp" #include "../../include/Utils.hpp" #include "../../include/Theme.hpp" #include "../../include/Config.hpp" #include "../../plugins/utils/WatchProgress.hpp" #include "../../include/QuickMedia.hpp" #include #include extern "C" { #include } #include #include #include namespace QuickMedia { static const char *youtube_client_version = "x-youtube-client-version: 2.20210622.10.00"; static const std::array invidious_urls = { "yewtu.be", "invidious.snopyta.org", "invidious.kavin.rocks", "vid.puffyan.us", "invidious.exonip.de", "ytprivate.com", "invidious.silkky.cloud", "invidious-us.kavin.rocks", "inv.riverside.rocks", "y.com.cm", "invidious.io.lol" }; std::string invidious_url_to_youtube_url(const std::string &url) { std::string result = url; for(const std::string &invidious_url : invidious_urls) { const size_t index = url.find(invidious_url); if(index != std::string::npos) { result.replace(index, invidious_url.size(), "youtube.com"); break; } } return result; } bool youtube_url_extract_id(const std::string &youtube_url, std::string &youtube_video_id) { size_t index = youtube_url.find("youtube.com/watch?v="); if(index != std::string::npos) { index += 20; size_t end_index = youtube_url.find("&", index); if(end_index == std::string::npos) end_index = youtube_url.size(); youtube_video_id = youtube_url.substr(index, end_index - index); return true; } index = youtube_url.find("youtu.be/watch?v="); if(index != std::string::npos) { index += 17; size_t end_index = youtube_url.find("&", index); if(end_index == std::string::npos) end_index = youtube_url.size(); youtube_video_id = youtube_url.substr(index, end_index - index); return true; } index = youtube_url.find("youtube.com/shorts/"); if(index != std::string::npos) { index += 19; size_t end_index = youtube_url.find("?", index); if(end_index == std::string::npos) end_index = youtube_url.size(); youtube_video_id = youtube_url.substr(index, end_index - index); return true; } index = youtube_url.find("youtube.com/live/"); if(index != std::string::npos) { index += 17; size_t end_index = youtube_url.find("?", index); if(end_index == std::string::npos) end_index = youtube_url.size(); youtube_video_id = youtube_url.substr(index, end_index - index); return true; } index = youtube_url.find("youtu.be/"); if(index != std::string::npos) { index += 9; size_t end_index = youtube_url.find("?", index); if(end_index == std::string::npos) end_index = youtube_url.size(); youtube_video_id = youtube_url.substr(index, end_index - index); return true; } index = youtube_url.find("watch?v="); if(index != std::string::npos) { index += 8; size_t end_index = youtube_url.find("&", index); if(end_index == std::string::npos) end_index = youtube_url.size(); youtube_video_id = youtube_url.substr(index, end_index - index); return true; } return false; } bool youtube_url_extract_channel_id(const std::string &youtube_url, std::string &channel_id, std::string &channel_url) { size_t index = youtube_url.find("youtube.com/c/"); if(index != std::string::npos) { index += 14; size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/c/" + channel_id; return true; } index = youtube_url.find("youtu.be/c/"); if(index != std::string::npos) { index += 11; size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/c/" + channel_id; return true; } index = youtube_url.find("youtube.com/channel/"); if(index != std::string::npos) { index += 20; size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/channel/" + channel_id; return true; } index = youtube_url.find("youtu.be/channel/"); if(index != std::string::npos) { index += 17; size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/channel/" + channel_id; return true; } index = youtube_url.find("youtube.com/@"); if(index != std::string::npos) { index += 12; // 13 - 1, to include @ size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/" + channel_id; return true; } index = youtube_url.find("youtu.be/@"); if(index != std::string::npos) { index += 9; // 10 - 1, to include @ size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/" + channel_id; return true; } index = youtube_url.find("youtube.com/user/"); if(index != std::string::npos) { index += 17; size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/user/" + channel_id; return true; } index = youtube_url.find("youtu.be/user/"); if(index != std::string::npos) { index += 14; size_t end_index = youtube_url.find("/", index); if(end_index == std::string::npos) end_index = youtube_url.size(); channel_id = youtube_url.substr(index, end_index - index); channel_url = "https://www.youtube.com/user/" + channel_id; return true; } return false; } static bool youtube_url_extract_playlist_id(const std::string &youtube_url, std::string &playlist_id) { size_t start_index = youtube_url.find("&list="); if(start_index == std::string::npos) return false; start_index += 6; size_t end_index = youtube_url.find("&", start_index); if(end_index == std::string::npos) end_index = youtube_url.size(); playlist_id = youtube_url.substr(start_index, end_index - start_index); return true; } static std::mutex cookies_mutex; static std::string cookies_filepath; static std::string api_key; static std::string ysc; static std::string visitor_info1_live; static std::string cpn; static std::string header_get_cookie(const char *str, size_t size, const char *cookies_key) { const int cookie_key_len = strlen(cookies_key); const char *cookie_p = (const char*)memmem(str, size, cookies_key, cookie_key_len); if(!cookie_p) return ""; cookie_p += cookie_key_len; const void *end_p = memchr(cookie_p, ';', (size_t)(str + size - cookie_p)); if(!end_p) end_p = str + size; return std::string(cookie_p, (const char*)end_p); } static std::vector get_cookies() { std::lock_guard lock(cookies_mutex); if(cookies_filepath.empty()) { cpn.resize(16); generate_random_characters(cpn.data(), cpn.size(), "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_", 64); Path cookies_filepath_p; if(get_cookies_filepath(cookies_filepath_p, "youtube-custom") != 0) { show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); return {}; } // TODO: Re-enable this if the api key ever changes in the future. // Maybe also put signature decryption in the same request? since it requests the same page. #if 0 //api_key = youtube_page_find_api_key(); #else api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; #endif if(get_file_type(cookies_filepath_p) == FileType::REGULAR) { cookies_filepath = cookies_filepath_p.data; std::string file_content; if(file_get_content(cookies_filepath_p, file_content) != 0) { show_notification("QuickMedia", "Failed to load cookies to view youtube comments", Urgency::CRITICAL); return {}; } const size_t line_end_index = file_content.find('\n'); if(line_end_index == std::string::npos) { show_notification("QuickMedia", "Failed to load cookies to view youtube comments", Urgency::CRITICAL); return {}; } ysc = file_content.substr(0, line_end_index); visitor_info1_live = file_content.substr(line_end_index + 1); } else { // TODO: This response also contains INNERTUBE_API_KEY which is the api key above. Maybe that should be parsed? // TODO: Is there any way to bypass this? this is needed to set VISITOR_INFO1_LIVE which is required to read comments std::string response; if(download_head_to_string("https://www.youtube.com/embed/watch?v=jNQXAC9IVRw&gl=US&hl=en", response, true) == DownloadResult::OK) { string_split(response, "\r\n", [](const char *str, size_t size){ if(size > 11 && memcmp(str, "set-cookie:", 11) == 0) { if(ysc.empty()) { std::string ysc_cookie = header_get_cookie(str + 11, size - 11, "YSC="); if(!ysc_cookie.empty()) ysc = std::move(ysc_cookie); } if(visitor_info1_live.empty()) { std::string visitor_info = header_get_cookie(str + 11, size - 11, "VISITOR_INFO1_LIVE="); if(!visitor_info.empty()) visitor_info1_live = std::move(visitor_info); } } return true; }); if(ysc.empty() || visitor_info1_live.empty() || file_overwrite_atomic(cookies_filepath_p, ysc + "\n" + visitor_info1_live) != 0) { show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); return {}; } } else { show_notification("QuickMedia", "Failed to fetch cookies to view youtube comments", Urgency::CRITICAL); return {}; } } } return { CommandArg{ "-H", "cookie: YSC=" + ysc + "; VISITOR_INFO1_LIVE=" + visitor_info1_live + "; CONSENT=YES+SE.sv+V10" } }; } // Sometimes youtube returns a redirect url (not in the header but in the body...). // TODO: Find why this happens and if there is a way bypass it. static std::string get_playback_url_recursive(std::string playback_url, int64_t &content_length) { std::vector additional_args = get_cookies(); additional_args.push_back({"--no-buffer", ""}); std::vector response_headers; const int max_redirects = 5; for(int i = 0; i < max_redirects; ++i) { std::string response_body; response_headers.clear(); download_to_string(playback_url, response_body, additional_args, true, true, false, &response_headers, 4096); if(response_headers.empty()) { fprintf(stderr, "Youtube video header not found\n"); return ""; } std::string content_type = header_extract_value(response_headers.back(), "content-type"); if(content_type.empty()) { fprintf(stderr, "Failed to find content-type in youtube video header\n"); return ""; } if(string_starts_with(content_type, "video") || string_starts_with(content_type, "audio")) { std::string content_length_str = header_extract_value(response_headers.back(), "content-length"); if(content_length_str.empty()) return ""; errno = 0; char *endptr; content_length = strtoll(content_length_str.c_str(), &endptr, 10); if(endptr == content_length_str.c_str() || errno != 0) return ""; return playback_url; } if(response_body.empty()) { fprintf(stderr, "Failed to redirect youtube video\n"); return ""; } playback_url = std::move(response_body); } return playback_url; } bool youtube_custom_redirect(std::string &video_url, std::string &audio_url, int64_t &video_content_length, int64_t &audio_content_length, std::function active_handler) { // TODO: Do this without threads int num_total_tasks = 0; AsyncTask tasks[2]; if(!video_url.empty()) { tasks[0] = AsyncTask([video_url, &video_content_length]() { return get_playback_url_recursive(std::move(video_url), video_content_length); }); ++num_total_tasks; } if(!audio_url.empty()) { tasks[1] = AsyncTask([audio_url, &audio_content_length]() { return get_playback_url_recursive(std::move(audio_url), audio_content_length); }); ++num_total_tasks; } if(num_total_tasks == 0) return false; int num_finished_tasks = 0; std::string *strings[2] = { &video_url, &audio_url }; while(true) { for(int i = 0; i < 2; ++i) { if(!tasks[i].ready()) continue; *strings[i] = tasks[i].get(); if(strings[i]->empty()) return false; ++num_finished_tasks; if(num_finished_tasks == num_total_tasks) return true; } if(!active_handler()) return false; std::this_thread::sleep_for(std::chrono::milliseconds(5)); } return true; } // This is a common setup of text in the youtube json static std::optional yt_json_get_text(const Json::Value &json, const char *root_name) { if(!json.isObject()) return std::nullopt; const Json::Value &text_json = json[root_name]; if(!text_json.isObject()) return std::nullopt; const Json::Value &simple_text_json = text_json["simpleText"]; if(simple_text_json.isString()) { return simple_text_json.asString(); } else { const Json::Value &runs_json = text_json["runs"]; if(!runs_json.isArray() || runs_json.empty()) return std::nullopt; std::string result; for(const Json::Value &first_runs_json : runs_json) { if(!first_runs_json.isObject()) continue; const Json::Value &text_json = first_runs_json["text"]; if(text_json.isString()) result += text_json.asString(); } if(!result.empty()) return result; } return std::nullopt; } struct Thumbnail { const char *url; int width; int height; }; enum class ThumbnailSize { SMALLEST, MEDIUM, LARGEST }; static std::optional yt_json_get_thumbnail(const Json::Value &thumbnail_json, ThumbnailSize thumbnail_size) { if(!thumbnail_json.isObject()) return std::nullopt; const Json::Value &thumbnails_json = thumbnail_json["thumbnails"]; if(!thumbnails_json.isArray()) return std::nullopt; std::vector thumbnails; for(const Json::Value &thumbnail_data_json : thumbnails_json) { if(!thumbnail_data_json.isObject()) continue; const Json::Value &url_json = thumbnail_data_json["url"]; if(!url_json.isString()) continue; const Json::Value &width_json = thumbnail_data_json["width"]; if(!width_json.isInt()) continue; const Json::Value &height_json = thumbnail_data_json["height"]; if(!height_json.isInt()) continue; thumbnails.push_back({ url_json.asCString(), width_json.asInt(), height_json.asInt() }); } if(thumbnails.empty()) return std::nullopt; switch(thumbnail_size) { case ThumbnailSize::SMALLEST: return *std::min_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { int size1 = thumbnail1.width * thumbnail1.height; int size2 = thumbnail2.width * thumbnail2.height; return size1 < size2; }); case ThumbnailSize::MEDIUM: { std::sort(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { int size1 = thumbnail1.width * thumbnail1.height; int size2 = thumbnail2.width * thumbnail2.height; return size1 < size2; }); return thumbnails[thumbnails.size() / 2]; } case ThumbnailSize::LARGEST: return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) { int size1 = thumbnail1.width * thumbnail1.height; int size2 = thumbnail2.width * thumbnail2.height; return size1 < size2; }); } return std::nullopt; } static bool video_is_live(const Json::Value &video_item_json) { if(!video_item_json.isObject()) return false; const Json::Value &badges_json = video_item_json["badges"]; if(!badges_json.isArray()) return false; for(const Json::Value &badge_json : badges_json) { if(!badge_json.isObject()) continue; const Json::Value &metadata_badge_renderer_json = badge_json["metadataBadgeRenderer"]; if(!metadata_badge_renderer_json.isObject()) continue; const Json::Value &style_json = metadata_badge_renderer_json["style"]; if(!style_json.isString()) continue; if(strcmp(style_json.asCString(), "BADGE_STYLE_TYPE_LIVE_NOW") == 0) return true; } return false; } static std::optional video_item_get_reel_published_date(const Json::Value &video_item_json) { const Json::Value &navigation_endpoint_json = video_item_json["navigationEndpoint"]; if(!navigation_endpoint_json.isObject()) return std::nullopt; const Json::Value &reel_watch_endpoint_json = navigation_endpoint_json["reelWatchEndpoint"]; if(!reel_watch_endpoint_json.isObject()) return std::nullopt; const Json::Value &overlay_json = reel_watch_endpoint_json["overlay"]; if(!overlay_json.isObject()) return std::nullopt; const Json::Value &reel_player_overlay_renderer_json = overlay_json["reelPlayerOverlayRenderer"]; if(!reel_player_overlay_renderer_json.isObject()) return std::nullopt; const Json::Value &reel_player_header_supported_renderers_json = reel_player_overlay_renderer_json["reelPlayerHeaderSupportedRenderers"]; if(!reel_player_header_supported_renderers_json.isObject()) return std::nullopt; const Json::Value &reel_player_header_renderer_json = reel_player_header_supported_renderers_json["reelPlayerHeaderRenderer"]; if(!reel_player_header_renderer_json.isObject()) return std::nullopt; return yt_json_get_text(reel_player_header_renderer_json, "timestampText"); } static std::shared_ptr parse_common_video_item(const Json::Value &video_item_json, std::unordered_set &added_videos) { if(!video_item_json.isObject()) return nullptr; const Json::Value &video_id_json = video_item_json["videoId"]; if(!video_id_json.isString()) return nullptr; std::string video_id_str = video_id_json.asString(); if(added_videos.find(video_id_str) != added_videos.end()) return nullptr; std::optional title = yt_json_get_text(video_item_json, "title"); std::optional headline = yt_json_get_text(video_item_json, "headline"); if(!title && !headline) return nullptr; if(!title) title = std::move(headline); std::optional video_info = yt_json_get_text(video_item_json, "videoInfo"); std::optional date = yt_json_get_text(video_item_json, "publishedTimeText"); std::optional view_count_text = yt_json_get_text(video_item_json, "viewCountText"); std::optional owner_text = yt_json_get_text(video_item_json, "shortBylineText"); std::optional description_snippet = yt_json_get_text(video_item_json, "descriptionSnippet"); std::optional length = yt_json_get_text(video_item_json, "lengthText"); if(!length) { const Json::Value &thumbnail_overlays_json = video_item_json["thumbnailOverlays"]; if(thumbnail_overlays_json.isArray() && !thumbnail_overlays_json.empty()) { const Json::Value &thumbnail_overlay_json = thumbnail_overlays_json[0]; if(thumbnail_overlay_json.isObject()) length = yt_json_get_text(thumbnail_overlay_json["thumbnailOverlayTimeStatusRenderer"], "text"); } } if(!date) date = video_item_get_reel_published_date(video_item_json); std::string scheduled_text; const Json::Value &upcoming_event_data_json = video_item_json["upcomingEventData"]; if(upcoming_event_data_json.isObject()) { const Json::Value &start_time_json = upcoming_event_data_json["startTime"]; if(!start_time_json.isString()) return nullptr; std::optional upcoming_event_text = yt_json_get_text(upcoming_event_data_json, "upcomingEventText"); if(!upcoming_event_text) return nullptr; time_t start_time = strtoll(start_time_json.asCString(), nullptr, 10); struct tm message_tm; localtime_r(&start_time, &message_tm); char time_str[128] = {0}; strftime(time_str, sizeof(time_str) - 1, "%Y %b %d, %a %H:%M", &message_tm); string_replace_all(upcoming_event_text.value(), "DATE_PLACEHOLDER", time_str); scheduled_text = std::move(upcoming_event_text.value()); } auto body_item = BodyItem::create(title.value()); std::string desc; if(video_info) { desc += video_info.value(); } else { if(view_count_text) desc += view_count_text.value(); if(date) { if(!desc.empty()) desc += " • "; desc += date.value(); } } if(!scheduled_text.empty()) { if(!desc.empty()) desc += " • "; desc += scheduled_text; } if(length) { if(!desc.empty()) desc += '\n'; desc += length.value(); } if(video_is_live(video_item_json)) { if(!desc.empty()) desc += '\n'; desc += "Live now"; } if(owner_text) { if(!desc.empty()) desc += '\n'; desc += owner_text.value(); } /*if(description_snippet) { if(!desc.empty()) desc += '\n'; desc += '\n'; std::string description_snippet_stripped = strip(description_snippet.value()); string_replace_all(description_snippet_stripped, "\n\n", "\n"); desc += std::move(description_snippet_stripped); }*/ body_item->set_description(std::move(desc)); body_item->set_description_color(get_theme().faded_text_color); if(scheduled_text.empty()) body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; body_item->thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/mqdefault.jpg"; body_item->thumbnail_size = mgl::vec2i(192, 108); added_videos.insert(video_id_str); return body_item; } static std::shared_ptr parse_channel_renderer(const Json::Value &channel_renderer_json) { if(!channel_renderer_json.isObject()) return nullptr; const Json::Value &channel_id_json = channel_renderer_json["channelId"]; if(!channel_id_json.isString()) return nullptr; std::optional title = yt_json_get_text(channel_renderer_json, "title"); if(!title) return nullptr; std::optional description = yt_json_get_text(channel_renderer_json, "descriptionSnippet"); std::optional video_count = yt_json_get_text(channel_renderer_json, "videoCountText"); std::optional subscribers = yt_json_get_text(channel_renderer_json, "subscriberCountText"); const Json::Value &thumbnail_json = channel_renderer_json["thumbnail"]; std::optional thumbnail = yt_json_get_thumbnail(thumbnail_json, ThumbnailSize::LARGEST); auto body_item = BodyItem::create(title.value()); std::string desc; if(subscribers) desc += subscribers.value(); if(video_count) { if(!desc.empty()) desc += " • "; desc += video_count.value(); } /*if(description) { if(!desc.empty()) desc += '\n'; desc += '\n'; std::string description_snippet_stripped = strip(description.value()); string_replace_all(description_snippet_stripped, "\n\n", "\n"); desc += std::move(description_snippet_stripped); }*/ body_item->set_description(std::move(desc)); body_item->set_description_color(get_theme().faded_text_color); body_item->url = "https://www.youtube.com/channel/" + channel_id_json.asString(); if(thumbnail) { if(string_starts_with(thumbnail->url, "https:")) body_item->thumbnail_url = thumbnail->url; else body_item->thumbnail_url = std::string("https:") + thumbnail->url; body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE; body_item->thumbnail_size.x = thumbnail->width; body_item->thumbnail_size.y = thumbnail->height; body_item->thumbnail_size = clamp_to_size(body_item->thumbnail_size, mgl::vec2i(136, 136)); } return body_item; } // Returns empty string if continuation token can't be found static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) { const Json::Value &continuation_item_renderer_json = item_section_renderer_json["continuationItemRenderer"]; if(!continuation_item_renderer_json.isObject()) return ""; const Json::Value &continuation_endpoint_json = continuation_item_renderer_json["continuationEndpoint"]; if(!continuation_endpoint_json.isObject()) return ""; const Json::Value &continuation_command_json = continuation_endpoint_json["continuationCommand"]; if(!continuation_command_json.isObject()) return ""; const Json::Value &token_json = continuation_command_json["token"]; if(!token_json.isString()) return ""; return token_json.asString(); } static std::shared_ptr parse_child_video_renderer(const Json::Value &child_video_renderer_json) { if(!child_video_renderer_json.isObject()) return nullptr; const Json::Value &video_id_json = child_video_renderer_json["videoId"]; if(!video_id_json.isString()) return nullptr; std::optional title = yt_json_get_text(child_video_renderer_json, "title"); std::optional length = yt_json_get_text(child_video_renderer_json, "lengthText"); std::string title_str = title.value_or("No title") + " • " + length.value_or("0:00"); auto body_item = BodyItem::create(std::move(title_str)); body_item->url = video_id_json.asString(); return body_item; } static bool navigation_endpoint_get_video_id(const Json::Value &navigation_endpoint_json, std::string &video_id) { if(!navigation_endpoint_json.isObject()) return false; const Json::Value &watch_endpoint_json = navigation_endpoint_json["watchEndpoint"]; if(!watch_endpoint_json.isObject()) return false; const Json::Value &video_id_json = watch_endpoint_json["videoId"]; if(!video_id_json.isString()) return false; video_id = video_id_json.asString(); return true; } static std::shared_ptr parse_playlist_renderer(const Json::Value &playlist_renderer_json) { if(!playlist_renderer_json.isObject()) return nullptr; const Json::Value &playlist_id_json = playlist_renderer_json["playlistId"]; const Json::Value &videos_json = playlist_renderer_json["videos"]; const Json::Value &video_count_json = playlist_renderer_json["videoCount"]; if(!playlist_id_json.isString()) return nullptr; std::optional video_count_short_text = yt_json_get_text(playlist_renderer_json, "videoCountShortText"); std::string video_count_text; if(video_count_short_text) video_count_text = video_count_short_text.value(); else if(video_count_json.isString()) video_count_text = video_count_json.asString(); std::optional title = yt_json_get_text(playlist_renderer_json, "title"); std::optional thumbnail; const Json::Value &thumbnail_json = playlist_renderer_json["thumbnail"]; if(thumbnail_json.isObject()) thumbnail = yt_json_get_thumbnail(thumbnail_json, ThumbnailSize::LARGEST); const Json::Value &thumbnails_json = playlist_renderer_json["thumbnails"]; if(thumbnails_json.isArray() && !thumbnails_json.empty()) thumbnail = yt_json_get_thumbnail(thumbnails_json[0], ThumbnailSize::LARGEST); std::string long_byline_text = yt_json_get_text(playlist_renderer_json, "longBylineText").value_or(""); if(long_byline_text.empty()) long_byline_text = yt_json_get_text(playlist_renderer_json, "shortBylineText").value_or(""); std::optional published_time_text = yt_json_get_text(playlist_renderer_json, "publishedTimeText"); std::string video_id; std::string description = std::move(long_byline_text); if(published_time_text) { if(!description.empty()) description += " • "; description += published_time_text.value(); } if(!description.empty()) description += '\n'; description += video_count_text + " video" + (strcmp(video_count_text.c_str(), "1") == 0 ? "" : "s"); if(videos_json.isArray()) { for(const Json::Value &video_json : videos_json) { if(!video_json.isObject()) continue; auto video_body_item = parse_child_video_renderer(video_json["childVideoRenderer"]); if(video_body_item) { //description += '\n'; //description += video_body_item->get_title(); if(video_id.empty()) video_id = video_body_item->url; } } } if(video_id.empty()) navigation_endpoint_get_video_id(playlist_renderer_json["navigationEndpoint"], video_id); if(video_id.empty()) return nullptr; auto body_item = BodyItem::create(title.value_or("No title")); body_item->set_description(std::move(description)); body_item->set_description_color(get_theme().faded_text_color); body_item->url = "https://www.youtube.com/watch?v=" + video_id + "&list=" + playlist_id_json.asString(); if(thumbnail) { body_item->thumbnail_url = thumbnail->url; body_item->thumbnail_size = { thumbnail->width, thumbnail->height }; } return body_item; } static void parse_item_section_renderer_shelf_renderer(const Json::Value &shelf_renderer_json, std::unordered_set &added_videos, BodyItems &result_items) { if(!shelf_renderer_json.isObject()) return; const Json::Value &item_content_json = shelf_renderer_json["content"]; if(!item_content_json.isObject()) return; const Json::Value &vertical_list_renderer_json = item_content_json["verticalListRenderer"]; if(!vertical_list_renderer_json.isObject()) return; const Json::Value &items_json = vertical_list_renderer_json["items"]; if(!items_json.isArray()) return; for(const Json::Value &item_json : items_json) { if(!item_json.isObject()) continue; std::shared_ptr body_item = parse_common_video_item(item_json["videoRenderer"], added_videos); if(body_item) result_items.push_back(std::move(body_item)); // TODO: youtube mix //body_item = parse_playlist_renderer(item_json["radioRenderer"]); //if(body_item) // result_items.push_back(std::move(body_item)); } } static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::unordered_set &added_videos, BodyItems &result_items) { if(!item_section_renderer_json.isObject()) return; const Json::Value &item_contents_json = item_section_renderer_json["contents"]; if(!item_contents_json.isArray()) return; std::shared_ptr body_item; for(const Json::Value &content_item_json : item_contents_json) { if(!content_item_json.isObject()) continue; body_item = parse_channel_renderer(content_item_json["channelRenderer"]); if(body_item) result_items.push_back(std::move(body_item)); body_item = parse_playlist_renderer(content_item_json["playlistRenderer"]); if(body_item) result_items.push_back(std::move(body_item)); // TODO: youtube mix //body_item = parse_playlist_renderer(content_item_json["radioRenderer"]); //if(body_item) // result_items.push_back(std::move(body_item)); parse_item_section_renderer_shelf_renderer(content_item_json["shelfRenderer"], added_videos, result_items); body_item = parse_common_video_item(content_item_json["videoRenderer"], added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } static std::shared_ptr parse_compact_video_renderer_json(const Json::Value &item_json, std::unordered_set &added_videos) { const Json::Value &compact_video_renderer_json = item_json["compactVideoRenderer"]; if(!compact_video_renderer_json.isObject()) return nullptr; return parse_common_video_item(compact_video_renderer_json, added_videos); } static std::string item_section_renderer_get_continuation(const Json::Value &item_section_renderer) { if(!item_section_renderer.isObject()) return ""; const Json::Value &contents_json = item_section_renderer["contents"]; if(!contents_json.isArray()) return ""; std::string continuation_token; for(const Json::Value &json_item : contents_json) { if(!json_item.isObject()) continue; continuation_token = item_section_renderer_get_continuation_token(json_item); if(!continuation_token.empty()) return continuation_token; } return ""; } static void parse_playlist_video_list(const Json::Value &playlist_video_list_json, const char *list_name, std::string &continuation_token, std::unordered_set &added_videos, BodyItems &body_items) { if(!playlist_video_list_json.isObject()) return; const Json::Value &contents_json = playlist_video_list_json[list_name]; if(!contents_json.isArray()) return; for(const Json::Value &content_json : contents_json) { if(!content_json.isObject()) continue; if(continuation_token.empty()) continuation_token = item_section_renderer_get_continuation_token(content_json); auto body_item = parse_common_video_item(content_json["playlistVideoRenderer"], added_videos); if(body_item) body_items.push_back(std::move(body_item)); } } static void parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set &added_videos, std::string &browse_id, BodyItems &body_items) { if(!json_root.isObject()) return; const Json::Value &endpoint_json = json_root["endpoint"]; if(endpoint_json.isObject()) { const Json::Value &browse_endpoint_json = endpoint_json["browseEndpoint"]; if(browse_endpoint_json.isObject()) { const Json::Value &browse_id_json = browse_endpoint_json["browseId"]; if(browse_id_json.isString()) browse_id = browse_id_json.asString(); } } const Json::Value *response_json = &json_root["response"]; if(!response_json->isObject()) response_json = &json_root; const Json::Value &contents_json = (*response_json)["contents"]; if(!contents_json.isObject()) return; const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"]; if(!tcbrr_json.isObject()) return; const Json::Value &tabs_json = tcbrr_json["tabs"]; if(!tabs_json.isArray()) return; for(const Json::Value &tab_json : tabs_json) { if(!tab_json.isObject()) continue; const Json::Value &tab_renderer_json = tab_json["tabRenderer"]; if(!tab_renderer_json.isObject()) continue; const Json::Value &content_json = tab_renderer_json["content"]; if(!content_json.isObject()) continue; const Json::Value §ion_list_renderer = content_json["sectionListRenderer"]; if(!section_list_renderer.isObject()) { const Json::Value &rich_grid_renderer_json = content_json["richGridRenderer"]; if(!rich_grid_renderer_json.isObject()) continue; const Json::Value &contents2_json = rich_grid_renderer_json["contents"]; if(!contents2_json.isArray()) continue; for(const Json::Value &content_item_json : contents2_json) { if(!content_item_json.isObject()) continue; if(continuation_token.empty()) continuation_token = item_section_renderer_get_continuation_token(content_item_json); const Json::Value &rich_item_renderer_json = content_item_json["richItemRenderer"]; if(!rich_item_renderer_json.isObject()) continue; const Json::Value &item_content_json = rich_item_renderer_json["content"]; if(!item_content_json.isObject()) continue; const Json::Value *video_renderer_json = &item_content_json["videoRenderer"]; if(!video_renderer_json->isObject()) { video_renderer_json = &item_content_json["reelItemRenderer"]; if(!video_renderer_json->isObject()) continue; } auto body_item = parse_common_video_item(*video_renderer_json, added_videos); if(body_item) body_items.push_back(std::move(body_item)); } continue; } const Json::Value &contents2_json = section_list_renderer["contents"]; if(!contents2_json.isArray()) continue; for(const Json::Value &content_item_json : contents2_json) { if(!content_item_json.isObject()) continue; const Json::Value &item_section_renderer_json = content_item_json["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; const Json::Value &item_contents_json = item_section_renderer_json["contents"]; if(!item_contents_json.isArray()) continue; for(const Json::Value &content_json : item_contents_json) { if(!content_json.isObject()) continue; parse_playlist_video_list(content_json["playlistVideoListRenderer"], "contents", continuation_token, added_videos, body_items); const Json::Value &grid_renderer_json = content_json["gridRenderer"]; if(!grid_renderer_json.isObject()) continue; const Json::Value &items_json = grid_renderer_json["items"]; if(!items_json.isArray()) continue; for(const Json::Value &item_json : items_json) { if(!item_json.isObject()) continue; if(continuation_token.empty()) continuation_token = item_section_renderer_get_continuation_token(item_json); std::shared_ptr body_item = parse_common_video_item(item_json["gridVideoRenderer"], added_videos); if(body_item) body_items.push_back(std::move(body_item)); body_item = parse_playlist_renderer(item_json["gridPlaylistRenderer"]); if(body_item) body_items.push_back(std::move(body_item)); } } } } } static void parse_section_list_renderer(const Json::Value §ion_list_renderer_json, std::string &continuation_token, BodyItems &result_items, std::unordered_set &added_videos) { if(!section_list_renderer_json.isObject()) return; const Json::Value &contents2_json = section_list_renderer_json["contents"]; if(!contents2_json.isArray()) return; for(const Json::Value &item_json : contents2_json) { if(!item_json.isObject()) continue; if(continuation_token.empty()) continuation_token = item_section_renderer_get_continuation_token(item_json); const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); } } static SearchResult invidious_get_popular_feed(Page *page, const std::string &invidious_instance, BodyItems &result_items) { Json::Value json_root; std::string err_msg; DownloadResult download_result = page->download_json(json_root, invidious_instance + "/api/v1/popular", {}); if(download_result != DownloadResult::OK) return download_result_to_search_result(download_result); if(!json_root.isArray()) return SearchResult::ERR; for(const Json::Value &item_json : json_root) { if(!item_json.isObject()) continue; const Json::Value &title_json = item_json["title"]; const Json::Value &video_id_json = item_json["videoId"]; const Json::Value &length_seconds_json = item_json["lengthSeconds"]; const Json::Value &author_json = item_json["author"]; const Json::Value &published_text_json = item_json["publishedText"]; const Json::Value &view_count_json = item_json["viewCount"]; if(!title_json.isString() || !video_id_json.isString()) continue; std::string video_id_str = video_id_json.asString(); auto body_item = BodyItem::create(title_json.asString()); std::string desc; if(view_count_json.isInt64()) { std::string views = number_separate_thousand_commas(std::to_string(view_count_json.asInt64())); desc += views + " view" + (view_count_json.asInt64() == 1 ? "" : "s"); } if(published_text_json.isString()) { if(!desc.empty()) desc += " • "; desc += published_text_json.asString(); } if(length_seconds_json.isInt64()) { if(!desc.empty()) desc += '\n'; desc += seconds_to_duration(length_seconds_json.asInt64()); } if(author_json.isString()) { if(!desc.empty()) desc += '\n'; desc += author_json.asString(); } body_item->set_description(std::move(desc)); body_item->set_description_color(get_theme().faded_text_color); body_item->url = "https://www.youtube.com/watch?v=" + video_id_str; body_item->thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/mqdefault.jpg"; body_item->thumbnail_size = mgl::vec2i(192, 108); result_items.push_back(std::move(body_item)); } return SearchResult::OK; } SearchResult YoutubeSearchPage::search(const std::string &str, BodyItems &result_items) { continuation_token.clear(); current_page = 0; added_videos.clear(); if(str.empty()) { const std::string &invidious_instance = get_config().youtube.invidious_instance; if(invidious_instance.empty()) { program->fill_recommended_items_from_json("youtube", program->load_recommended_json("youtube"), result_items); return SearchResult::OK; } else { return invidious_get_popular_feed(this, invidious_instance, result_items); } } // TODO: Find this search url from youtube.com/... searchbox.js, and the url to that script from youtube.com/ html std::string url = "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&hl=en&gl=us&sugexp=rdcfrc%2Ccfro%3D1%2Cfp.cfr%3D1&gs_rn=64&gs_ri=youtube&ds=yt&cp=34&gs_id=f&xhr=t&xssi=t&q="; url += url_param_encode(str); if(!video_id.empty()) url += "&video_id=" + video_id; std::vector additional_args = { { "-H", "origin: https://www.youtube.com" }, { "-H", "referer: https://www.youtube.com/" } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); std::string website_data; DownloadResult result = download_to_string(url, website_data, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_search_result(result); const size_t json_start_index = website_data.find('['); if(json_start_index == std::string::npos) return SearchResult::ERR; Json::Value json_root; Json::CharReaderBuilder json_builder; std::unique_ptr json_reader(json_builder.newCharReader()); std::string json_errors; if(!json_reader->parse(&website_data[json_start_index], &website_data[website_data.size()], &json_root, &json_errors)) { fprintf(stderr, "youtube search error: %s\n", json_errors.c_str()); return SearchResult::ERR; } if(!json_root.isArray() || json_root.size() < 2) return SearchResult::ERR; const Json::Value &search_result_list_json = json_root[1]; if(!search_result_list_json.isArray()) return SearchResult::ERR; for(const Json::Value &json_item : search_result_list_json) { if(!json_item.isArray() || json_item.empty()) continue; const Json::Value &search_result_json = json_item[0]; if(!search_result_json.isString()) continue; auto body_item = BodyItem::create(search_result_json.asString()); body_item->url = body_item->get_title(); result_items.push_back(std::move(body_item)); } if(result_items.empty() || !strcase_equals(str.c_str(), result_items.front()->get_title().c_str())) { auto body_item = BodyItem::create(str); body_item->url = str; result_items.insert(result_items.begin(), std::move(body_item)); } return SearchResult::OK; } PluginResult YoutubeSearchPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = search_get_continuation(search_url, continuation_token, result_items); if(plugin_result != PluginResult::OK) return plugin_result; ++current_page; } return PluginResult::OK; } PluginResult YoutubeSearchPage::submit(const SubmitArgs &args, std::vector &result_tabs) { if(args.url.empty()) return PluginResult::OK; if(strncmp(args.url.c_str(), "https://www.youtube.com/channel/", 32) == 0) { // TODO: Make all pages (for all services) lazy fetch in a similar manner! YoutubeChannelPage::create_each_type(program, args.url, "", args.title, result_tabs); } else if(strstr(args.url.c_str(), "&list=")) { result_tabs.push_back(Tab{create_body(false, true), std::make_unique(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)}); } else { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url), nullptr}); } return PluginResult::OK; } static void search_page_submit_suggestion_handler(const Json::Value &json_item, std::string &continuation_token, BodyItems &result_items, std::unordered_set &added_videos) { if(!json_item.isObject()) return; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) return; const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) return; const Json::Value &tcsrr_json = contents_json["twoColumnSearchResultsRenderer"]; if(!tcsrr_json.isObject()) return; const Json::Value &primary_contents_json = tcsrr_json["primaryContents"]; if(!primary_contents_json.isObject()) return; parse_section_list_renderer(primary_contents_json["sectionListRenderer"], continuation_token, result_items, added_videos); } PluginResult YoutubeSearchPage::submit_suggestion(const SubmitArgs &args, BodyItems &result_items) { continuation_token.clear(); current_page = 0; added_videos.clear(); search_url = "https://www.youtube.com/results?search_query="; search_url += url_param_encode(args.url); std::vector additional_args = { { "-H", "x-spf-referer: " + search_url }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, { "-H", "referer: " + search_url } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, search_url + "&pbj=1&gl=US&hl=en", std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(json_root.isObject()) { search_page_submit_suggestion_handler(json_root, continuation_token, result_items, added_videos); return PluginResult::OK; } if(!json_root.isArray()) return PluginResult::ERR; for(const Json::Value &json_item : json_root) { search_page_submit_suggestion_handler(json_item, continuation_token, result_items, added_videos); } return PluginResult::OK; } PluginResult YoutubeSearchPage::lazy_fetch(BodyItems &result_items) { continuation_token.clear(); current_page = 0; added_videos.clear(); search("", result_items); get_cookies(); return PluginResult::OK; } bool YoutubeSearchPage::reload_on_page_change() { return get_config().youtube.invidious_instance.empty(); } static void search_page_search_get_continuation(const Json::Value &json_item, std::string &new_continuation_token, std::unordered_set &added_videos, BodyItems &result_items) { if(!json_item.isObject()) return; const Json::Value &response_json = json_item["response"]; if(!response_json.isObject()) return; const Json::Value &on_response_received_commands_json = response_json["onResponseReceivedCommands"]; if(!on_response_received_commands_json.isArray()) return; for(const Json::Value &response_received_command : on_response_received_commands_json) { if(!response_received_command.isObject()) continue; const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"]; if(!append_continuation_items_action_json.isObject()) continue; const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"]; if(!continuation_items_json.isArray()) continue; for(const Json::Value &continuation_item : continuation_items_json) { if(!continuation_item.isObject()) continue; if(new_continuation_token.empty()) { // Note: item_section_renderer is compatible with continuation_item new_continuation_token = item_section_renderer_get_continuation_token(continuation_item); } const Json::Value &item_section_renderer_json = continuation_item["itemSectionRenderer"]; if(!item_section_renderer_json.isObject()) continue; parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); } } } PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { if(current_continuation_token.empty()) return PluginResult::OK; std::string next_url = url + "&pbj=1&gl=US&hl=en&ctoken=" + current_continuation_token; std::string new_continuation_token; std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", youtube_client_version }, { "-H", "referer: " + url } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(json_root.isObject()) { search_page_search_get_continuation(json_root, new_continuation_token, added_videos, result_items); continuation_token = std::move(new_continuation_token); return PluginResult::OK; } if(!json_root.isArray()) return PluginResult::ERR; for(const Json::Value &json_item : json_root) { search_page_search_get_continuation(json_item, new_continuation_token, added_videos, result_items); } continuation_token = std::move(new_continuation_token); return PluginResult::OK; } PluginResult YoutubeCommentsPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = lazy_fetch(result_items); if(plugin_result != PluginResult::OK) return plugin_result; ++current_page; } return PluginResult::OK; } PluginResult YoutubeCommentsPage::submit(const SubmitArgs &args, std::vector &result_tabs) { if(args.url.empty()) return PluginResult::OK; const BodyItem *body_item = (BodyItem*)args.userdata; result_tabs.push_back(Tab{create_body(), std::make_unique(program, video_url, args.url, body_item), nullptr}); return PluginResult::OK; } static std::string item_section_continuation_get_continuation_token(const Json::Value &item_section_continuation_json) { if(!item_section_continuation_json.isObject()) return ""; const Json::Value &continuations_json = item_section_continuation_json["continuations"]; if(!continuations_json.isArray()) return ""; for(const Json::Value &json_item : continuations_json) { if(!json_item.isObject()) continue; const Json::Value &next_continuation_data_json = json_item["nextContinuationData"]; if(!next_continuation_data_json.isObject()) continue; const Json::Value &continuation_json = next_continuation_data_json["continuation"]; if(continuation_json.isString()) return continuation_json.asString(); } return ""; } static std::string comment_thread_renderer_get_replies_continuation(const Json::Value &comment_thread_renderer_json) { if(!comment_thread_renderer_json.isObject()) return ""; const Json::Value &replies_json = comment_thread_renderer_json["replies"]; if(!replies_json.isObject()) return ""; const Json::Value &comment_replies_renderer_json = replies_json["commentRepliesRenderer"]; std::string continuation = item_section_renderer_get_continuation(comment_replies_renderer_json); if(!continuation.empty()) return continuation; return item_section_continuation_get_continuation_token(comment_replies_renderer_json); } // Returns empty string if comment is not hearted static std::string comment_renderer_get_hearted_tooltip(const Json::Value &comment_renderer_json) { const Json::Value &action_buttons_json = comment_renderer_json["actionButtons"]; if(!action_buttons_json.isObject()) return ""; const Json::Value &comment_action_buttons_renderer_json = action_buttons_json["commentActionButtonsRenderer"]; if(!comment_action_buttons_renderer_json.isObject()) return ""; const Json::Value &creator_heart_json = comment_action_buttons_renderer_json["creatorHeart"]; if(!creator_heart_json.isObject()) return ""; const Json::Value &creator_heart_renderer_json = creator_heart_json["creatorHeartRenderer"]; if(!creator_heart_renderer_json.isObject()) return ""; const Json::Value &hearted_tooltip_json = creator_heart_renderer_json["heartedTooltip"]; if(!hearted_tooltip_json.isString()) return ""; return hearted_tooltip_json.asString(); } static std::shared_ptr comment_renderer_to_body_item(const Json::Value &comment_renderer_json) { if(!comment_renderer_json.isObject()) return nullptr; std::optional author_text = yt_json_get_text(comment_renderer_json, "authorText"); if(!author_text) return nullptr; std::string author = author_text.value(); std::optional published_time_text = yt_json_get_text(comment_renderer_json, "publishedTimeText"); if(published_time_text) author += " - " + published_time_text.value(); auto body_item = BodyItem::create(""); body_item->set_author(std::move(author)); std::string description; const Json::Value &author_is_channel_owner_json = comment_renderer_json["authorIsChannelOwner"]; if(author_is_channel_owner_json.isBool() && author_is_channel_owner_json.asBool()) body_item->set_author_color(mgl::Color(150, 255, 150)); std::optional comment = yt_json_get_text(comment_renderer_json, "contentText"); if(comment) description = comment.value(); std::optional thumbnail = yt_json_get_thumbnail(comment_renderer_json["authorThumbnail"], ThumbnailSize::SMALLEST); if(thumbnail) { body_item->thumbnail_url = thumbnail->url; body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE; body_item->thumbnail_size.x = thumbnail->width; body_item->thumbnail_size.y = thumbnail->height; body_item->thumbnail_size = body_item->thumbnail_size; } std::optional vote_count = yt_json_get_text(comment_renderer_json, "voteCount"); if(!description.empty()) description += '\n'; description += "👍 " + vote_count.value_or("0"); const Json::Value &reply_count_json = comment_renderer_json["replyCount"]; if(reply_count_json.isInt64() && reply_count_json.asInt64() > 0) { if(!description.empty()) description += '\n'; if(reply_count_json.asInt64() == 1) description += "1 reply"; else description += std::to_string(reply_count_json.asInt64()) + " replies"; } std::string hearted_tooltip = comment_renderer_get_hearted_tooltip(comment_renderer_json); if(!hearted_tooltip.empty()) { if(!description.empty()) description += " - "; description += std::move(hearted_tooltip); } body_item->set_description(std::move(description)); body_item->userdata = body_item.get(); return body_item; } static std::shared_ptr comment_entity_payload_to_body_item(const Json::Value &comment_entity_payload_json, std::string &heart_active_tooltip_text) { if(!comment_entity_payload_json.isObject()) return nullptr; const Json::Value &key_json = comment_entity_payload_json["key"]; if(!key_json.isString()) return nullptr; const Json::Value &properties_json = comment_entity_payload_json["properties"]; if(!properties_json.isObject()) return nullptr; const Json::Value &author_json = comment_entity_payload_json["author"]; if(!author_json.isObject()) return nullptr; const Json::Value &toolbar_json = comment_entity_payload_json["toolbar"]; if(!toolbar_json.isObject()) return nullptr; const Json::Value &content1_json = properties_json["content"]; if(!content1_json.isObject()) return nullptr; const Json::Value &content2_json = content1_json["content"]; if(!content2_json.isString()) return nullptr; const Json::Value &display_name_json = author_json["displayName"]; if(!display_name_json.isString()) return nullptr; std::string author = display_name_json.asString(); const Json::Value &published_time_json = properties_json["publishedTime"]; if(published_time_json.isString()) author += " - " + published_time_json.asString(); auto body_item = BodyItem::create(""); body_item->set_author(std::move(author)); body_item->url = key_json.asString(); const Json::Value &is_creator_json = author_json["isCreator"]; if(is_creator_json.isBool() && is_creator_json.asBool()) body_item->set_author_color(mgl::Color(150, 255, 150)); std::string description = content2_json.asString(); const Json::Value &avatar_thumbnail_url = author_json["avatarThumbnailUrl"]; if(avatar_thumbnail_url.isString()) { body_item->thumbnail_url = avatar_thumbnail_url.asString(); body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE; body_item->thumbnail_size.x = 48; body_item->thumbnail_size.y = 48; } const Json::Value &like_count_liked_json = toolbar_json["likeCountLiked"]; if(!description.empty()) description += '\n'; description += std::string("👍 ") + (like_count_liked_json.isString() ? like_count_liked_json.asString() : "0"); const Json::Value &reply_count_json = toolbar_json["replyCount"]; if(reply_count_json.isString() && reply_count_json.asCString()[0] != '\0' && reply_count_json.asCString()[0] != '0') { if(!description.empty()) description += '\n'; std::string reply_count_str = reply_count_json.asString(); if(reply_count_str == "1") description += "1 reply"; else description += std::move(reply_count_str) + " replies"; } const Json::Value &heart_active_tooltip_json = toolbar_json["heartActiveTooltip"]; if(heart_active_tooltip_json.isString()) { heart_active_tooltip_text = heart_active_tooltip_json.asString(); } body_item->set_description(std::move(description)); body_item->userdata = body_item.get(); return body_item; } static std::string continuation_item_renderer_get_continuation_token(const Json::Value &continuation_item_renderer_json) { if(!continuation_item_renderer_json.isObject()) return ""; const Json::Value &button_json = continuation_item_renderer_json["button"]; if(!button_json.isObject()) return ""; const Json::Value &button_renderer_json = button_json["buttonRenderer"]; if(!button_renderer_json.isObject()) return ""; const Json::Value &command_json = button_renderer_json["command"]; if(!command_json.isObject()) return ""; const Json::Value &continuation_command_json = command_json["continuationCommand"]; if(!continuation_command_json.isObject()) return ""; const Json::Value &token_json = continuation_command_json["token"]; if(!token_json.isString()) return ""; return token_json.asString(); } static std::string get_comment_key(const Json::Value &comment_thread_renderer_json) { std::string result; if(!comment_thread_renderer_json.isObject()) return result; const Json::Value &comment_view_model_json = comment_thread_renderer_json["commentViewModel"]; if(!comment_view_model_json.isObject()) return result; const Json::Value &comment_view_model2_json = comment_view_model_json["commentViewModel"]; if(!comment_view_model2_json.isObject()) return result; const Json::Value &comment_key_json = comment_view_model2_json["commentKey"]; if(comment_key_json.isString()) result = comment_key_json.asString(); return result; } static PluginResult fetch_comments_received_endpoints(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token, std::unordered_map &comment_reply_tokens_by_key) { const Json::Value &on_response_received_endpoints_json = json_root["onResponseReceivedEndpoints"]; if(!on_response_received_endpoints_json.isArray()) return PluginResult::ERR; std::string new_continuation_token; for(const Json::Value &json_item : on_response_received_endpoints_json) { if(!json_item.isObject()) continue; const Json::Value *append_continuation_items_action_json = &json_item["reloadContinuationItemsCommand"]; if(!append_continuation_items_action_json->isObject()) { append_continuation_items_action_json = &json_item["appendContinuationItemsAction"]; if(!append_continuation_items_action_json->isObject()) continue; } const Json::Value &continuation_items_json = (*append_continuation_items_action_json)["continuationItems"]; if(!continuation_items_json.isArray()) continue; for(const Json::Value &continuation_item_json : continuation_items_json) { if(!continuation_item_json.isObject()) continue; if(new_continuation_token.empty()) new_continuation_token = item_section_renderer_get_continuation_token(continuation_item_json); const Json::Value &comment_thread_renderer_json = continuation_item_json["commentThreadRenderer"]; if(comment_thread_renderer_json.isObject()) { std::string comment_key = get_comment_key(comment_thread_renderer_json); if(!comment_key.empty()) { comment_reply_tokens_by_key[comment_key] = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json); continue; } const Json::Value &comment_json = comment_thread_renderer_json["comment"]; if(!comment_json.isObject()) continue; auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]); if(!body_item) continue; body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json); result_items.push_back(std::move(body_item)); } else { auto body_item = comment_renderer_to_body_item(continuation_item_json["commentRenderer"]); if(body_item) result_items.push_back(std::move(body_item)); if(new_continuation_token.empty()) new_continuation_token = continuation_item_renderer_get_continuation_token(continuation_item_json["continuationItemRenderer"]); } } } continuation_token = std::move(new_continuation_token); return PluginResult::OK; } static PluginResult fetch_comments_continuation_contents(const Json::Value &json_root, BodyItems &result_items, std::string &continuation_token, std::unordered_map &comment_reply_tokens_by_key) { const Json::Value &continuation_contents_json = json_root["continuationContents"]; if(!continuation_contents_json.isObject()) return PluginResult::ERR; const Json::Value *item_section_continuation_json = &continuation_contents_json["itemSectionContinuation"]; if(!item_section_continuation_json->isObject()) { item_section_continuation_json = &continuation_contents_json["commentRepliesContinuation"]; if(!item_section_continuation_json->isObject()) return PluginResult::ERR; } const Json::Value &contents_json = (*item_section_continuation_json)["contents"]; if(!contents_json.isArray()) return PluginResult::ERR; std::string new_continuation_token; for(const Json::Value &json_item : contents_json) { if(!json_item.isObject()) continue; const Json::Value &comment_thread_renderer_json = json_item["commentThreadRenderer"]; if(comment_thread_renderer_json.isObject()) { std::string comment_key = get_comment_key(comment_thread_renderer_json); if(!comment_key.empty()) { comment_reply_tokens_by_key[comment_key] = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json); continue; } const Json::Value &comment_json = comment_thread_renderer_json["comment"]; if(!comment_json.isObject()) continue; auto body_item = comment_renderer_to_body_item(comment_json["commentRenderer"]); if(!body_item) continue; body_item->url = comment_thread_renderer_get_replies_continuation(comment_thread_renderer_json); result_items.push_back(std::move(body_item)); } else { auto body_item = comment_renderer_to_body_item(json_item["commentRenderer"]); if(body_item) result_items.push_back(std::move(body_item)); if(new_continuation_token.empty()) new_continuation_token = continuation_item_renderer_get_continuation_token(json_item["continuationItemRenderer"]); } } if(new_continuation_token.empty()) new_continuation_token = item_section_continuation_get_continuation_token(*item_section_continuation_json); continuation_token = std::move(new_continuation_token); return PluginResult::OK; } static PluginResult fetch_comments_framework_updates(const Json::Value &json_root, BodyItems &result_items, const std::unordered_map &comment_reply_tokens_by_key) { const Json::Value &framework_updates_json = json_root["frameworkUpdates"]; if(!framework_updates_json.isObject()) return PluginResult::ERR; const Json::Value &entity_batch_update_json = framework_updates_json["entityBatchUpdate"]; if(!entity_batch_update_json.isObject()) return PluginResult::ERR; const Json::Value &mutations_json = entity_batch_update_json["mutations"]; if(!mutations_json.isArray()) return PluginResult::ERR; std::string heart_active_tooltip_text; for(const Json::Value &item_json : mutations_json) { if(!item_json.isObject()) continue; const Json::Value &payload_json = item_json["payload"]; if(!payload_json.isObject()) continue; const Json::Value &eng_toolbar_state_ent_payload_json = payload_json["engagementToolbarStateEntityPayload"]; if(eng_toolbar_state_ent_payload_json.isObject()) { const Json::Value &heart_state_json = eng_toolbar_state_ent_payload_json["heartState"]; if(heart_state_json.isString() && strcmp(heart_state_json.asCString(), "TOOLBAR_HEART_STATE_HEARTED") == 0 && !result_items.empty()) { std::string description = result_items.back()->get_description(); if(!description.empty()) description += " - "; description += heart_active_tooltip_text; result_items.back()->set_description(std::move(description)); continue; } } const Json::Value &comment_entity_payload_json = payload_json["commentEntityPayload"]; if(!comment_entity_payload_json.isObject()) continue; auto body_item = comment_entity_payload_to_body_item(comment_entity_payload_json, heart_active_tooltip_text); if(!body_item) continue; auto it = comment_reply_tokens_by_key.find(body_item->url); if(it == comment_reply_tokens_by_key.end()) { body_item->url.clear(); } else { body_item->url = it->second; } result_items.push_back(std::move(body_item)); } return PluginResult::OK; } static PluginResult fetch_comments(Page *page, const std::string &video_url, std::string &continuation_token, BodyItems &result_items) { if(continuation_token.empty()) return PluginResult::OK; std::vector cookies = get_cookies(); std::string next_url = "https://www.youtube.com/youtubei/v1/next?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false"; Json::Value request_json(Json::objectValue); Json::Value context_json(Json::objectValue); Json::Value client_json(Json::objectValue); client_json["hl"] = "en"; client_json["gl"] = "US"; client_json["deviceMake"] = ""; client_json["deviceModel"] = ""; client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; client_json["clientName"] = "WEB"; client_json["clientVersion"] = "2.20210622.10.00"; client_json["osName"] = "X11"; client_json["osVersion"] = ""; client_json["originalUrl"] = video_url; context_json["client"] = std::move(client_json); request_json["context"] = std::move(context_json); request_json["continuation"] = continuation_token; Json::StreamWriterBuilder json_builder; json_builder["commentStyle"] = "None"; json_builder["indentation"] = ""; std::vector additional_args = { { "-H", "content-type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, { "--data-raw", Json::writeString(json_builder, request_json) } }; additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = page->download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(!json_root.isObject()) return PluginResult::ERR; std::unordered_map comment_reply_tokens_by_key; fetch_comments_received_endpoints(json_root, result_items, continuation_token, comment_reply_tokens_by_key); fetch_comments_continuation_contents(json_root, result_items, continuation_token, comment_reply_tokens_by_key); fetch_comments_framework_updates(json_root, result_items, comment_reply_tokens_by_key); return PluginResult::OK; } PluginResult YoutubeCommentsPage::lazy_fetch(BodyItems &result_items) { return fetch_comments(this, video_url, continuation_token, result_items); } PluginResult YoutubeCommentRepliesPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = lazy_fetch(result_items, false); if(plugin_result != PluginResult::OK) return plugin_result; ++current_page; } return PluginResult::OK; } PluginResult YoutubeCommentRepliesPage::submit(const SubmitArgs &args, std::vector&) { return PluginResult::OK; } PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items) { return lazy_fetch(result_items, true); } PluginResult YoutubeCommentRepliesPage::lazy_fetch(BodyItems &result_items, bool first_fetch) { if(first_fetch) { auto body_item = BodyItem::create(""); *body_item = *replied_to_body_item; body_item->set_author("(OP) " + body_item->get_author()); result_items.push_back(std::move(body_item)); } return fetch_comments(this, video_url, continuation_token, result_items); } static const char* youtube_channel_page_type_get_endpoint(YoutubeChannelPage::Type type) { switch(type) { case YoutubeChannelPage::Type::VIDEOS: return "videos"; case YoutubeChannelPage::Type::SHORTS: return "shorts"; case YoutubeChannelPage::Type::LIVE: return "streams"; case YoutubeChannelPage::Type::PLAYLISTS: return "playlists"; } return ""; } static std::string youtube_channel_page_type_to_title(const std::string &channel_name, YoutubeChannelPage::Type type) { switch(type) { case YoutubeChannelPage::Type::VIDEOS: return channel_name + " Videos"; case YoutubeChannelPage::Type::SHORTS: return channel_name + " Shorts"; case YoutubeChannelPage::Type::LIVE: return channel_name + " Live videos"; case YoutubeChannelPage::Type::PLAYLISTS: return channel_name + " Playlists"; } return ""; } // static void YoutubeChannelPage::create_each_type(Program *program, std::string url, std::string continuation_token, std::string title, std::vector &tabs) { tabs.push_back(Tab{program->create_body(false, true), std::make_unique(program, url, continuation_token, title, Type::VIDEOS), program->create_search_bar("Search...", 350)}); tabs.push_back(Tab{program->create_body(false, true), std::make_unique(program, url, continuation_token, title, Type::SHORTS), program->create_search_bar("Search...", 350)}); tabs.push_back(Tab{program->create_body(false, true), std::make_unique(program, url, continuation_token, title, Type::LIVE), program->create_search_bar("Search...", 350)}); tabs.push_back(Tab{program->create_body(false, true), std::make_unique(program, url, continuation_token, title, Type::PLAYLISTS), program->create_search_bar("Search...", 350)}); } YoutubeChannelPage::YoutubeChannelPage(Program *program, std::string url, std::string continuation_token, std::string title, Type type) : LazyFetchPage(program), TrackablePage(title, url), url(url), continuation_token(std::move(continuation_token)), title(youtube_channel_page_type_to_title(title, type)), type(type) {} SearchResult YoutubeChannelPage::search(const std::string &str, BodyItems &result_items) { added_videos.clear(); continuation_token.clear(); current_page = 0; if(str.empty()) return plugin_result_to_search_result(lazy_fetch(result_items)); std::string channel_id; std::string channel_url; if(!youtube_url_extract_channel_id(url, channel_id, channel_url)) { fprintf(stderr, "Error: failed to extract youtube channel id from url: %s\n", url.c_str()); return SearchResult::ERR; } std::vector cookies = get_cookies(); std::string next_url = "https://www.youtube.com/youtubei/v1/browse?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false"; Json::Value request_json(Json::objectValue); Json::Value context_json(Json::objectValue); Json::Value client_json(Json::objectValue); client_json["hl"] = "en"; client_json["gl"] = "US"; client_json["deviceMake"] = ""; client_json["deviceModel"] = ""; client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; client_json["clientName"] = "WEB"; client_json["clientVersion"] = "2.20210622.10.00"; client_json["osName"] = "X11"; client_json["osVersion"] = ""; client_json["originalUrl"] = url + "/search?query=" + url_param_encode(str); context_json["client"] = std::move(client_json); request_json["context"] = std::move(context_json); request_json["browseId"] = channel_id; request_json["query"] = str; request_json["params"] = "EgZzZWFyY2jyBgQKAloA"; Json::StreamWriterBuilder json_builder; json_builder["commentStyle"] = "None"; json_builder["indentation"] = ""; std::vector additional_args = { { "-H", "authority: www.youtube.com" }, { "-H", "x-origin: https://www.youtube.com" }, { "-H", "content-type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, { "-H", "referer: " + url + "/search?query=" + url_param_encode(str) }, { "--data-raw", Json::writeString(json_builder, request_json) } }; additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_search_result(result); if(!json_root.isObject()) return SearchResult::ERR; const Json::Value &contents_json = json_root["contents"]; if(!contents_json.isObject()) return SearchResult::ERR; const Json::Value &two_column_browse_results_renderer_json = contents_json["twoColumnBrowseResultsRenderer"]; if(!two_column_browse_results_renderer_json.isObject()) return SearchResult::ERR; const Json::Value &tabs_json = two_column_browse_results_renderer_json["tabs"]; if(!tabs_json.isArray()) return SearchResult::ERR; for(const Json::Value &json_item : tabs_json) { if(!json_item.isObject()) continue; const Json::Value &expandable_tab_renderer_json = json_item["expandableTabRenderer"]; if(!expandable_tab_renderer_json.isObject()) continue; const Json::Value &content_json = expandable_tab_renderer_json["content"]; if(!content_json.isObject()) continue; parse_section_list_renderer(content_json["sectionListRenderer"], continuation_token, result_items, added_videos); } return SearchResult::OK; } PluginResult YoutubeChannelPage::get_page(const std::string&, int page, BodyItems &result_items) { while(current_page < page) { PluginResult plugin_result = search_get_continuation(url, continuation_token, result_items); if(plugin_result != PluginResult::OK) return plugin_result; ++current_page; } return PluginResult::OK; } static void channel_page_continuation_get_content(const Json::Value &continuation_items_json, std::string &new_continuation_token, std::unordered_set &added_videos, BodyItems &result_items) { if(!continuation_items_json.isArray()) return; for(const Json::Value &item_json : continuation_items_json) { if(!item_json.isObject()) continue; if(new_continuation_token.empty()) new_continuation_token = item_section_renderer_get_continuation_token(item_json); const Json::Value &grid_video_renderer_json = item_json["gridVideoRenderer"]; const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"]; const Json::Value &rich_item_renderer_json = item_json["richItemRenderer"]; if(grid_video_renderer_json.isObject()) { auto body_item = parse_common_video_item(grid_video_renderer_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } else if(item_section_renderer_json.isObject()) { parse_item_section_renderer(item_section_renderer_json, added_videos, result_items); } else if(rich_item_renderer_json.isObject()) { const Json::Value &content_json = rich_item_renderer_json["content"]; if(!content_json.isObject()) continue; const Json::Value &video_renderer_json = content_json["videoRenderer"]; if(!video_renderer_json.isObject()) continue; auto body_item = parse_common_video_item(video_renderer_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } } PluginResult YoutubeChannelPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { if(current_continuation_token.empty()) return PluginResult::OK; std::vector cookies = get_cookies(); std::string next_url = "https://www.youtube.com/youtubei/v1/browse?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false"; Json::Value request_json(Json::objectValue); Json::Value context_json(Json::objectValue); Json::Value client_json(Json::objectValue); client_json["hl"] = "en"; client_json["gl"] = "US"; client_json["deviceMake"] = ""; client_json["deviceModel"] = ""; client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; client_json["clientName"] = "WEB"; client_json["clientVersion"] = "2.20210622.10.00"; client_json["osName"] = "X11"; client_json["osVersion"] = ""; client_json["originalUrl"] = url + "/videos"; context_json["client"] = std::move(client_json); request_json["context"] = std::move(context_json); request_json["continuation"] = current_continuation_token; Json::StreamWriterBuilder json_builder; json_builder["commentStyle"] = "None"; json_builder["indentation"] = ""; std::vector additional_args = { { "-H", "authority: www.youtube.com" }, { "-H", "x-origin: https://www.youtube.com" }, { "-H", "content-type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, { "-H", "referer: " + url + "/videos" }, { "--data-raw", Json::writeString(json_builder, request_json) } }; additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(!json_root.isObject()) return PluginResult::ERR; std::string new_continuation_token; result_items.clear(); const Json::Value &on_response_received_actions_json = json_root["onResponseReceivedActions"]; if(on_response_received_actions_json.isArray()) { for(const Json::Value &json_item : on_response_received_actions_json) { if(!json_item.isObject()) continue; const Json::Value &append_continuation_items_action_json = json_item["appendContinuationItemsAction"]; if(!append_continuation_items_action_json.isObject()) continue; channel_page_continuation_get_content(append_continuation_items_action_json["continuationItems"], new_continuation_token, added_videos, result_items); } } else { // What a cluster-fuck. Sometimes youtube returns the same front page twice, so we try to parse it // and if we dont get any new items then request the page again (I hope we got a new continuation key???). const Json::Value &continuation_contents_json = json_root["continuationContents"]; if(!continuation_contents_json.isObject()) return PluginResult::ERR; const Json::Value &rich_grid_continuation_json = continuation_contents_json["richGridContinuation"]; if(!rich_grid_continuation_json.isObject()) return PluginResult::ERR; channel_page_continuation_get_content(rich_grid_continuation_json["contents"], new_continuation_token, added_videos, result_items); if(result_items.empty()) { // Hopefully we dont have an infinite loop... continuation_token = std::move(new_continuation_token); return search_get_continuation(url, current_continuation_token, result_items); } } continuation_token = std::move(new_continuation_token); return PluginResult::OK; } PluginResult YoutubeChannelPage::submit(const SubmitArgs &args, std::vector &result_tabs) { if(args.url.empty()) return PluginResult::OK; if(strstr(args.url.c_str(), "&list=")) { result_tabs.push_back(Tab{create_body(false, true), std::make_unique(program, args.url, args.title), create_search_bar("Filter...", SEARCH_DELAY_FILTER)}); } else { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url), nullptr}); } return PluginResult::OK; } PluginResult YoutubeChannelPage::lazy_fetch(BodyItems &result_items) { added_videos.clear(); continuation_token.clear(); std::vector additional_args = { { "-H", "x-spf-referer: " + url }, { "-H", "x-youtube-client-name: 1" }, { "-H", "x-spf-previous: " + url }, { "-H", youtube_client_version }, { "-H", "referer: " + url } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, url + "/" + youtube_channel_page_type_get_endpoint(type) + "?pbj=1&gl=US&hl=en", std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); std::string browse_id; if(json_root.isObject()) { //search_page_submit_suggestion_handler(json_root, continuation_token, result_items, added_videos); parse_channel_videos(json_root, continuation_token, added_videos, browse_id, result_items); } else if(json_root.isArray()) { for(const Json::Value &json_item : json_root) { //search_page_submit_suggestion_handler(json_root, continuation_token, result_items, added_videos); parse_channel_videos(json_item, continuation_token, added_videos, browse_id, result_items); } } else { return PluginResult::ERR; } if(!browse_id.empty()) url = "https://www.youtube.com/channel/" + std::move(browse_id); return PluginResult::OK; } TrackResult YoutubeChannelPage::track(const std::string&) { std::string channel_id; std::string channel_url; if(!youtube_url_extract_channel_id(url, channel_id, channel_url)) { show_notification("QuickMedia", "Unable to get channel id from " + url, Urgency::CRITICAL); return TrackResult::ERR; } Path subscriptions_path = get_storage_dir().join("subscriptions"); if(create_directory_recursive(subscriptions_path) != 0) { show_notification("QuickMedia", "Failed to create directory: " + subscriptions_path.data, Urgency::CRITICAL); return TrackResult::ERR; } subscriptions_path.join("youtube.txt"); std::unordered_set channel_ids; std::string subscriptions_str; FileType file_type = get_file_type(subscriptions_path); if(file_type == FileType::REGULAR) { if(file_get_content(subscriptions_path, subscriptions_str) == 0) { string_split(subscriptions_str, '\n', [&channel_ids](const char *str, size_t size) { std::string line(str, size); line = strip(line); if(!line.empty()) channel_ids.insert(std::move(line)); return true; }); } else { show_notification("QuickMedia", "Failed to read " + subscriptions_path.data, Urgency::CRITICAL); abort(); } } auto it = channel_ids.find(channel_id); if(it == channel_ids.end()) { channel_ids.insert(channel_id); show_notification("QuickMedia", "Subscribed", Urgency::LOW); } else { channel_ids.erase(it); show_notification("QuickMedia", "Unsubscribed", Urgency::LOW); } std::string channel_ids_str; for(auto &it : channel_ids) { if(!channel_ids_str.empty()) channel_ids_str += '\n'; channel_ids_str += std::move(it); } if(file_overwrite_atomic(subscriptions_path, channel_ids_str) != 0) { show_notification("QuickMedia", "Failed to update subscriptions list with " + channel_id, Urgency::CRITICAL); return TrackResult::ERR; } return TrackResult::OK; } PluginResult YoutubeSubscriptionsPage::submit(const SubmitArgs &args, std::vector &result_tabs) { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url), nullptr}); return PluginResult::OK; } struct SubscriptionEntry { std::string title; std::string video_id; time_t published = 0; }; struct SubscriptionData { std::vector subscription_entry; std::string author; bool inside_title = false; bool inside_entry = false; }; static int string_view_equals(HtmlStringView *self, const char *sub) { const size_t sub_len = strlen(sub); return self->size == sub_len && memcmp(self->data, sub, sub_len) == 0; } PluginResult YoutubeSubscriptionsPage::lazy_fetch(BodyItems &result_items) { Path subscriptions_path = get_storage_dir().join("subscriptions").join("youtube.txt"); std::string subscriptions_str; if(file_get_content(subscriptions_path, subscriptions_str) != 0) return PluginResult::OK; // TODO: Make a async task pool to handle this more efficiently std::vector channel_ids; string_split(subscriptions_str, '\n', [&channel_ids](const char *str, size_t size) { std::string line(str, size); line = strip(line); if(!line.empty()) channel_ids.push_back(std::move(line)); return true; }); std::vector task_results; size_t async_task_index = 0; const time_t time_now = time(nullptr); for(const std::string &channel_id : channel_ids) { if(program_is_dead_in_current_thread()) return PluginResult::OK; subscription_load_tasks[async_task_index] = AsyncTask>([&channel_id, time_now]() -> std::vector { std::string website_data; DownloadResult result = download_to_string("https://www.youtube.com/feeds/videos.xml?channel_id=" + url_param_encode(channel_id), website_data, {}, false); if(result != DownloadResult::OK) { auto body_item = BodyItem::create("Failed to fetch videos for channel: " + channel_id); return {YoutubeSubscriptionTaskResult{body_item, time_now}}; } SubscriptionData subscription_data; html_parser_parse(website_data.data(), website_data.size(), [](HtmlParser *html_parser, HtmlParseType parse_type, void *userdata) { SubscriptionData &subscription_data = *(SubscriptionData*)userdata; if(!subscription_data.inside_entry && subscription_data.author.empty()) { if(parse_type == HTML_PARSE_TAG_START && string_view_equals(&html_parser->tag_name, "title")) { subscription_data.inside_title = true; return 0; } else if(parse_type == HTML_PARSE_TAG_END && string_view_equals(&html_parser->tag_name, "title")) { subscription_data.inside_title = false; subscription_data.author.assign(html_parser->text_stripped.data, html_parser->text_stripped.size); return 0; } } if(parse_type == HTML_PARSE_TAG_START && string_view_equals(&html_parser->tag_name, "entry")) { subscription_data.subscription_entry.push_back({}); subscription_data.inside_entry = true; return 0; } else if(parse_type == HTML_PARSE_TAG_END && string_view_equals(&html_parser->tag_name, "entry")) { subscription_data.inside_entry = false; return 0; } if(!subscription_data.inside_entry) return 0; if(string_view_equals(&html_parser->tag_name, "title") && parse_type == HTML_PARSE_TAG_END) { subscription_data.subscription_entry.back().title.assign(html_parser->text_stripped.data, html_parser->text_stripped.size); } else if(string_view_equals(&html_parser->tag_name, "yt:videoId") && parse_type == HTML_PARSE_TAG_END) { subscription_data.subscription_entry.back().video_id.assign(html_parser->text_stripped.data, html_parser->text_stripped.size); } else if(string_view_equals(&html_parser->tag_name, "published") && parse_type == HTML_PARSE_TAG_END) { std::string published_str(html_parser->text_stripped.data, html_parser->text_stripped.size); subscription_data.subscription_entry.back().published = iso_utc_to_unix_time(published_str.c_str()); } return 0; }, &subscription_data); std::vector results; for(SubscriptionEntry &subscription_entry : subscription_data.subscription_entry) { if(subscription_entry.title.empty() || subscription_entry.video_id.empty() || subscription_entry.published == 0) continue; html_unescape_sequences(subscription_entry.title); auto body_item = BodyItem::create(std::move(subscription_entry.title)); std::string description = "Uploaded " + seconds_to_relative_time_str(time_now - subscription_entry.published); if(!subscription_data.author.empty()) { description += '\n'; description += subscription_data.author; } body_item->set_description(std::move(description)); body_item->set_description_color(get_theme().faded_text_color); body_item->url = "https://www.youtube.com/watch?v=" + subscription_entry.video_id; body_item->thumbnail_url = "https://img.youtube.com/vi/" + subscription_entry.video_id + "/mqdefault.jpg"; body_item->thumbnail_size = mgl::vec2i(192, 108); results.push_back({std::move(body_item), subscription_entry.published}); } return results; }); ++async_task_index; if(async_task_index == subscription_load_tasks.size()) { async_task_index = 0; for(auto &load_task : subscription_load_tasks) { if(!load_task.valid()) continue; auto new_task_results = load_task.get(); task_results.insert(task_results.end(), std::move_iterator(new_task_results.begin()), std::move_iterator(new_task_results.end())); } } } for(size_t i = 0; i < async_task_index; ++i) { auto &load_task = subscription_load_tasks[i]; if(!load_task.valid()) continue; auto new_task_results = load_task.get(); task_results.insert(task_results.end(), std::move_iterator(new_task_results.begin()), std::move_iterator(new_task_results.end())); } std::sort(task_results.begin(), task_results.end(), [](const YoutubeSubscriptionTaskResult &sub_data1, const YoutubeSubscriptionTaskResult &sub_data2) { return sub_data1.timestamp > sub_data2.timestamp; }); result_items.reserve(task_results.size()); for(auto &task_result : task_results) { result_items.push_back(std::move(task_result.body_item)); } return PluginResult::OK; } PluginResult YoutubeRelatedVideosPage::submit(const SubmitArgs &args, std::vector &result_tabs) { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url), nullptr}); return PluginResult::OK; } YoutubePlaylistPage::YoutubePlaylistPage(Program *program, const std::string &url, std::string title) : LazyFetchPage(program), title(std::move(title)) { if(!youtube_url_extract_playlist_id(url, playlist_id)) fprintf(stderr, "Error: failed to extract playlist id from url: %s\n", url.c_str()); } PluginResult YoutubePlaylistPage::get_page(const std::string&, int, BodyItems &result_items) { if(reached_end) return PluginResult::OK; std::vector cookies = get_cookies(); std::string next_url = "https://www.youtube.com/youtubei/v1/browse?key=" + url_param_encode(api_key) + "&gl=US&hl=en&prettyPrint=false"; Json::Value client_json(Json::objectValue); client_json["hl"] = "en"; client_json["gl"] = "US"; client_json["deviceMake"] = ""; client_json["deviceModel"] = ""; client_json["userAgent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; client_json["clientName"] = "WEB"; client_json["clientVersion"] = "2.20210622.10.00"; client_json["osName"] = "X11"; client_json["osVersion"] = ""; client_json["originalUrl"] = "https://www.youtube.com/playlist?list=" + playlist_id; Json::Value context_json(Json::objectValue); context_json["client"] = std::move(client_json); Json::Value request_json(Json::objectValue); request_json["context"] = std::move(context_json); if(continuation_token.empty()) request_json["browseId"] = "VL" + playlist_id; else request_json["continuation"] = continuation_token; Json::StreamWriterBuilder json_builder; json_builder["commentStyle"] = "None"; json_builder["indentation"] = ""; std::vector additional_args = { { "-H", "content-type: application/json" }, { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, { "--data-raw", Json::writeString(json_builder, request_json) } }; additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true); if(result != DownloadResult::OK) return download_result_to_plugin_result(result); if(!json_root.isObject()) return PluginResult::ERR; std::unordered_set added_videos; std::string browse_id; continuation_token.clear(); const Json::Value &on_response_received_actions_json = json_root["onResponseReceivedActions"]; if(on_response_received_actions_json.isArray()) { for(const Json::Value &json_item : on_response_received_actions_json) { if(!json_item.isObject()) continue; const Json::Value &append_continuation_items_action_json = json_item["appendContinuationItemsAction"]; if(!append_continuation_items_action_json.isObject()) continue; parse_playlist_video_list(append_continuation_items_action_json, "continuationItems", continuation_token, added_videos, result_items); } } else { parse_channel_videos(json_root, continuation_token, added_videos, browse_id, result_items); } if(continuation_token.empty()) reached_end = true; return PluginResult::OK; } PluginResult YoutubePlaylistPage::submit(const SubmitArgs &args, std::vector &result_tabs) { result_tabs.push_back(Tab{nullptr, std::make_unique(program, args.url, true, true), nullptr}); return PluginResult::OK; } PluginResult YoutubePlaylistPage::lazy_fetch(BodyItems &result_items) { return get_page("", 0, result_items); } static std::string two_column_watch_next_results_get_comments_continuation_token(const Json::Value &tcwnr_json) { const Json::Value &results_json = tcwnr_json["results"]; if(!results_json.isObject()) return ""; const Json::Value &results2_json = results_json["results"]; if(!results2_json.isObject()) return ""; const Json::Value &contents_json = results2_json["contents"]; if(!contents_json.isArray()) return ""; std::string comments_continuation_token; for(const Json::Value &content_item_json : contents_json) { if(!content_item_json.isObject()) continue; comments_continuation_token = item_section_renderer_get_continuation(content_item_json["itemSectionRenderer"]); if(!comments_continuation_token.empty()) return comments_continuation_token; } return ""; } static int youtube_url_timestamp_to_seconds(const std::string ×tamp) { int hours = 0; int minutes = 0; int seconds = 0; if(sscanf(timestamp.c_str(), "%dh%dm%ds", &hours, &minutes, &seconds) == 3) return (hours * 60 * 60) + (minutes * 60) + seconds; if(sscanf(timestamp.c_str(), "%dm%ds", &minutes, &seconds) == 2) return (minutes * 60) + seconds; if(sscanf(timestamp.c_str(), "%d", &seconds) == 1) return seconds; return 0; } static void youtube_url_remove_timestamp(std::string &url, std::string ×tamp) { size_t timestamp_start = url.find("&t="); if(timestamp_start == std::string::npos) { timestamp_start = url.find("?t="); if(timestamp_start == std::string::npos) return; } size_t timestamp_end = url.find("&", timestamp_start + 3); if(timestamp_end == std::string::npos) timestamp_end = url.size(); int timestamp_seconds = youtube_url_timestamp_to_seconds(url.substr(timestamp_start + 3, timestamp_end - (timestamp_start + 3))); timestamp = std::to_string(timestamp_seconds); url.erase(timestamp_start, timestamp_end - timestamp_start); return; } YoutubeVideoPage::YoutubeVideoPage(Program *program, std::string url, bool autoplay, bool autoplay_next_item) : VideoPage(program, "", autoplay), goto_next_item(autoplay_next_item) { set_url(std::move(url)); } void YoutubeVideoPage::set_url(std::string new_url) { timestamp.clear(); new_url = invidious_url_to_youtube_url(new_url); youtube_url_remove_timestamp(new_url, timestamp); VideoPage::set_url(std::move(new_url)); } std::string YoutubeVideoPage::get_url_timestamp() { if(!timestamp.empty()) return timestamp; if(!get_config().youtube.load_progress) return ""; std::string video_id; if(!youtube_url_extract_id(url, video_id)) { fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); return ""; } // TODO: Remove very old videos, to not make this file too large which slows this down on slow harddrives std::unordered_map watch_progress = get_watch_progress_for_plugin("youtube"); auto it = watch_progress.find(video_id); if(it == watch_progress.end()) return ""; // If we are very close to the end then start from the beginning. // This is the same behavior as mpv. // This is better because we dont want the video player to stop immediately after we start playing and we dont get any chance to seek. if(it->second.time_pos_sec + 10.0 >= it->second.duration_sec) return ""; else return std::to_string(it->second.time_pos_sec); } static void video_page_related_media_handler(const Json::Value &json_root, std::string &comments_continuation_token, BodyItems &result_items) { if(!json_root.isObject()) return; const Json::Value &response_json = json_root["response"]; if(!response_json.isObject()) return; const Json::Value &contents_json = response_json["contents"]; if(!contents_json.isObject()) return; const Json::Value &tcwnr_json = contents_json["twoColumnWatchNextResults"]; if(!tcwnr_json.isObject()) return; if(comments_continuation_token.empty()) comments_continuation_token = two_column_watch_next_results_get_comments_continuation_token(tcwnr_json); const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"]; if(!secondary_results_json.isObject()) return; const Json::Value &secondary_results2_json = secondary_results_json["secondaryResults"]; if(!secondary_results2_json.isObject()) return; const Json::Value &results_json = secondary_results2_json["results"]; if(!results_json.isArray()) return; std::unordered_set added_videos; for(const Json::Value &item_json : results_json) { if(!item_json.isObject()) continue; auto body_item = parse_compact_video_renderer_json(item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); const Json::Value &compact_autoplay_renderer_json = item_json["compactAutoplayRenderer"]; if(!compact_autoplay_renderer_json.isObject()) continue; const Json::Value &item_contents_json = compact_autoplay_renderer_json["contents"]; if(!item_contents_json.isArray()) continue; for(const Json::Value &content_item_json : item_contents_json) { if(!content_item_json.isObject()) continue; auto body_item = parse_compact_video_renderer_json(content_item_json, added_videos); if(body_item) result_items.push_back(std::move(body_item)); } } } BodyItems YoutubeVideoPage::get_related_media(const std::string &url) { comments_continuation_token.clear(); BodyItems result_items; std::string video_id; if(!youtube_url_extract_id(url, video_id)) { fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); return result_items; } std::vector additional_args = { { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version }, }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); // TODO: Remove this code completely and replace with existing player? api Json::Value json_root; DownloadResult download_result = download_json(json_root, "https://www.youtube.com/watch?v=" + video_id + "&pbj=1&gl=US&hl=en", additional_args, true); if(download_result != DownloadResult::OK) return result_items; if(json_root.isObject()) { video_page_related_media_handler(json_root, comments_continuation_token, result_items); return result_items; } if(!json_root.isArray()) return result_items; for(const Json::Value &json_item : json_root) { video_page_related_media_handler(json_item, comments_continuation_token, result_items); } return result_items; } static std::shared_ptr video_details_to_body_item(const YoutubeVideoDetails &video_details) { auto body_item = BodyItem::create(video_details.title); std::string description; if(!video_details.views.empty()) { description = number_separate_thousand_commas(video_details.views) + " view" + (video_details.views == "1" ? "" : "s"); } if(!video_details.author.empty()) { if(!description.empty()) description += '\n'; description += video_details.author; } if(!video_details.description.empty()) { if(!description.empty()) description += "\n\n"; description += video_details.description; } if(!description.empty()) body_item->set_description(std::move(description)); return body_item; } PluginResult YoutubeVideoPage::get_related_pages(const BodyItems &related_videos, const std::string &channel_url, std::vector &result_tabs) { auto description_page_body = create_body(); description_page_body->append_item(video_details_to_body_item(video_details)); auto related_page_body = create_body(false, true); related_page_body->set_items(related_videos); result_tabs.push_back(Tab{std::move(description_page_body), std::make_unique(program), nullptr}); result_tabs.push_back(Tab{create_body(), std::make_unique(program, url, comments_continuation_token), nullptr}); result_tabs.push_back(Tab{std::move(related_page_body), std::make_unique(program), create_search_bar("Search...", SEARCH_DELAY_FILTER)}); YoutubeChannelPage::create_each_type(program, channel_url, "", "Channel", result_tabs); return PluginResult::OK; } static std::map http_params_parse(const std::string &http_params) { std::map result; string_split(http_params, '&', [&result](const char *str, size_t size) { const void *split_p = memchr(str, '=', size); if(split_p == nullptr) return true; std::string key(str, (const char*)split_p - str); std::string value((const char*)split_p + 1, (str + size) - ((const char*)split_p + 1)); key = url_param_decode(key); value = url_param_decode(value); result[std::move(key)] = std::move(value); return true; }); return result; } static const YoutubeVideoFormat* get_highest_resolution_mp4_non_av1(const std::vector &video_formats, int max_height) { for(const YoutubeVideoFormat &video_format : video_formats) { if(video_format.height <= max_height && video_format.base.mime_type.find("mp4") != std::string::npos && video_format.base.mime_type.find("av01") == std::string::npos) return &video_format; } return nullptr; } static const YoutubeVideoFormat* get_highest_resolution_non_mp4(const std::vector &video_formats, int max_height) { for(const YoutubeVideoFormat &video_format : video_formats) { if(video_format.height <= max_height && video_format.base.mime_type.find("mp4") == std::string::npos) return &video_format; } return nullptr; } std::string YoutubeVideoPage::get_download_url(int max_height) { return url; } std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio, std::string &ext) { if(use_youtube_dl_fallback) { has_embedded_audio = youtube_dl_audio_fallback_url.empty(); ext = ".webm"; // TODO: Guess for now return youtube_dl_video_fallback_url; } if(!livestream_url.empty() && video_formats.empty() && audio_formats.empty()) { has_embedded_audio = true; return livestream_url; } if(video_formats.empty()) { has_embedded_audio = false; return ""; } const YoutubeVideoFormat *chosen_video_format = nullptr; const YoutubeVideoFormat *best_mp4 = get_highest_resolution_mp4_non_av1(video_formats, max_height); const YoutubeVideoFormat *best_non_mp4 = get_highest_resolution_non_mp4(video_formats, max_height); // We prefer mp4 (h264) because it has the best hardware decoding support if(best_mp4 && (!best_non_mp4 || (best_mp4->height >= best_non_mp4->height && best_mp4->fps >= best_non_mp4->fps))) { chosen_video_format = best_mp4; } else if(best_non_mp4) { chosen_video_format = best_non_mp4; } if(!chosen_video_format) chosen_video_format = &video_formats.back(); fprintf(stderr, "Choosing youtube video format: width: %d, height: %d, fps: %d, bitrate: %d, mime type: %s\n", chosen_video_format->width, chosen_video_format->height, chosen_video_format->fps, chosen_video_format->base.bitrate, chosen_video_format->base.mime_type.c_str()); has_embedded_audio = chosen_video_format->has_embedded_audio; if(chosen_video_format->base.mime_type.find("mp4") != std::string::npos) ext = ".mp4"; else if(chosen_video_format->base.mime_type.find("webm") != std::string::npos) ext = ".webm"; return chosen_video_format->base.url; } std::string YoutubeVideoPage::get_audio_url(std::string &ext) { if(use_youtube_dl_fallback) { ext = ".opus"; // TODO: Guess for now return youtube_dl_audio_fallback_url; } if(audio_formats.empty()) return ""; const YoutubeAudioFormat *chosen_audio_format = &audio_formats.front(); fprintf(stderr, "Choosing youtube audio format: bitrate: %d, mime type: %s\n", chosen_audio_format->base.bitrate, chosen_audio_format->base.mime_type.c_str()); if(chosen_audio_format->base.mime_type.find("mp4") != std::string::npos) ext = ".m4a"; else if(chosen_audio_format->base.mime_type.find("webm") != std::string::npos) ext = ".opus"; // TODO: Detect if vorbis (.ogg) or opus (.opus) else if(chosen_audio_format->base.mime_type.find("opus") != std::string::npos) ext = ".opus"; return chosen_audio_format->base.url; } // Returns -1 if timestamp is in an invalid format static int youtube_comment_timestamp_to_seconds(const char *str, size_t size) { if(size > 30) return -1; char timestamp[32]; memcpy(timestamp, str, size); timestamp[size] = '\0'; int hours = 0; int minutes = 0; int seconds = 0; if(sscanf(timestamp, "%d:%d:%d", &hours, &minutes, &seconds) == 3) return (hours * 60 * 60) + (minutes * 60) + seconds; if(sscanf(timestamp, "%d:%d", &minutes, &seconds) == 2) return (minutes * 60) + seconds; return -1; } static int get_start_of_comment_timestamp(const char *str, size_t size) { for(int i = (int)size - 1; i >= 0; --i) { char c = str[i]; if(c == ' ' || c == '\t') return i + 1; } return -1; } static int start_of_timestamp_title(const char *str, size_t size) { for(int i = 0; i < (int)size; ++i) { char c = str[i]; if(c != '-' && c != ':' && c != '.' && c != ' ' && c != '\t') return i; } return -1; } static int end_of_timestamp_title(const char *str, size_t size) { for(int i = (int)size - 1; i >= 0; --i) { char c = str[i]; if(c != '-' && c != ':' && c != '.' && c != ' ' && c != '\t') return i + 1; } return -1; } static std::vector youtube_description_extract_chapters(const std::string &description) { std::vector result; string_split(description, '\n', [&result](const char *str, size_t size) { strip(str, size, &size); if(size == 0) return true; const char *first_space_p = (const char*)memchr(str, ' ', size); if(!first_space_p) return true; int timestamp_seconds = youtube_comment_timestamp_to_seconds(str, first_space_p - str); if(timestamp_seconds != -1) { // Timestamp at the start of the line size -= (first_space_p - str); str = first_space_p; const int timestamp_title_start = start_of_timestamp_title(str, size); if(timestamp_title_start == -1) return true; str += timestamp_title_start; size -= timestamp_title_start; } else { // Timestamp at the end of the line const int timestamp_start = get_start_of_comment_timestamp(str, size); if(timestamp_start == -1) return true; timestamp_seconds = youtube_comment_timestamp_to_seconds(str + timestamp_start, size - timestamp_start); if(timestamp_seconds == -1) return true; const int timestamp_title_end = end_of_timestamp_title(str, timestamp_start); if(timestamp_title_end == -1) return true; size = timestamp_title_end; } MediaChapter chapter; chapter.start_seconds = timestamp_seconds; chapter.title.assign(str, size); chapter.title = strip(chapter.title); result.push_back(std::move(chapter)); return true; }); return result; } static void subtitle_url_set_vtt_format(std::string &subtitle_url) { const size_t index = subtitle_url.find("&fmt="); if(index == std::string::npos) { subtitle_url += "&fmt=vtt"; return; } size_t end_index = subtitle_url.find('&'); if(end_index == std::string::npos) end_index = subtitle_url.size(); subtitle_url.replace(index, end_index - index, "&fmt=vtt"); } static void parse_caption_tracks(const Json::Value &caption_tracks, std::map &subtitle_urls_by_lang_code) { if(!caption_tracks.isArray()) return; for(const Json::Value &caption_track : caption_tracks) { if(!caption_track.isObject()) continue; const Json::Value &base_url_json = caption_track["baseUrl"]; const Json::Value &language_code_json = caption_track["languageCode"]; const Json::Value &kind_json = caption_track["kind"]; // kind = asr = auto generated subtitles. We dont want those! if(!base_url_json.isString() || !language_code_json.isString() || (kind_json.isString() && strcmp(kind_json.asCString(), "asr") == 0)) continue; std::string base_url = base_url_json.asString(); subtitle_url_set_vtt_format(base_url); std::string language_code = language_code_json.asString(); std::optional title = yt_json_get_text(caption_track, "name"); SubtitleData subtitle_data; subtitle_data.url = std::move(base_url); if(title) subtitle_data.title = std::move(title.value()); else subtitle_data.title = language_code; subtitle_urls_by_lang_code[std::move(language_code)] = std::move(subtitle_data); } } static void video_details_clear(YoutubeVideoDetails &video_details) { video_details.title.clear(); video_details.author.clear(); video_details.views.clear(); video_details.description.clear(); video_details.duration = 0.0; } static void sponsorblock_add_chapters(Page *page, const std::string &url, int min_votes, std::vector &chapters) { std::string video_id; if(!youtube_url_extract_id(url, video_id)) { fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); return; } const std::string sponsorblock_url = "https://sponsor.ajay.app/api/skipSegments?videoID=" + video_id; Json::Value json_root; if(page->download_json(json_root, sponsorblock_url, {}) != DownloadResult::OK) return; if(!json_root.isArray()) return; for(const Json::Value &item_json : json_root) { if(!item_json.isObject()) continue; const Json::Value &category_json = item_json["category"]; const Json::Value &action_type_json = item_json["actionType"]; const Json::Value &segment_json = item_json["segment"]; const Json::Value &votes_json = item_json["votes"]; if(!category_json.isString() || !action_type_json.isString() || !segment_json.isArray() || !votes_json.isInt()) continue; if(strcmp(category_json.asCString(), "sponsor") != 0 || strcmp(action_type_json.asCString(), "skip") != 0) continue; if(segment_json.size() != 2) continue; if(!segment_json[0].isDouble() || !segment_json[1].isDouble()) continue; if(votes_json.asInt() < min_votes) continue; MediaChapter ad_start; ad_start.start_seconds = segment_json[0].asDouble(); ad_start.title = "Ad start"; chapters.push_back(std::move(ad_start)); MediaChapter ad_end; ad_end.start_seconds = segment_json[1].asDouble(); ad_end.title = "Ad end"; chapters.push_back(std::move(ad_end)); } } PluginResult YoutubeVideoPage::parse_video_response(const Json::Value &json_root, std::string &title, std::string &channel_url, std::vector &chapters, std::string &err_str) { livestream_url.clear(); video_formats.clear(); audio_formats.clear(); subtitle_urls_by_lang_code.clear(); title.clear(); channel_url.clear(); chapters.clear(); youtube_dl_video_fallback_url.clear(); youtube_dl_audio_fallback_url.clear(); video_details_clear(video_details); if(!json_root.isObject()) return PluginResult::ERR; const Json::Value &video_details_json = json_root["videoDetails"]; if(video_details_json.isObject()) { const Json::Value &channel_id_json = video_details_json["channelId"]; if(channel_id_json.isString()) channel_url = "https://www.youtube.com/channel/" + channel_id_json.asString(); const Json::Value &title_json = video_details_json["title"]; const Json::Value &author_json = video_details_json["author"]; const Json::Value &view_count_json = video_details_json["viewCount"]; const Json::Value &short_description_json = video_details_json["shortDescription"]; const Json::Value &length_seconds_json = video_details_json["lengthSeconds"]; if(title_json.isString()) video_details.title = title_json.asString(); if(author_json.isString()) video_details.author = author_json.asString(); if(view_count_json.isString()) video_details.views = view_count_json.asString(); if(short_description_json.isString()) video_details.description = short_description_json.asString(); if(length_seconds_json.isString()) { const char *length_seconds_str = length_seconds_json.asCString(); int duration = 0; to_num(length_seconds_str, strlen(length_seconds_str), duration); video_details.duration = duration; } title = video_details.title; if(!video_details.description.empty()) chapters = youtube_description_extract_chapters(video_details.description); } const Json::Value &captions_json = json_root["captions"]; if(captions_json.isObject()) { const Json::Value &player_captions_tracklist_renderer_json = captions_json["playerCaptionsTracklistRenderer"]; if(player_captions_tracklist_renderer_json.isObject()) parse_caption_tracks(player_captions_tracklist_renderer_json["captionTracks"], subtitle_urls_by_lang_code); } const Json::Value &playback_tracing_json = json_root["playbackTracking"]; if(playback_tracing_json.isObject()) { if(playback_url.empty()) { const Json::Value &video_stats_playback_url_json = playback_tracing_json["videostatsPlaybackUrl"]; if(video_stats_playback_url_json.isObject()) { const Json::Value &base_url_json = video_stats_playback_url_json["baseUrl"]; if(base_url_json.isString()) playback_url = base_url_json.asString(); } } if(watchtime_url.empty()) { const Json::Value &video_stats_watchtime_url_json = playback_tracing_json["videostatsWatchtimeUrl"]; if(video_stats_watchtime_url_json.isObject()) { const Json::Value &base_url_json = video_stats_watchtime_url_json["baseUrl"]; if(base_url_json.isString()) watchtime_url = base_url_json.asString(); } } if(tracking_url.empty()) { const Json::Value &p_tracking_url_json = playback_tracing_json["ptrackingUrl"]; if(p_tracking_url_json.isObject()) { const Json::Value &base_url_json = p_tracking_url_json["baseUrl"]; if(base_url_json.isString()) tracking_url = base_url_json.asString(); } } } const Json::Value &playability_status_json = json_root["playabilityStatus"]; if(playability_status_json.isObject()) { const Json::Value &status_json = playability_status_json["status"]; if(status_json.isString() && (strcmp(status_json.asCString(), "UNPLAYABLE") == 0 || strcmp(status_json.asCString(), "LOGIN_REQUIRED") == 0)) { fprintf(stderr, "Failed to load youtube video, trying with yt-dlp instead\n"); if(program->youtube_dl_extract_url(url, youtube_dl_video_fallback_url, youtube_dl_audio_fallback_url)) { if(get_config().youtube.sponsorblock.enable) sponsorblock_add_chapters(this, url, get_config().youtube.sponsorblock.min_votes, chapters); use_youtube_dl_fallback = true; return PluginResult::OK; } else { const Json::Value &reason_json = playability_status_json["reason"]; if(reason_json.isString()) err_str = reason_json.asString(); fprintf(stderr, "Unable to play video, status: %s, reason: %s\n", status_json.asCString(), reason_json.isString() ? reason_json.asCString() : "Unknown"); return PluginResult::ERR; } } } const Json::Value *streaming_data_json = &json_root["streamingData"]; if(!streaming_data_json->isObject()) return PluginResult::ERR; // TODO: Verify if this always works (what about copyrighted live streams?), also what about choosing video quality for live stream? Maybe use mpv --hls-bitrate option? const Json::Value &hls_manifest_url_json = (*streaming_data_json)["hlsManifestUrl"]; if(hls_manifest_url_json.isString()) livestream_url = hls_manifest_url_json.asString(); /* const Json::Value &dash_manifest_url_json = (*streaming_data_json)["dashManifestUrl"]; if(livestream_url.empty() && dash_manifest_url_json.isString()) { // TODO: mpv cant properly play dash videos. Video goes back and replays. // So for now return here (get_video_info only hash dash stream and no hls stream) which will fallback to the player youtube endpoint which has hls stream. return PluginResult::ERR; } */ parse_formats(*streaming_data_json); if(video_formats.empty() && audio_formats.empty() && livestream_url.empty()) return PluginResult::ERR; std::sort(video_formats.begin(), video_formats.end(), [](const YoutubeVideoFormat &format1, const YoutubeVideoFormat &format2) { return format1.base.bitrate > format2.base.bitrate; }); std::sort(audio_formats.begin(), audio_formats.end(), [](const YoutubeAudioFormat &format1, const YoutubeAudioFormat &format2) { return format1.base.bitrate > format2.base.bitrate; }); if(get_config().youtube.sponsorblock.enable) sponsorblock_add_chapters(this, url, get_config().youtube.sponsorblock.min_votes, chapters); return PluginResult::OK; } PluginResult YoutubeVideoPage::load(const SubmitArgs&, VideoInfo &video_info, std::string &err_str) { std::string video_id; if(!youtube_url_extract_id(url, video_id)) { fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); return PluginResult::ERR; } const int num_request_types = 1; std::string request_data[num_request_types] = { R"END( {"context":{"client":{"hl":"en","gl":"US","clientName":"IOS","clientVersion":"17.33.2","deviceModel":"iPhone14,3"}},"videoId":"%VIDEO_ID%"} )END", }; std::string client_names[num_request_types] = { "1" }; std::string client_versions[num_request_types] = { "2.20210622.10.00" }; for(int i = 0; i < num_request_types; ++i) { string_replace_all(request_data[i], "%VIDEO_ID%", video_id); std::vector additional_args = { { "-H", "Content-Type: application/json" }, { "-H", "X-YouTube-Client-Name: " + client_names[i] }, { "-H", "X-YouTube-Client-Version: " + client_versions[i] }, { "--data-raw", std::move(request_data[i]) } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); Json::Value json_root; DownloadResult download_result = download_json(json_root, "https://www.youtube.com/youtubei/v1/player?key=" + api_key + "&gl=US&hl=en&prettyPrint=false", additional_args, true); if(download_result != DownloadResult::OK) continue; PluginResult result = parse_video_response(json_root, video_info.title, video_info.channel_url, video_info.chapters, err_str); if(result == PluginResult::OK) { err_str.clear(); video_info.duration = video_details.duration; return PluginResult::OK; } } return PluginResult::ERR; } void YoutubeVideoPage::mark_watched() { if(playback_url.empty()) { fprintf(stderr, "Failed to mark video as watched because playback_url is empty\n"); return; } std::vector additional_args = { { "-H", "x-youtube-client-name: 1" }, { "-H", youtube_client_version } }; std::vector cookies = get_cookies(); additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); std::string response; DownloadResult download_result = download_to_string(playback_url + "&ver=2&cpn=" + cpn + "&gl=US&hl=en", response, std::move(additional_args), true); if(download_result != DownloadResult::OK) { fprintf(stderr, "Failed to mark video as watched because the http request failed\n"); return; } } void YoutubeVideoPage::get_subtitles(SubtitleData &subtitle_data) { auto it = subtitle_urls_by_lang_code.find("en"); if(it != subtitle_urls_by_lang_code.end()) { subtitle_data = it->second; return; } it = subtitle_urls_by_lang_code.find("en-US"); if(it != subtitle_urls_by_lang_code.end()) { subtitle_data = it->second; return; } } static bool parse_cipher_format(const Json::Value &format, YoutubeFormat &youtube_format) { std::map cipher_params; const Json::Value &cipher_json = format["cipher"]; if(cipher_json.isString()) { cipher_params = http_params_parse(cipher_json.asString()); } else { const Json::Value &signature_cipher_json = format["signatureCipher"]; if(signature_cipher_json.isString()) cipher_params = http_params_parse(signature_cipher_json.asString()); } std::string &url = cipher_params["url"]; if(url.empty()) return false; if(cipher_params.empty()) { youtube_format.url = url; return true; } youtube_format.url = std::move(url); return true; } void YoutubeVideoPage::parse_format(const Json::Value &format_json, bool is_adaptive) { if(!format_json.isArray()) return; for(const Json::Value &format : format_json) { if(!format.isObject()) continue; if(is_adaptive) { // TODO: Fix. Some streams use &sq=num instead of index const Json::Value &index_range_json = format["indexRange"]; if(index_range_json.isNull()) { fprintf(stderr, "Ignoring adaptive stream without indexRange\n"); continue; } } // TODO: Support HDR? const Json::Value &quality_label_json = format["qualityLabel"]; if(quality_label_json.isString() && strstr(quality_label_json.asCString(), "HDR")) continue; YoutubeFormat youtube_format_base; const Json::Value &itag_json = format["itag"]; if(!itag_json.isInt() || itag_json.asInt() == 22) continue; // TODO: itag 22 video format is broken right now server-side for some reason const Json::Value &mime_type_json = format["mimeType"]; if(!mime_type_json.isString()) continue; youtube_format_base.mime_type = mime_type_json.asString(); const Json::Value &bitrate_json = format["bitrate"]; if(!bitrate_json.isInt()) continue; youtube_format_base.bitrate = bitrate_json.asInt(); if(strncmp(youtube_format_base.mime_type.c_str(), "video/", 6) == 0) { bool has_embedded_audio = false; const char *codecs_p = strstr(youtube_format_base.mime_type.c_str(), "codecs=\""); if(codecs_p) { codecs_p += 8; const char *codecs_sep_p = strchr(codecs_p, ','); const char *codecs_end_p = strchr(codecs_p, '"'); has_embedded_audio = (codecs_sep_p && (!codecs_end_p || codecs_sep_p < codecs_end_p)); } YoutubeVideoFormat video_format; video_format.base = std::move(youtube_format_base); video_format.has_embedded_audio = has_embedded_audio; const Json::Value &width_json = format["width"]; if(!width_json.isInt()) continue; video_format.width = width_json.asInt(); const Json::Value &height_json = format["height"]; if(!height_json.isInt()) continue; video_format.height = height_json.asInt(); const Json::Value &fps_json = format["fps"]; if(!fps_json.isInt()) continue; video_format.fps = fps_json.asInt(); const Json::Value &url_json = format["url"]; if(url_json.isString()) { video_format.base.url = url_json.asString(); } else { if(!parse_cipher_format(format, video_format.base)) continue; } video_formats.push_back(std::move(video_format)); } else if(strncmp(youtube_format_base.mime_type.c_str(), "audio/", 6) == 0) { // Some youtube videos have multiple audio tracks and sometimes the audio tracks are in the same language // and one audio track may be descriptive/commentary. We only want the original audio for now const Json::Value &audio_track_json = format["audioTrack"]; if(audio_track_json.isObject()) { const Json::Value &audio_is_default_json = audio_track_json["audioIsDefault"]; if(audio_is_default_json.isBool() && !audio_is_default_json.asBool()) continue; } YoutubeAudioFormat audio_format; audio_format.base = std::move(youtube_format_base); const Json::Value &url_json = format["url"]; if(url_json.isString()) { audio_format.base.url = url_json.asString(); } else { if(!parse_cipher_format(format, audio_format.base)) continue; } audio_formats.push_back(std::move(audio_format)); } } } void YoutubeVideoPage::parse_formats(const Json::Value &streaming_data_json) { const Json::Value &formats_json = streaming_data_json["formats"]; parse_format(formats_json, false); const Json::Value &adaptive_formats_json = streaming_data_json["adaptiveFormats"]; parse_format(adaptive_formats_json, true); } void YoutubeVideoPage::set_watch_progress(int64_t time_pos_sec, int64_t duration_sec) { std::string video_id; if(!youtube_url_extract_id(url, video_id)) { show_notification("QuickMedia", "Failed to extract youtube id from " + url); return; } set_watch_progress_for_plugin("youtube", video_id, time_pos_sec, duration_sec, video_id); } }