From 89383cff1ba5d8a928262fcb4c40382a981c78c8 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Fri, 11 Jun 2021 04:51:03 +0200 Subject: Remove dependency on youtube-dl for streaming youtube, resulting in faster video startup --- src/plugins/Youtube.cpp | 269 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 266 insertions(+), 3 deletions(-) (limited to 'src/plugins/Youtube.cpp') diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index d9df409..9cec69c 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -1,10 +1,12 @@ #include "../../plugins/Youtube.hpp" +#include "../../plugins/youtube/Signature.hpp" #include "../../include/Storage.hpp" #include "../../include/NetUtils.hpp" #include "../../include/StringUtils.hpp" #include "../../include/Scale.hpp" #include "../../include/Notification.hpp" #include "../../include/Utils.hpp" +#include extern "C" { #include } @@ -13,6 +15,30 @@ extern "C" { #include namespace QuickMedia { + bool youtube_url_extract_id(const std::string &youtube_url, std::string &youtube_video_id) { + size_t index = youtube_url.find("youtube.com/watch?v="); + if(index != std::string::npos) { + index += 20; + size_t end_index = youtube_url.find("&", index); + if(end_index == std::string::npos) + end_index = youtube_url.size(); + youtube_video_id = youtube_url.substr(index, end_index - index); + return true; + } + + index = youtube_url.find("youtu.be/"); + if(index != std::string::npos) { + index += 9; + size_t end_index = youtube_url.find("?", index); + if(end_index == std::string::npos) + end_index = youtube_url.size(); + youtube_video_id = youtube_url.substr(index, end_index - index); + return true; + } + + return false; + } + // This is a common setup of text in the youtube json static std::optional yt_json_get_text(const Json::Value &json, const char *root_name) { if(!json.isObject()) @@ -257,7 +283,10 @@ namespace QuickMedia { body_item->set_description_color(sf::Color(179, 179, 179)); body_item->url = "https://www.youtube.com/channel/" + channel_id_json.asString(); if(thumbnail) { - body_item->thumbnail_url = std::string("https:") + thumbnail->url; + if(string_starts_with(thumbnail->url, "https:")) + body_item->thumbnail_url = thumbnail->url; + else + body_item->thumbnail_url = std::string("https:") + thumbnail->url; body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE; body_item->thumbnail_size.x = thumbnail->width; body_item->thumbnail_size.y = thumbnail->height; @@ -386,13 +415,16 @@ namespace QuickMedia { static std::vector get_cookies() { std::lock_guard lock(cookies_mutex); if(cookies_filepath.empty()) { + YoutubeSignatureDecryptor::get_instance(); + Path cookies_filepath_p; if(get_cookies_filepath(cookies_filepath_p, "youtube") != 0) { show_notification("QuickMedia", "Failed to create youtube cookies file", Urgency::CRITICAL); return {}; } - // TODO: Re-enable this if the api key ever changes in the future + // TODO: Re-enable this if the api key ever changes in the future. + // Maybe also put signature decryption in the same request? since it requests the same page. #if 0 //api_key = youtube_page_find_api_key(); #else @@ -655,6 +687,11 @@ namespace QuickMedia { return PluginResult::OK; } + PluginResult YoutubeSearchPage::lazy_fetch(BodyItems&) { + get_cookies(); + return PluginResult::OK; + } + PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string ¤t_continuation_token, BodyItems &result_items) { std::string next_url = url + "&pbj=1&ctoken=" + current_continuation_token; @@ -1800,11 +1837,237 @@ namespace QuickMedia { return std::make_unique(program, xsrf_token, comments_continuation_token); } - std::unique_ptr YoutubeVideoPage::create_related_videos_page(Program *program, const std::string&, const std::string&) { + std::unique_ptr YoutubeVideoPage::create_related_videos_page(Program *program) { return std::make_unique(program); } std::unique_ptr YoutubeVideoPage::create_channels_page(Program *program, const std::string &channel_url) { return std::make_unique(program, channel_url, "", "Channel videos"); } + + static std::map http_params_parse(const std::string &http_params) { + std::map result; + string_split(http_params, '&', [&result](const char *str, size_t size) { + const void *split_p = memchr(str, '=', size); + if(split_p == nullptr) + return true; + + std::string key(str, (const char*)split_p - str); + std::string value((const char*)split_p + 1, (str + size) - ((const char*)split_p + 1)); + key = url_param_decode(key); + value = url_param_decode(value); + result[std::move(key)] = std::move(value); + return true; + }); + return result; + } + + static std::string url_extract_param(const std::string &url, const std::string ¶m) { + std::string param_s = param + "="; + size_t index = url.find(param_s); + if(index == std::string::npos) + return ""; + + index += param_s.size(); + size_t end = url.find('&', index); + if(end == std::string::npos) + end = url.size(); + + return url.substr(index, end - index); + } + + std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio) { + if(video_formats.empty()) { + has_embedded_audio = true; + return ""; + } + + for(const auto &video_format : video_formats) { + if(video_format.height <= max_height) { + has_embedded_audio = video_format.has_embedded_audio; + return video_format.base.url; + } + } + + has_embedded_audio = video_formats.back().has_embedded_audio; + return video_formats.back().base.url; + } + + std::string YoutubeVideoPage::get_audio_url() { + if(audio_formats.empty()) + return ""; + + return audio_formats.front().base.url; + } + + PluginResult YoutubeVideoPage::load() { + video_formats.clear(); + audio_formats.clear(); + + std::string video_id; + if(!youtube_url_extract_id(url, video_id)) { + fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); + return PluginResult::ERR; + } + + std::vector additional_args = { + { "-H", "x-youtube-client-name: 1" }, + { "-H", "x-youtube-client-version: 2.20200626.03.00" } + }; + + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + + std::string response; + DownloadResult download_result = download_to_string("https://www.youtube.com/get_video_info?html5=1&video_id=" + video_id + "&eurl=https://www.youtube.googleapis.com/v/" + video_id, response, std::move(additional_args), true); // TODO: true? + if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); + + std::string player_response_param = url_extract_param(response, "player_response"); + player_response_param = url_param_decode(player_response_param); + + Json::Value json_root; + Json::CharReaderBuilder json_builder; + std::unique_ptr json_reader(json_builder.newCharReader()); + std::string json_errors; + if(!json_reader->parse(player_response_param.data(), player_response_param.data() + player_response_param.size(), &json_root, &json_errors)) { + fprintf(stderr, "Failed to read param as json, error: %s\n", json_errors.c_str()); + return PluginResult::ERR; + } + + if(!json_root.isObject()) + return PluginResult::ERR; + + const Json::Value &streaming_data_json = json_root["streamingData"]; + if(!streaming_data_json.isObject()) + return PluginResult::ERR; + + parse_formats(streaming_data_json); + + if(video_formats.empty() && audio_formats.empty()) + return PluginResult::ERR; + + std::sort(video_formats.begin(), video_formats.end(), [](const YoutubeVideoFormat &format1, const YoutubeVideoFormat &format2) { + return format1.base.bitrate > format2.base.bitrate; + }); + + std::sort(audio_formats.begin(), audio_formats.end(), [](const YoutubeAudioFormat &format1, const YoutubeAudioFormat &format2) { + return format1.base.bitrate > format2.base.bitrate; + }); + + return PluginResult::OK; + } + + static bool parse_cipher_format(const Json::Value &format, YoutubeFormat &youtube_format) { + std::map cipher_params; + const Json::Value &cipher_json = format["cipher"]; + if(cipher_json.isString()) { + cipher_params = http_params_parse(cipher_json.asString()); + } else { + const Json::Value &signature_cipher_json = format["signatureCipher"]; + if(signature_cipher_json.isString()) + cipher_params = http_params_parse(signature_cipher_json.asString()); + } + + const std::string &url = cipher_params["url"]; + if(cipher_params.empty() || url.empty()) + return false; + + std::string url_decoded = url_param_decode(url); + + const std::string &s = cipher_params["s"]; + const std::string &sp = cipher_params["sp"]; + std::string sig_key; + std::string sig_value; + if(YoutubeSignatureDecryptor::get_instance().decrypt(s, sp, sig_key, sig_value)) + url_decoded += "&" + std::move(sig_key) + "=" + std::move(sig_value); + + youtube_format.url = std::move(url_decoded); + return true; + } + + void YoutubeVideoPage::parse_format(const Json::Value &format_json, bool is_adaptive) { + if(!format_json.isArray()) + return; + + for(const Json::Value &format : format_json) { + if(!format.isObject()) + continue; + + if(is_adaptive) { + // TODO: Fix. Some streams use sq/ instead of index + const Json::Value &index_range_json = format["indexRange"]; + if(index_range_json.isNull()) { + fprintf(stderr, "Ignoring adaptive stream without indexRange\n"); + continue; + } + } + + YoutubeFormat youtube_format_base; + + const Json::Value &mime_type_json = format["mimeType"]; + if(!mime_type_json.isString()) continue; + + const Json::Value &bitrate_json = format["bitrate"]; + if(!bitrate_json.isInt()) continue; + youtube_format_base.bitrate = bitrate_json.asInt(); + + if(strncmp(mime_type_json.asCString(), "video/", 6) == 0) { + bool has_embedded_audio = false; + const char *codecs_p = strstr(mime_type_json.asCString(), "codecs=\""); + if(codecs_p) { + codecs_p += 8; + const char *codecs_sep_p = strchr(codecs_p, ','); + const char *codecs_end_p = strchr(codecs_p, '"'); + has_embedded_audio = (codecs_sep_p != nullptr && (!codecs_end_p || codecs_sep_p < codecs_end_p)); + } + + YoutubeVideoFormat video_format; + video_format.base = std::move(youtube_format_base); + video_format.has_embedded_audio = has_embedded_audio; + + const Json::Value &width_json = format["width"]; + if(!width_json.isInt()) continue; + video_format.width = width_json.asInt(); + + const Json::Value &height_json = format["height"]; + if(!height_json.isInt()) continue; + video_format.height = height_json.asInt(); + + const Json::Value &fps_json = format["fps"]; + if(!fps_json.isInt()) continue; + video_format.fps = fps_json.asInt(); + + const Json::Value &url_json = format["url"]; + if(url_json.isString()) { + video_format.base.url = url_json.asString(); + } else { + if(!parse_cipher_format(format, video_format.base)) + continue; + } + + video_formats.push_back(std::move(video_format)); + } else if(strncmp(mime_type_json.asCString(), "audio/", 6) == 0) { + YoutubeAudioFormat audio_format; + audio_format.base = std::move(youtube_format_base); + + const Json::Value &url_json = format["url"]; + if(url_json.isString()) { + audio_format.base.url = url_json.asString(); + } else { + if(!parse_cipher_format(format, audio_format.base)) + continue; + } + + audio_formats.push_back(std::move(audio_format)); + } + } + } + + void YoutubeVideoPage::parse_formats(const Json::Value &streaming_data_json) { + const Json::Value &formats_json = streaming_data_json["formats"]; + parse_format(formats_json, false); + + const Json::Value &adaptive_formats_json = streaming_data_json["adaptiveFormats"]; + parse_format(adaptive_formats_json, false); + } } \ No newline at end of file -- cgit v1.2.3