From 38202de4f953fca28aa884246ced0aadf0d25a4d Mon Sep 17 00:00:00 2001 From: dec05eba Date: Fri, 25 Jun 2021 12:44:53 +0200 Subject: Add a http server proxy for better youtube downloading (bypassing rate limit cased by http range header). Fix youtube live streams --- src/DownloadUtils.cpp | 22 +- src/Program.cpp | 9 +- src/QuickMedia.cpp | 159 ++++++- src/VideoPlayer.cpp | 5 +- src/plugins/Youtube.cpp | 129 +++--- src/plugins/youtube/YoutubeMediaProxy.cpp | 733 ++++++++++++++++++++++++++++++ 6 files changed, 966 insertions(+), 91 deletions(-) create mode 100644 src/plugins/youtube/YoutubeMediaProxy.cpp (limited to 'src') diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index 5ab79bb..5be1990 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -29,19 +29,13 @@ namespace QuickMedia { return 0; } - // Returns 0 if content length is not found - static long get_content_length(const std::string &header) { - std::string content_length_str = header_extract_value(header, "content-length"); - if(content_length_str.empty()) - return 0; - - errno = 0; - char *endptr; - const long content_length_num = strtol(content_length_str.c_str(), &endptr, 10); - if(endptr != content_length_str.c_str() && errno == 0) - return content_length_num; + static bool http_is_redirect(const std::string &header, size_t size) { + size_t end_of_first_line = header.find("\r\n"); + if(end_of_first_line == std::string::npos) + return false; - return 0; + size_t find_index = header.find(" 30"); + return find_index != std::string::npos && find_index < size; } static int accumulate_string_with_header(char *data, int size, void *userdata) { @@ -55,9 +49,9 @@ namespace QuickMedia { size_t end_of_headers_index = download_userdata->header->find("\r\n\r\n"); if(end_of_headers_index != std::string::npos) { while(true) { - const long content_length = get_content_length(download_userdata->header->substr(0, end_of_headers_index)); // TODO: Do not create a copy of the header string + const bool is_redirect = http_is_redirect(*download_userdata->header, end_of_headers_index); end_of_headers_index += 4; - if(content_length == 0 && download_userdata->header->size() - end_of_headers_index > 0) { + if(is_redirect && download_userdata->header->size() - end_of_headers_index > 0) { download_userdata->header->erase(download_userdata->header->begin(), download_userdata->header->begin() + end_of_headers_index); end_of_headers_index = download_userdata->header->find("\r\n\r\n"); if(end_of_headers_index == std::string::npos) diff --git a/src/Program.cpp b/src/Program.cpp index 5220a4c..57d7c61 100644 --- a/src/Program.cpp +++ b/src/Program.cpp @@ -76,6 +76,9 @@ public: thread_local CurrentThreadProgram current_thread_program; int exec_program_pipe(const char **args, ReadProgram *read_program) { + read_program->pid = -1; + read_program->read_fd = -1; + /* 1 arguments */ if(args[0] == NULL) return -1; @@ -192,8 +195,9 @@ int exec_program(const char **args, ProgramOutputCallback output_callback, void int wait_program(pid_t process_id) { int status; if(waitpid(process_id, &status, 0) == -1) { + int err = -errno; perror("waitpid failed"); - return -errno; + return err; } if(!WIFEXITED(status)) @@ -206,8 +210,9 @@ int wait_program_non_blocking(pid_t process_id, int *status) { int s; int wait_result = waitpid(process_id, &s, WNOHANG); if(wait_result == -1) { + int err = -errno; perror("waitpid failed"); - *status = -errno; + *status = err; return 0; } else if(wait_result == 0) { /* the child process is still running */ diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 7305788..e6e4719 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -29,6 +29,7 @@ #include "../include/Utils.hpp" #include "../include/Tabs.hpp" #include "../include/Theme.hpp" +#include "../plugins/youtube/YoutubeMediaProxy.hpp" #include "../include/gui/Button.hpp" #include "../external/hash-library/sha256.h" @@ -1049,6 +1050,13 @@ namespace QuickMedia { return PluginResult::OK; } + static void check_youtube_dl_installed(const std::string &plugin_name) { + if(!is_program_executable_by_name("youtube-dl")) { + show_notification("QuickMedia", "youtube-dl needs to be installed to play " + plugin_name + " videos", Urgency::CRITICAL); + abort(); + } + } + void Program::load_plugin_by_name(std::vector &tabs, int &start_tab_index, FileManagerMimeType fm_mime_type, FileSelectionHandler file_selection_handler) { if(!plugin_name || plugin_name[0] == '\0') return; @@ -1208,18 +1216,22 @@ namespace QuickMedia { video_content_page(nullptr, youtube_video_page.get(), "", false, nullptr, body_items, 0); } } else if(strcmp(plugin_name, "pornhub") == 0) { + check_youtube_dl_installed(plugin_name); auto search_page = std::make_unique(this, "https://www.pornhub.com/", sf::Vector2i(320/1.5f, 180/1.5f)); add_pornhub_handlers(search_page.get()); tabs.push_back(Tab{create_body(false, true), std::move(search_page), create_search_bar("Search...", 500)}); } else if(strcmp(plugin_name, "spankbang") == 0) { + check_youtube_dl_installed(plugin_name); auto search_page = std::make_unique(this, "https://spankbang.com/", sf::Vector2i(500/2.5f, 281/2.5f)); add_spankbang_handlers(search_page.get()); tabs.push_back(Tab{create_body(false, true), std::move(search_page), create_search_bar("Search...", 500)}); } else if(strcmp(plugin_name, "xvideos") == 0) { + check_youtube_dl_installed(plugin_name); auto search_page = std::make_unique(this, "https://www.xvideos.com/", sf::Vector2i(352/1.5f, 198/1.5f)); add_xvideos_handlers(search_page.get()); tabs.push_back(Tab{create_body(false, true), std::move(search_page), create_search_bar("Search...", 500)}); } else if(strcmp(plugin_name, "xhamster") == 0) { + check_youtube_dl_installed(plugin_name); auto search_page = std::make_unique(this, "https://xhamster.com/", sf::Vector2i(240, 135)); add_xhamster_handlers(search_page.get()); tabs.push_back(Tab{create_body(false, true), std::move(search_page), create_search_bar("Search...", 500)}); @@ -2469,6 +2481,11 @@ namespace QuickMedia { return true; } + // TODO: Test with video that has hlsManifestUrl + static bool youtube_url_is_live_stream(const std::string &url) { + return url.find("yt_live_broadcast") != std::string::npos; + } + #define CLEANMASK(mask) ((mask) & (ShiftMask|ControlMask|Mod1Mask|Mod4Mask|Mod5Mask)) void Program::video_content_page(Page *parent_page, VideoPage *video_page, std::string video_title, bool download_if_streaming_fails, Body *parent_body, BodyItems &next_play_items, int play_index, int *parent_body_page, const std::string &parent_page_search) { @@ -2496,6 +2513,12 @@ namespace QuickMedia { XSync(disp, False); }; + std::unique_ptr youtube_video_media_proxy; + std::unique_ptr youtube_audio_media_proxy; + AsyncTask youtube_downloader_task; + int youtube_video_content_length = 0; + int youtube_audio_content_length = 0; + std::string channel_url; AsyncTask video_tasks; std::function video_event_callback; @@ -2514,7 +2537,7 @@ namespace QuickMedia { std::string prev_start_time; std::vector media_chapters; - auto load_video_error_check = [this, &prev_start_time, &media_chapters, &in_seeking, &video_url, &audio_url, &has_embedded_audio, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable { + auto load_video_error_check = [this, &youtube_downloader_task, &youtube_video_media_proxy, &youtube_audio_media_proxy, &youtube_video_content_length, &youtube_audio_content_length, &prev_start_time, &media_chapters, &in_seeking, &video_url, &audio_url, &has_embedded_audio, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable { video_player.reset(); channel_url.clear(); video_loaded = false; @@ -2530,7 +2553,7 @@ namespace QuickMedia { audio_url.clear(); has_embedded_audio = true; - TaskResult load_result = run_task_with_loading_screen([this, video_page, &new_title, &channel_url, &media_chapters, largest_monitor_height, &has_embedded_audio, &video_url, &audio_url, &is_audio_only, &previous_page, is_youtube, download_if_streaming_fails]() { + TaskResult load_result = run_task_with_loading_screen([this, video_page, &youtube_video_content_length, &youtube_audio_content_length, &new_title, &channel_url, &media_chapters, largest_monitor_height, &has_embedded_audio, &video_url, &audio_url, &is_audio_only, &previous_page, is_youtube, download_if_streaming_fails]() { if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK) return false; @@ -2555,6 +2578,37 @@ namespace QuickMedia { return false; } + if(is_youtube) { + // TODO: Do these requests in parallel + std::pair media_url_content_lengths[2] = { + std::make_pair(&video_url, &youtube_video_content_length), + std::make_pair(&audio_url, &youtube_audio_content_length), + }; + for(int i = 0; i < 2; ++i) { + if(media_url_content_lengths[i].first->empty() || youtube_url_is_live_stream(*media_url_content_lengths[i].first)) { + *media_url_content_lengths[i].second = 0; + continue; + } + + std::string headers; + if(download_head_to_string(*media_url_content_lengths[i].first, headers) != DownloadResult::OK) + return false; + + std::string content_length = header_extract_value(headers, "content-length"); + if(content_length.empty()) + return false; + + errno = 0; + char *endptr; + const long content_length_tmp = strtol(content_length.c_str(), &endptr, 10); + if(endptr != content_length.c_str() && errno == 0) { + *media_url_content_lengths[i].second = content_length_tmp; + } else { + return false; + } + } + } + return true; }); @@ -2579,9 +2633,82 @@ namespace QuickMedia { prev_start_time = start_time; watched_videos.insert(video_page->get_url()); + // TODO: Sync sequences + //audio_url.clear(); + //video_url.clear(); + //is_audio_only = true; + + std::string v = video_url; + std::string a = audio_url; + if(is_youtube) { + if(youtube_video_media_proxy) + youtube_video_media_proxy->stop(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->stop(); + + if(youtube_downloader_task.valid()) + youtube_downloader_task.cancel(); + + youtube_video_media_proxy.reset(); + youtube_audio_media_proxy.reset(); + + struct MediaProxyMetadata { + std::unique_ptr *media_proxy; + std::string *url; + int content_length; + }; + + MediaProxyMetadata media_proxies[2] = { + { &youtube_video_media_proxy, &v, youtube_video_content_length }, + { &youtube_audio_media_proxy, &a, youtube_audio_content_length } + }; + int num_proxied_media = 0; + for(int i = 0; i < 2; ++i) { + if(media_proxies[i].url->empty() || youtube_url_is_live_stream(*media_proxies[i].url)) + continue; + + *media_proxies[i].media_proxy = std::make_unique(); + if(!(*media_proxies[i].media_proxy)->start(*media_proxies[i].url, media_proxies[i].content_length)) { + show_notification("QuickMedia", "Failed to load start youtube media proxy", Urgency::CRITICAL); + current_page = previous_page; + go_to_previous_page = true; + return; + } + + std::string media_proxy_addr; + if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) { + show_notification("QuickMedia", "Failed to load start youtube media proxy", Urgency::CRITICAL); + current_page = previous_page; + go_to_previous_page = true; + return; + } + + *media_proxies[i].url = std::move(media_proxy_addr); + ++num_proxied_media; + } + + if(num_proxied_media > 0) { + youtube_downloader_task = AsyncTask([&youtube_video_media_proxy, &youtube_audio_media_proxy]() { + sf::Clock timer; + const double sleep_time_millisec = 1; + while(!program_is_dead_in_current_thread()) { + if(youtube_video_media_proxy) + youtube_video_media_proxy->update(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->update(); + + const int sleep_time = sleep_time_millisec - timer.restart().asMilliseconds(); + if(sleep_time > 0) + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); + } + }); + } + } video_player = std::make_unique(is_audio_only, use_system_mpv_config, is_matrix && !is_youtube, video_event_callback, on_window_create, resources_root, largest_monitor_height, plugin_name); - VideoPlayer::Error err = video_player->load_video(video_url.c_str(), audio_url.c_str(), window.getSystemHandle(), is_youtube, video_title, start_time, media_chapters); + VideoPlayer::Error err = video_player->load_video(v.c_str(), a.c_str(), window.getSystemHandle(), is_youtube, video_title, start_time, media_chapters); if(err != VideoPlayer::Error::OK) { std::string err_msg = "Failed to play url: "; err_msg += video_page->get_url(); @@ -2787,11 +2914,23 @@ namespace QuickMedia { bool page_changed = false; double resume_start_time = 0.0; - page_loop(tabs, 1, [this, &page_changed, &resume_start_time](const std::vector &new_tabs) { + page_loop(tabs, 1, [this, &page_changed, &resume_start_time, &youtube_video_media_proxy, &youtube_audio_media_proxy, &youtube_downloader_task](const std::vector &new_tabs) { if(!page_changed && new_tabs.size() == 1 && new_tabs[0].page->get_type() == PageTypez::VIDEO) { video_player->get_time_in_file(&resume_start_time); video_player.reset(); page_changed = true; + + if(youtube_video_media_proxy) + youtube_video_media_proxy->stop(); + + if(youtube_audio_media_proxy) + youtube_audio_media_proxy->stop(); + + if(youtube_downloader_task.valid()) + youtube_downloader_task.cancel(); + + youtube_video_media_proxy.reset(); + youtube_audio_media_proxy.reset(); } }); @@ -6339,6 +6478,7 @@ namespace QuickMedia { if(exec_program_pipe(args, &read_program) != 0) return false; + // TODO: Remove this async task and make the fd non blocking instead header_reader = AsyncTask([this]{ char tmp_buf[1024]; while(true) { @@ -6368,10 +6508,10 @@ namespace QuickMedia { } void stop(bool download_completed) override { - if(read_program.pid != -1) - close(read_program.pid); if(read_program.read_fd != -1) - kill(read_program.read_fd, SIGTERM); + close(read_program.read_fd); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); if(!download_completed) remove(output_filepath_tmp.data.c_str()); //header_reader.cancel(); @@ -6476,6 +6616,7 @@ namespace QuickMedia { return false; } + // TODO: Remove this async task and make the fd non blocking instead youtube_dl_output_reader = AsyncTask([this]{ char line[128]; char progress_c[10]; @@ -6524,8 +6665,8 @@ namespace QuickMedia { void stop(bool) override { if(read_program_file) fclose(read_program_file); - if(read_program.read_fd != -1) - kill(read_program.read_fd, SIGTERM); + if(read_program.pid != -1) + kill(read_program.pid, SIGTERM); // TODO: Remove the temporary files created by youtube-dl (if !download_completed) //header_reader.cancel(); } diff --git a/src/VideoPlayer.cpp b/src/VideoPlayer.cpp index 2407955..65e894f 100644 --- a/src/VideoPlayer.cpp +++ b/src/VideoPlayer.cpp @@ -232,7 +232,8 @@ namespace QuickMedia { } int flags = fcntl(sockets[0], F_GETFL, 0); - fcntl(sockets[0], F_SETFL, flags | O_NONBLOCK); + if(flags != -1) // TODO: Proper error handling + fcntl(sockets[0], F_SETFL, flags | O_NONBLOCK); connected_to_ipc = true; return Error::OK; @@ -342,7 +343,7 @@ namespace QuickMedia { ssize_t bytes_read = read(sockets[0], buffer, sizeof(buffer)); if(bytes_read == -1) { int err = errno; - if(err != EAGAIN) { + if(err != EAGAIN && err != EWOULDBLOCK) { fprintf(stderr, "Failed to read from ipc socket, error: %s\n", strerror(err)); return Error::FAIL_TO_READ; } diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp index 4df1358..f399687 100644 --- a/src/plugins/Youtube.cpp +++ b/src/plugins/Youtube.cpp @@ -2022,9 +2022,9 @@ R"END( } std::string YoutubeVideoPage::get_video_url(int max_height, bool &has_embedded_audio) { - if(!hls_manifest_url.empty()) { + if(!livestream_url.empty()) { has_embedded_audio = true; - return hls_manifest_url; + return livestream_url; } if(video_formats.empty()) { @@ -2100,6 +2100,7 @@ R"END( return result; } + // TODO: Extract innertube_api_key from response? PluginResult YoutubeVideoPage::get_video_info(const std::string &video_id, Json::Value &json_root) { std::vector additional_args = get_cookies(); @@ -2121,65 +2122,13 @@ R"END( return PluginResult::OK; } - PluginResult YoutubeVideoPage::load(std::string &title, std::string &channel_url, std::vector &chapters) { - hls_manifest_url.clear(); + PluginResult YoutubeVideoPage::parse_video_response(Json::Value &json_root, std::string &title, std::string &channel_url, std::vector &chapters) { + livestream_url.clear(); video_formats.clear(); audio_formats.clear(); - - std::string video_id; - if(!youtube_url_extract_id(url, video_id)) { - fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); - return PluginResult::ERR; - } - - #if 0 - std::string request_data = key_api_request_data; - string_replace_all(request_data, "%VIDEO_ID%", video_id); - - std::vector additional_args = { - { "-H", "Content-Type: application/json" }, - { "-H", "x-youtube-client-name: 1" }, - { "-H", youtube_client_version }, - { "--data-raw", std::move(request_data) } - }; - - std::vector cookies = get_cookies(); - additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); - - Json::Value json_root; - DownloadResult download_result = download_json(json_root, "https://www.youtube.com/youtubei/v1/player?key=" + api_key + "&gl=US&hl=en", additional_args, true); - if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); - - if(!json_root.isObject()) - return PluginResult::ERR; - - const Json::Value *streaming_data_json = &json_root["streamingData"]; - if(!streaming_data_json->isObject()) { - const Json::Value &playability_status_json = json_root["playabilityStatus"]; - if(playability_status_json.isObject()) { - const Json::Value &status_json = playability_status_json["status"]; - const Json::Value &reason_json = playability_status_json["reason"]; - fprintf(stderr, "Warning: youtube video loading failed, reason: (status: %s, reason: %s), trying with get_video_info endpoint instead\n", status_json.isString() ? status_json.asCString() : "unknown", reason_json.isString() ? reason_json.asCString() : "unknown"); - - json_root = Json::Value(Json::nullValue); - PluginResult result = get_video_info(video_id, json_root); - if(result != PluginResult::OK) - return result; - - if(!json_root.isObject()) - return PluginResult::ERR; - - streaming_data_json = &json_root["streamingData"]; - if(!streaming_data_json->isObject()) - return PluginResult::ERR; - } - return PluginResult::ERR; - } - #else - Json::Value json_root; - PluginResult result = get_video_info(video_id, json_root); - if(result != PluginResult::OK) - return result; + title.clear(); + channel_url.clear(); + chapters.clear(); if(!json_root.isObject()) return PluginResult::ERR; @@ -2187,13 +2136,22 @@ R"END( const Json::Value *streaming_data_json = &json_root["streamingData"]; if(!streaming_data_json->isObject()) return PluginResult::ERR; - #endif // TODO: Verify if this always works (what about copyrighted live streams?), also what about choosing video quality for live stream? Maybe use mpv --hls-bitrate option? const Json::Value &hls_manifest_url_json = (*streaming_data_json)["hlsManifestUrl"]; - if(hls_manifest_url_json.isString()) { - hls_manifest_url = hls_manifest_url_json.asString(); - } else { + if(hls_manifest_url_json.isString()) + livestream_url = hls_manifest_url_json.asString(); + + /* + const Json::Value &dash_manifest_url_json = (*streaming_data_json)["dashManifestUrl"]; + if(livestream_url.empty() && dash_manifest_url_json.isString()) { + // TODO: mpv cant properly play dash videos. Video goes back and replays. + // So for now return here (get_video_info only hash dash stream and no hls stream) which will fallback to the player youtube endpoint which has hls stream. + return PluginResult::ERR; + } + */ + + if(livestream_url.empty()) { parse_formats(*streaming_data_json); if(video_formats.empty() && audio_formats.empty()) return PluginResult::ERR; @@ -2257,6 +2215,49 @@ R"END( return PluginResult::OK; } + PluginResult YoutubeVideoPage::load(std::string &title, std::string &channel_url, std::vector &chapters) { + livestream_url.clear(); + video_formats.clear(); + audio_formats.clear(); + + std::string video_id; + if(!youtube_url_extract_id(url, video_id)) { + fprintf(stderr, "Failed to extract youtube id from %s\n", url.c_str()); + return PluginResult::ERR; + } + + Json::Value json_root; + PluginResult result = get_video_info(video_id, json_root); + if(result != PluginResult::OK) + return result; + + // Getting streams might fail for some videos that do not allow videos to be embedded when using get_video_info endpoint. + // TODO: Does that means for videos that do not allow to be embedded and are age restricted wont work? + result = parse_video_response(json_root, title, channel_url, chapters); + if(result == PluginResult::OK) { + return PluginResult::OK; + } else { + std::string request_data = key_api_request_data; + string_replace_all(request_data, "%VIDEO_ID%", video_id); + + std::vector additional_args = { + { "-H", "Content-Type: application/json" }, + { "-H", "x-youtube-client-name: 1" }, + { "-H", youtube_client_version }, + { "--data-raw", std::move(request_data) } + }; + + std::vector cookies = get_cookies(); + additional_args.insert(additional_args.end(), cookies.begin(), cookies.end()); + + Json::Value json_root; + DownloadResult download_result = download_json(json_root, "https://www.youtube.com/youtubei/v1/player?key=" + api_key + "&gl=US&hl=en", additional_args, true); + if(download_result != DownloadResult::OK) return download_result_to_plugin_result(download_result); + + return parse_video_response(json_root, title, channel_url, chapters); + } + } + void YoutubeVideoPage::mark_watched() { if(playback_url.empty()) { fprintf(stderr, "Failed to mark video as watched because playback_url is empty\n"); @@ -2317,7 +2318,7 @@ R"END( continue; if(is_adaptive) { - // TODO: Fix. Some streams use sq/ instead of index + // TODO: Fix. Some streams use &sq=num instead of index const Json::Value &index_range_json = format["indexRange"]; if(index_range_json.isNull()) { fprintf(stderr, "Ignoring adaptive stream without indexRange\n"); diff --git a/src/plugins/youtube/YoutubeMediaProxy.cpp b/src/plugins/youtube/YoutubeMediaProxy.cpp new file mode 100644 index 0000000..e8d0383 --- /dev/null +++ b/src/plugins/youtube/YoutubeMediaProxy.cpp @@ -0,0 +1,733 @@ +#include "../../../plugins/youtube/YoutubeMediaProxy.hpp" +#include "../../../include/NetUtils.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +// TODO: What if the client sends a new header without reconnecting? is that even allowed by http standard? +// TODO: Detect when download has finished (and close connection). + +namespace QuickMedia { + static const int MAX_BUFFER_SIZE = 65536; + static const int RANGE = 524287; + static const char download_error_response_msg[] = + "HTTP/1.1 500 Internal Server Error\r\n" + "Content-Length: 0\r\n\r\n"; + + static bool set_non_blocking(int fd) { + const int flags = fcntl(fd, F_GETFL, 0); + if(fd == -1 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) + return false; + return true; + } + + // TODO: Restrict range end to remote file size (content-length which we have saved). + // TODO: Check if custom youtube redirect code is needed + bool YoutubeMediaProxy::start_download(const std::string &media_url, ReadProgram &read_program, int range_start, bool include_header, bool is_livestream, int livestream_sequence) { + std::string r = std::to_string(range_start) + "-" + std::to_string(range_start + RANGE); + + std::string url = media_url + "&rn=" + std::to_string(rn) + "&rbuf=" + std::to_string(rbuf); + std::vector args = { "curl", + //"-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", + //"-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f" }; + + if(is_livestream) { + if(livestream_sequence != -1) + url += "&sq=" + std::to_string(livestream_sequence); + } else { + args.insert(args.end(), { "-r", r.c_str() }); + } + + if(include_header) + args.push_back("-i"); + + //fprintf(stderr, "url: %s\n", url.c_str()); + + args.insert(args.end(), { "--", url.c_str(), nullptr }); + + if(exec_program_pipe(args.data(), &read_program) != 0) + return false; + + if(!set_non_blocking(read_program.read_fd)) { + perror("start_download: failed to set curl pipe non blocking mode"); + close(read_program.read_fd); + kill(read_program.pid, SIGTERM); + read_program.read_fd = -1; + read_program.pid = -1; + return false; + } + + ++rn; + rbuf += 3000; + if(rbuf > 75000) rbuf = 75000; + return true; + } + + YoutubeStaticMediaProxy::~YoutubeStaticMediaProxy() { + stop(); + } + + bool YoutubeStaticMediaProxy::start(const std::string &youtube_media_url, int content_length) { + if(socket_fd != -1) + return false; + + socket_fd = socket(AF_INET, SOCK_STREAM, 0); + if(socket_fd == -1) { + perror("YoutubeStaticMediaProxy::start: socket failed"); + return false; + } + + socklen_t response_sock_addr_len; + struct sockaddr_in server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + server_addr.sin_port = htons(0); + + if(bind(socket_fd, (struct sockaddr*)&server_addr, sizeof(server_addr)) == -1) { + perror("YoutubeStaticMediaProxy::start: bind failed"); + goto err; + } + + if(listen(socket_fd, 2) == -1) { + perror("YoutubeStaticMediaProxy::start: listen failed"); + goto err; + } + + if(!set_non_blocking(socket_fd)) { + perror("YoutubeStaticMediaProxy::start: failed to set socket non blocking mode"); + goto err; + } + + struct sockaddr_in response_sock_addr; + response_sock_addr_len = sizeof(response_sock_addr); + if(getsockname(socket_fd, (struct sockaddr*)&response_sock_addr, &response_sock_addr_len) == -1) { + perror("YoutubeStaticMediaProxy::start: getsockname failed"); + goto err; + } + + port = ntohs(response_sock_addr.sin_port); + this->youtube_media_url = youtube_media_url; + this->content_length = content_length; + return true; + + err: + if(downloader_read_program.read_fd != -1) + close(downloader_read_program.read_fd); + if(downloader_read_program.pid != -1) + kill(downloader_read_program.pid, SIGTERM); + close(socket_fd); + socket_fd = -1; + return false; + } + + void YoutubeStaticMediaProxy::stop() { + if(downloader_read_program.read_fd != -1) { + close(downloader_read_program.read_fd); + downloader_read_program.read_fd = -1; + } + + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; + } + + if(client_fd != -1) { + close(client_fd); + client_fd = -1; + } + + if(socket_fd != -1) { + close(socket_fd); + socket_fd = -1; + } + + clear_download_state(); + client_request_buffer.clear(); + client_request_finished = false; + } + + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update() { + if(socket_fd == -1) + return Error::OK; + + if(client_fd == -1) { + client_fd = accept_client(); + if(client_fd == -1) + return Error::OK; + } else { + const int new_client_fd = accept_client(); + if(new_client_fd != -1) { + on_client_disconnect(); + client_fd = new_client_fd; + } + } + + Error err = read_client_data(); + if(err != Error::OK || !client_request_finished) + return err; + if(downloader_read_program.pid == -1) + return Error::ERROR; + return handle_download(); + } + + bool YoutubeStaticMediaProxy::get_address(std::string &address) { + if(socket_fd == -1) + return false; + + address = "http://127.0.0.1:" + std::to_string(port); + return true; + } + + void YoutubeStaticMediaProxy::on_client_disconnect() { + client_request_buffer.clear(); + client_request_finished = false; + + if(client_fd != -1) { + close(client_fd); + client_fd = -1; + } + + if(downloader_read_program.pid != -1) + kill(downloader_read_program.pid, SIGTERM); + + update_download_program_status(true); + } + + // Returns 0 if start range is not found + static int header_extract_start_range(const std::string &header) { + std::string range = header_extract_value(header, "range"); + if(range.empty()) + return 0; + + int start_range = 0; + if(sscanf(range.c_str(), " bytes=%d", &start_range) != 1) + return 0; + + return start_range; + } + + // TODO: What about hls (live streams)? need to test with that. There may not be a need to use YoutubeMediaProxy for that case (in that case document it). + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::read_client_data() { + if(client_request_finished) + return Error::OK; + + char read_buffer[4096]; + const ssize_t num_bytes_read = read(client_fd, read_buffer, sizeof(read_buffer)); + if(num_bytes_read == -1) { + const int err = errno; + if(err == EAGAIN || err == EWOULDBLOCK) { + return Error::OK; + } else if(err == EPIPE || err == ECONNRESET) { + //fprintf(stderr, "YoutubeStaticMediaProxy::read_client_data: client disconnected\n"); + on_client_disconnect(); + return Error::ERROR; + } else { + perror("YoutubeStaticMediaProxy::read_client_data: read failed"); + return Error::ERROR; + } + } else if(num_bytes_read == 0) { + //fprintf(stderr, "YoutubeStaticMediaProxy::read_client_data: client disconnected\n"); + on_client_disconnect(); + return Error::ERROR; + } + + client_request_buffer.append(read_buffer, num_bytes_read); + const size_t header_end = client_request_buffer.find("\r\n\r\n"); + if(header_end != std::string::npos) { + client_request_buffer.erase(header_end + 4); + client_request_finished = true; + + int new_start_range = header_extract_start_range(client_request_buffer); + //fprintf(stderr, "got new range from client: %d\n", new_start_range); + if(new_start_range >= 0) { + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; + } + clear_download_state(); + update_download_program_status(false, new_start_range, true); + } + } else { + if(client_request_buffer.size() > MAX_BUFFER_SIZE) { + client_request_finished = true; + fprintf(stderr, "YoutubeStaticMediaProxy::read_client_data: buffer is full (malicious client?)\n"); + return Error::ERROR; + } + } + return Error::OK; + } + + void YoutubeStaticMediaProxy::clear_download_state() { + download_header.clear(); + download_header_finished = false; + download_header_sent = false; + download_header_remaining_sent = false; + download_header_written_offset = 0; + } + + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update_download_program_status(bool client_disconnected, int new_range_start, bool restart_download) { + int program_status = 0; + if(downloader_read_program.pid != -1) { + if(client_disconnected) { + wait_program(downloader_read_program.pid); + } else { + if(!wait_program_non_blocking(downloader_read_program.pid, &program_status)) + return Error::OK; + } + downloader_read_program.pid = -1; + } + + if(downloader_read_program.read_fd != -1) { + close(downloader_read_program.read_fd); + downloader_read_program.read_fd = -1; + } + + // TODO: Why is this not 0 when download finishes? + if(program_status != 0) { + //fprintf(stderr, "YoutubeStaticMediaProxy::update_download_program_status: download failed, exit status: %d\n", program_status); + if(client_fd != -1) { + write(client_fd, download_error_response_msg, sizeof(download_error_response_msg) - 1); + close(client_fd); + client_fd = -1; + client_request_buffer.clear(); + client_request_finished = false; + } + return Error::ERROR; + } + + if(client_disconnected) { + current_download_range = 0; + } else { + current_download_range += RANGE + 1; + } + + if(new_range_start != -1) { + download_range_start = new_range_start; + current_download_range = download_range_start; + } + + if(new_range_start == -1) { + download_header_finished = true; + download_header_sent = true; + download_header_remaining_sent = true; + } else { + clear_download_state(); + } + + if(client_disconnected) { + clear_download_state(); + return Error::ERROR; + } + + if(!restart_download) + return Error::OK; + + const bool start_download_success = start_download(youtube_media_url, downloader_read_program, current_download_range, new_range_start != -1); + if(!start_download_success) { + fprintf(stderr, "YoutubeStaticMediaProxy::update_download_program_status: failed to start download\n"); + if(client_fd != -1) { + write(client_fd, download_error_response_msg, sizeof(download_error_response_msg) - 1); + close(client_fd); + client_fd = -1; + client_request_buffer.clear(); + client_request_finished = false; + } + + clear_download_state(); + return Error::ERROR; + } + + return Error::OK; + } + + static void header_replace_content_length(std::string &header, size_t header_size, int new_content_length) { + if(new_content_length < 0) + new_content_length = 0; + + const char *content_length_p = strcasestr(header.c_str(), "content-length:"); + if(!content_length_p) + return; + + const size_t content_length_start = (content_length_p + 15) - header.c_str(); + if(content_length_start >= header_size) + return; + const size_t line_end = header.find("\r\n", content_length_start); + header.replace(content_length_start, line_end - content_length_start, std::to_string(new_content_length)); + } + + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::continue_send(const char *buffer_start, size_t total_bytes_to_write, int &buffer_offset) { + int num_bytes_to_write = total_bytes_to_write - buffer_offset; + //assert(num_bytes_to_write >= 0); + if(num_bytes_to_write < 0) num_bytes_to_write = 0; + if(num_bytes_to_write == 0) { + buffer_offset = 0; + return Error::OK; + } + + const ssize_t num_bytes_written = write(client_fd, buffer_start + buffer_offset, num_bytes_to_write); + if(num_bytes_written == -1) { + const int err = errno; + if(err == EAGAIN || err == EWOULDBLOCK) { + return Error::OK; + } else if(err == EPIPE || err == ECONNRESET) { + //fprintf(stderr, "YoutubeStaticMediaProxy::continue_send: client disconnected\n"); + on_client_disconnect(); + return Error::ERROR; + } else { + perror("YoutubeStaticMediaProxy::continue_send: write failed"); + return Error::ERROR; + } + } else if(num_bytes_written == 0) { + //fprintf(stderr, "YoutubeStaticMediaProxy::continue_send: client disconnected\n"); + on_client_disconnect(); + return Error::ERROR; + } else if(num_bytes_written != num_bytes_to_write) { + buffer_offset += num_bytes_written; + } else { + buffer_offset = 0; + } + return Error::OK; + } + + static bool http_is_redirect(const char *header, size_t size) { + const void *end_of_first_line_p = memmem(header, size, "\r\n", 2); + if(!end_of_first_line_p) + return false; + return memmem(header, (const char*)end_of_first_line_p - header, " 30", 3) != nullptr; + } + + static size_t find_start_of_first_non_redirect_header(const char *headers, size_t size, size_t &header_end) { + const char *start = headers; + while(size > 0) { + const void *end_of_header = memmem(headers, size, "\r\n\r\n", 4); + if(!end_of_header) + return std::string::npos; + + const size_t offset_to_end_of_headers = ((const char*)end_of_header + 4) - headers; + if(!http_is_redirect(headers, offset_to_end_of_headers)) { + header_end = (headers - start) + offset_to_end_of_headers; + return headers - start; + } + + headers += offset_to_end_of_headers; + size -= offset_to_end_of_headers; + } + return std::string::npos; + } + + YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::handle_download() { + // TODO: Maybe read even if write is being slow and failing? + if(download_read_buffer_offset == 0) { + downloader_num_read_bytes = read(downloader_read_program.read_fd, download_read_buffer, sizeof(download_read_buffer)); + if(downloader_num_read_bytes == -1) { + const int err = errno; + if(err == EAGAIN || err == EWOULDBLOCK) { + return Error::OK; + } else { + perror("YoutubeStaticMediaProxy::handle_download: curl read failed"); + return Error::ERROR; + } + } else if(downloader_num_read_bytes == 0) { + Error err = update_download_program_status(false, -1, true); + if(err != Error::OK) + return err; + } + } + + if(!download_header_finished) { + download_header.append(download_read_buffer, downloader_num_read_bytes); + size_t header_end = std::string::npos; + const size_t offset_to_start_of_header = find_start_of_first_non_redirect_header(download_header.c_str(), download_header.size(), header_end); + if(header_end != std::string::npos) { + download_header.erase(0, offset_to_start_of_header); + header_end -= offset_to_start_of_header; + + download_header_finished = true; + download_header_sent = false; + download_header_remaining_sent = false; + download_header_written_offset = 0; + download_header_offset_to_end_of_header = header_end; + download_read_buffer_offset = -1; + + header_replace_content_length(download_header, header_end, content_length - download_range_start); + } else { + if(download_header.size() > MAX_BUFFER_SIZE) { + fprintf(stderr, "YoutubeStaticMediaProxy::handle_download: buffer is full (malicious server?)\n"); + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; + } + clear_download_state(); + update_download_program_status(false, 0, false); + return Error::ERROR; + } + } + } + + if(download_header_finished && !download_header_sent) { + Error err = continue_send(download_header.data(), download_header_offset_to_end_of_header, download_header_written_offset); + if(err != Error::OK) + return err; + + if(download_header_written_offset == 0) { + download_header_sent = true; + download_header_written_offset = download_header_offset_to_end_of_header; + } + } + + if(download_header_finished && !download_header_remaining_sent) { + Error err = continue_send(download_header.data(), download_header.size(), download_header_written_offset); + if(err != Error::OK) + return err; + + if(download_header_written_offset == 0) { + download_header_remaining_sent = true; + download_read_buffer_offset = 0; + return Error::OK; + } + } + + if(download_header_remaining_sent) + return continue_send(download_read_buffer, downloader_num_read_bytes, download_read_buffer_offset); + + return Error::OK; + } + + int YoutubeStaticMediaProxy::accept_client() { + struct sockaddr_in client_addr; + socklen_t client_addr_len = sizeof(client_addr); + int new_client_fd = accept(socket_fd, (struct sockaddr*)&client_addr, &client_addr_len); + if(new_client_fd == -1) { + const int err = errno; + if(err == EAGAIN || err == EWOULDBLOCK) { + return -1; + } else { + perror("YoutubeStaticMediaProxy::accept_client accept failed"); + return -1; + } + } + + if(!set_non_blocking(new_client_fd)) { + perror("YoutubeStaticMediaProxy::accept_client: failed to set client socket non blocking mode"); + close(new_client_fd); + return -1; + } + + //fprintf(stderr, "YoutubeStaticMediaProxy::accept_client: client connected!\n"); + return new_client_fd; + } + + YoutubeLiveStreamMediaProxy::~YoutubeLiveStreamMediaProxy() { + stop(); + } + + bool YoutubeLiveStreamMediaProxy::start(const std::string &youtube_media_url, int) { + fd[0] = -1; + fd[1] = -1; + if(pipe(fd) == -1) { + perror("YoutubeLiveStreamMediaProxy::start: failed to open pipe"); + return false; + } + + //if(socketpair(AF_UNIX, SOCK_STREAM, 0, fd) == -1) { + // perror("YoutubeLiveStreamMediaProxy::start: failed to open pipe"); + // return false; + //} + + for(int i = 0; i < 2; ++i) { + if(!set_non_blocking(fd[i])) { + stop(); + return false; + } + } + + if(!start_download(youtube_media_url, downloader_read_program, 0, true, true)) { + stop(); + return false; + } + + this->youtube_media_url = youtube_media_url; + return true; + } + + void YoutubeLiveStreamMediaProxy::stop() { + for(int i = 0; i < 2; ++i) { + if(fd[i] != -1) { + close(fd[i]); + fd[i] = -1; + } + } + + if(downloader_read_program.read_fd != -1) { + close(downloader_read_program.read_fd); + downloader_read_program.read_fd = -1; + } + + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; + } + } + + YoutubeMediaProxy::Error YoutubeLiveStreamMediaProxy::update_download_program_status() { + int program_status = 0; + if(!wait_program_non_blocking(downloader_read_program.pid, &program_status)) + return Error::OK; + + downloader_read_program.pid = -1; + if(downloader_read_program.read_fd != -1) { + close(downloader_read_program.read_fd); + downloader_read_program.read_fd = -1; + } + + // TODO: Why is this not 0 when download finishes? + if(program_status != 0) { + //fprintf(stderr, "YoutubeLiveStreamMediaProxy::update_download_program_status: download failed, exit status: %d\n", program_status); + stop(); + return Error::ERROR; + } + + ++livestream_sequence_num; + const bool start_download_success = start_download(youtube_media_url, downloader_read_program, 0, false, true, livestream_sequence_num); + if(!start_download_success) { + fprintf(stderr, "YoutubeLiveStreamMediaProxy::update_download_program_status: failed to start download\n"); + stop(); + return Error::ERROR; + } + + return Error::OK; + } + + YoutubeMediaProxy::Error YoutubeLiveStreamMediaProxy::continue_send(const char *buffer_start, size_t total_bytes_to_write, int &buffer_offset) { + int num_bytes_to_write = total_bytes_to_write - buffer_offset; + //assert(num_bytes_to_write >= 0); + if(num_bytes_to_write < 0) num_bytes_to_write = 0; + if(num_bytes_to_write == 0) { + buffer_offset = 0; + return Error::OK; + } + + const ssize_t num_bytes_written = write(fd[1], buffer_start + buffer_offset, num_bytes_to_write); + if(num_bytes_written == -1) { + const int err = errno; + if(err == EAGAIN || err == EWOULDBLOCK) { + return Error::OK; + } else if(err == EPIPE || err == ECONNRESET) { + //fprintf(stderr, "YoutubeLiveStreamMediaProxy::continue_send: client disconnected\n"); + stop(); + return Error::ERROR; + } else { + perror("YoutubeLiveStreamMediaProxy::continue_send: write failed"); + return Error::ERROR; + } + } else if(num_bytes_written == 0) { + //fprintf(stderr, "YoutubeLiveStreamMediaProxy::continue_send: client disconnected\n"); + stop(); + return Error::ERROR; + } else if(num_bytes_written != num_bytes_to_write) { + buffer_offset += num_bytes_written; + } else { + buffer_offset = 0; + } + return Error::OK; + } + + YoutubeMediaProxy::Error YoutubeLiveStreamMediaProxy::update() { + if(fd[1] == -1 || downloader_read_program.read_fd == -1) + return Error::OK; + + if(download_read_buffer_offset == 0) { + downloader_num_read_bytes = read(downloader_read_program.read_fd, download_read_buffer, sizeof(download_read_buffer)); + if(downloader_num_read_bytes == -1) { + const int err = errno; + if(err == EAGAIN || err == EWOULDBLOCK) { + return Error::OK; + } else { + perror("YoutubeLiveStreamMediaProxy::update: curl read failed"); + return Error::ERROR; + } + } else if(downloader_num_read_bytes == 0) { + Error err = update_download_program_status(); + if(err != Error::OK) + return err; + } + } + + if(!download_header_finished) { + download_header.append(download_read_buffer, downloader_num_read_bytes); + size_t header_end = std::string::npos; + const size_t offset_to_start_of_header = find_start_of_first_non_redirect_header(download_header.c_str(), download_header.size(), header_end); + if(header_end != std::string::npos) { + download_header.erase(0, offset_to_start_of_header); + header_end -= offset_to_start_of_header; + + download_header_finished = true; + download_header_remaining_sent = false; + download_header_written_offset = header_end; + download_read_buffer_offset = -1; + fprintf(stderr, "header: |%.*s|\n", download_header_written_offset, download_header.c_str()); + + if(livestream_sequence_num == -1) { + // TODO: What about |header_end|? + std::string sequence_num = header_extract_value(download_header, "x-sequence-num"); + fprintf(stderr, "server sequence num: |%s|\n", sequence_num.c_str()); + if(sequence_num.empty()) + fprintf(stderr, "YoutubeLiveStreamMediaProxy::handle_download: missing sequence num from server\n"); + else + livestream_sequence_num = strtol(sequence_num.c_str(), nullptr, 10); + } + } else { + if(download_header.size() > MAX_BUFFER_SIZE) { + fprintf(stderr, "YoutubeLiveStreamMediaProxy::handle_download: buffer is full (malicious server?)\n"); + if(downloader_read_program.pid != -1) { + kill(downloader_read_program.pid, SIGTERM); + wait_program(downloader_read_program.pid); + downloader_read_program.pid = -1; + } + download_header_finished = true; + return Error::ERROR; + } + } + } + + if(download_header_finished && !download_header_remaining_sent) { + Error err = continue_send(download_header.data(), download_header.size(), download_header_written_offset); + if(err != Error::OK) + return err; + + if(download_header_written_offset == 0) { + download_header_remaining_sent = true; + download_read_buffer_offset = 0; + return Error::OK; + } + } + + if(download_header_remaining_sent) + return continue_send(download_read_buffer, downloader_num_read_bytes, download_read_buffer_offset); + + return Error::OK; + } + + bool YoutubeLiveStreamMediaProxy::get_address(std::string &address) { + if(fd[0] == -1) + return false; + + address = "fd://" + std::to_string(fd[0]); + return true; + } +} \ No newline at end of file -- cgit v1.2.3