From 80696a506afcee0cca48c22448adacb4aea6eece Mon Sep 17 00:00:00 2001 From: dec05eba Date: Fri, 11 Mar 2022 07:07:04 +0100 Subject: youtube: use mpv stream_cb instead of proxy server --- src/Downloader.cpp | 279 +++++++++---- src/QuickMedia.cpp | 81 +--- src/Storage.cpp | 4 +- src/plugins/FileManager.cpp | 2 +- src/plugins/youtube/YoutubeMediaProxy.cpp | 623 ------------------------------ 5 files changed, 214 insertions(+), 775 deletions(-) delete mode 100644 src/plugins/youtube/YoutubeMediaProxy.cpp (limited to 'src') diff --git a/src/Downloader.cpp b/src/Downloader.cpp index 89d83cb..24f670d 100644 --- a/src/Downloader.cpp +++ b/src/Downloader.cpp @@ -4,6 +4,8 @@ #include "../include/Notification.hpp" #include #include +#include +#include namespace QuickMedia { static bool youtube_url_is_live_stream(const std::string &url) { @@ -112,9 +114,9 @@ namespace QuickMedia { } std::lock_guard lock(content_length_mutex); - size_t output_file_size = 0; + int64_t output_file_size = 0; file_get_size(output_filepath_tmp, &output_file_size); - size_t downloaded_size = std::min((int64_t)output_file_size, content_length); + int64_t downloaded_size = std::min((int64_t)output_file_size, content_length); if(content_length == -1) { progress_text = std::to_string(output_file_size / 1024) + "/Unknown"; @@ -127,7 +129,7 @@ namespace QuickMedia { } // TODO: Take into consideration time overflow? - size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check); + int64_t downloaded_diff = std::max((int64_t)0, downloaded_size - downloaded_since_last_check); download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; downloaded_since_last_check = downloaded_size; @@ -279,79 +281,178 @@ namespace QuickMedia { return download_speed_text; } + static int64_t seek_fn(YoutubeReadProgram *program, int64_t offset) { + if(program->read_program.pid != -1) { + kill(program->read_program.pid, SIGTERM); + int status; + waitpid(program->read_program.pid, &status, 0); + program->read_program.pid = -1; + } + + if(program->read_program.read_fd != -1) { + close(program->read_program.read_fd); + program->read_program.read_fd = -1; + } + + program->offset = offset; + + char range[64]; + snprintf(range, sizeof(range), "%lld-%lld", (long long)program->offset, (long long)program->offset + 5242870LL); + const char *args[] = { "curl", + "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", + "--no-buffer", + "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "-g", "-s", "-L", "-f", + "-r", range, "--", program->url, nullptr }; + int res = exec_program_pipe(args, &program->read_program); + if(res != 0) + return -1; + + int flags = fcntl(program->read_program.read_fd, F_GETFL, 0); + if(flags != -1) + fcntl(program->read_program.read_fd, F_SETFL, flags | O_NONBLOCK); + + return offset; + } + + static int64_t read_fn(YoutubeReadProgram *program, char *buf, uint64_t nbytes) { + ssize_t bytes_read = read(program->read_program.read_fd, buf, nbytes); + if(bytes_read > 0) { + program->offset += bytes_read; + program->bytes_downloaded += bytes_read; + } else if(bytes_read == 0) { + // End of current range, progress to next range + bytes_read = seek_fn(program, program->offset); + if(bytes_read >= 0) { + bytes_read = read(program->read_program.read_fd, buf, nbytes); + if(bytes_read > 0) { + program->offset += bytes_read; + program->bytes_downloaded += bytes_read; + } + } + } else if(bytes_read < 0) { + if(errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) + return 0; + } + return bytes_read; + } + + static void close_fn(YoutubeReadProgram *program) { + if(program->read_program.pid != -1) { + kill(program->read_program.pid, SIGTERM); + int status; + waitpid(program->read_program.pid, &status, 0); + program->read_program.pid = -1; + } + + if(program->read_program.read_fd != -1) { + close(program->read_program.read_fd); + program->read_program.read_fd = -1; + } + + if(program->url) { + free(program->url); + program->url = nullptr; + } + } + + static YoutubeReadProgram* open_fn(const char *uri) { + YoutubeReadProgram *program = new YoutubeReadProgram(); + program->read_program.read_fd = -1; + program->read_program.pid = -1; + program->url = strdup(uri); + program->offset = 0; + if(seek_fn(program, program->offset) < 0) { + delete program; + return nullptr; + } + return program; + } + YoutubeDownloader::YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath) : Downloader("", output_filepath), video_metadata(video_metadata), audio_metadata(audio_metadata) { for(int i = 0; i < 2; ++i) - downloaders[i] = nullptr; + program[i] = nullptr; + } + + YoutubeDownloader::~YoutubeDownloader() { + for(int i = 0; i < 2; ++i) { + if(program[i]) { + close_fn(program[i]); + delete program[i]; + program[i] = nullptr; + } + } } bool YoutubeDownloader::start() { - struct MediaProxyMetadata { - std::unique_ptr *media_proxy; + struct MediaMetadataInfo { MediaMetadata *media_metadata; - std::unique_ptr *downloader; std::string *output_filepath; bool tmp_audio_file; }; const bool has_video = !video_metadata.url.empty(); - MediaProxyMetadata media_proxies[2] = { - { &youtube_video_media_proxy, &video_metadata, &downloaders[0], &video_output_filepath, false }, - { &youtube_audio_media_proxy, &audio_metadata, &downloaders[1], &audio_output_filepath, has_video } + MediaMetadataInfo media_info[2] = { + { &video_metadata, &video_output_filepath, false }, + { &audio_metadata, &audio_output_filepath, has_video } }; - int num_proxied_media = 0; + int num_streams = 0; for(int i = 0; i < 2; ++i) { - media_proxies[i].output_filepath->clear(); - if(media_proxies[i].media_metadata->url.empty() || youtube_url_is_live_stream(media_proxies[i].media_metadata->url)) + media_info[i].output_filepath->clear(); + if(media_info[i].media_metadata->url.empty() || youtube_url_is_live_stream(media_info[i].media_metadata->url)) continue; - *media_proxies[i].media_proxy = std::make_unique(); - if(!(*media_proxies[i].media_proxy)->start(media_proxies[i].media_metadata->url, media_proxies[i].media_metadata->content_length)) { - fprintf(stderr, "Failed to load start youtube media proxy\n"); - return false; - } + if(media_info[i].tmp_audio_file) + *media_info[i].output_filepath = output_filepath + ".audio"; + else + *media_info[i].output_filepath = output_filepath; - std::string media_proxy_addr; - if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) { - fprintf(stderr, "Failed to load start youtube media proxy\n"); + program[i] = open_fn(media_info[i].media_metadata->url.c_str()); + if(!program[i]) { + fprintf(stderr, "Error: open failed for %s\n", media_info[i].media_metadata->url.c_str()); return false; } + program[i]->content_length = media_info[i].media_metadata->content_length; + program[i]->output_filepath = *media_info[i].output_filepath; + program[i]->output_filepath_tmp = program[i]->output_filepath + ".tmp"; - if(media_proxies[i].tmp_audio_file) - *media_proxies[i].output_filepath = output_filepath + ".audio"; - else - *media_proxies[i].output_filepath = output_filepath; + program[i]->progress_text = "0 bytes/Unknown"; + program[i]->download_speed_text = "Unknown/s"; - remove(media_proxies[i].output_filepath->c_str()); - fprintf(stderr, "Downloading %s to %s\n", media_proxy_addr.c_str(), media_proxies[i].output_filepath->c_str()); - *media_proxies[i].downloader = std::make_unique(media_proxy_addr, *media_proxies[i].output_filepath, media_proxies[i].media_metadata->content_length); - *media_proxies[i].output_filepath = (*media_proxies[i].downloader)->get_output_filepath(); - if(!(*media_proxies[i].downloader)->start()) - return false; - ++num_proxied_media; - } + remove(program[i]->output_filepath.c_str()); + remove(program[i]->output_filepath_tmp.c_str()); - if(num_proxied_media == 0) { - return false; - } + downloader_tasks[i] = AsyncTask([this, i]() { + FILE *f = fopen(program[i]->output_filepath_tmp.c_str(), "wb"); + if(!f) + return false; + + char buffer[8192]; + while(!program_is_dead_in_current_thread()) { + if(program[i]->offset >= program[i]->content_length) + break; - downloader_task = AsyncTask([this]() { - // TODO: Poll instead of sleep - while(!program_is_dead_in_current_thread()) { - if(youtube_video_media_proxy) - youtube_video_media_proxy->update(); + int64_t bytes_read = read_fn(program[i], buffer, sizeof(buffer)); + if(bytes_read > 0) + fwrite(buffer, 1, bytes_read, f); + } - if(youtube_audio_media_proxy) - youtube_audio_media_proxy->update(); + fclose(f); + close_fn(program[i]); + return true; + }); - usleep(1000); - } - }); + ++num_streams; + } + + if(num_streams == 0) + return false; return true; } @@ -362,27 +463,22 @@ namespace QuickMedia { } bool YoutubeDownloader::stop(bool download_completed) { + bool success = true; for(int i = 0; i < 2; ++i) { - if(downloaders[i]) { - downloaders[i]->stop(download_completed); - downloaders[i].reset(); + if(program[i]) { + downloader_tasks[i].cancel(); + if(downloader_tasks[i].ready() && !downloader_tasks[i].get()) + success = false; + if(rename_atomic(program[i]->output_filepath_tmp.c_str(), program[i]->output_filepath.c_str()) != 0) + success = false; + close_fn(program[i]); } } - if(youtube_video_media_proxy) - youtube_video_media_proxy->stop(); - - if(youtube_audio_media_proxy) - youtube_audio_media_proxy->stop(); - - downloader_task.cancel(); - youtube_video_media_proxy.reset(); - youtube_audio_media_proxy.reset(); - - if(!download_completed) + if(!download_completed || !success) return false; - bool success = true; + success = true; if(!video_metadata.url.empty() && !audio_metadata.url.empty()) { std::string tmp_file = video_output_filepath + ".tmp.mkv"; if(ffmpeg_merge_audio_and_video(video_output_filepath, audio_output_filepath, tmp_file)) { @@ -406,9 +502,9 @@ namespace QuickMedia { int num_downloaders = 0; for(int i = 0; i < 2; ++i) { - if(downloaders[i]) { + if(program[i]) { ++num_downloaders; - DownloadUpdateStatus update_status = downloaders[i]->update(); + DownloadUpdateStatus update_status = update(i); switch(update_status) { case DownloadUpdateStatus::DOWNLOADING: ++num_downloading; @@ -432,33 +528,68 @@ namespace QuickMedia { } float YoutubeDownloader::get_progress() { - return get_min_progress_downloader()->get_progress(); + YoutubeReadProgram *min_program = get_min_progress_downloader(); + return min_program->progress; } std::string YoutubeDownloader::get_progress_text() { - return get_min_progress_downloader()->get_progress_text(); + YoutubeReadProgram *min_program = get_min_progress_downloader(); + return min_program->progress_text; } std::string YoutubeDownloader::get_download_speed_text() { - return get_min_progress_downloader()->get_download_speed_text(); + YoutubeReadProgram *min_program = get_min_progress_downloader(); + return min_program->download_speed_text; } - CurlDownloader* YoutubeDownloader::get_min_progress_downloader() { + YoutubeReadProgram* YoutubeDownloader::get_min_progress_downloader() { int min_index = -1; float progress = 999999.0f; for(int i = 0; i < 2; ++i) { - if(downloaders[i]) { - float item_progress = downloaders[i]->get_progress(); - if(item_progress < progress) { + if(program[i]) { + if(program[i]->progress < progress) { min_index = i; - progress = item_progress; + progress = program[i]->progress; } } } - //assert(min_index != -1); - // Shouldn't happen if(min_index == -1) min_index = 0; - return downloaders[min_index].get(); + return program[min_index]; + } + + DownloadUpdateStatus YoutubeDownloader::update(size_t program_index) { + bool finished = false; + YoutubeReadProgram *youtube_program = program[program_index]; + auto &downloader_task = downloader_tasks[program_index]; + + if(youtube_program->finished) + return DownloadUpdateStatus::FINISHED; + + if(downloader_task.ready()) { + if(downloader_task.get()) { + finished = true; + youtube_program->finished = true; + } else { + return DownloadUpdateStatus::ERROR; + } + } + + int64_t output_file_size = 0; + file_get_size(youtube_program->output_filepath_tmp, &output_file_size); + int64_t downloaded_size = std::min((int64_t)output_file_size, youtube_program->content_length); + + int64_t percentage = 0; + if(output_file_size > 0) + percentage = (double)downloaded_size / (double)youtube_program->content_length * 100.0; + youtube_program->progress = (double)percentage / 100.0; + youtube_program->progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(youtube_program->content_length) + " (" + std::to_string(percentage) + "%)"; + + // TODO: Take into consideration time overflow? + int64_t downloaded_diff = std::max((int64_t)0, downloaded_size - youtube_program->downloaded_since_last_check); + youtube_program->download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s"; + youtube_program->downloaded_since_last_check = downloaded_size; + + return finished ? DownloadUpdateStatus::FINISHED : DownloadUpdateStatus::DOWNLOADING; } } diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index 49bfd36..4242fa1 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -40,7 +40,6 @@ #include "../include/Storage.hpp" #include "../include/AsyncImageLoader.hpp" #include -#include "../plugins/youtube/YoutubeMediaProxy.hpp" #include "../include/gui/Button.hpp" #include "../external/hash-library/sha256.h" @@ -3090,9 +3089,6 @@ namespace QuickMedia { update_window_focus = true; }; - std::unique_ptr youtube_video_media_proxy; - std::unique_ptr youtube_audio_media_proxy; - AsyncTask youtube_downloader_task; int64_t youtube_video_content_length = 0; int64_t youtube_audio_content_length = 0; @@ -3211,65 +3207,10 @@ namespace QuickMedia { std::string v = video_url; std::string a = audio_url; if(is_youtube) { - if(youtube_video_media_proxy) - youtube_video_media_proxy->stop(); - - if(youtube_audio_media_proxy) - youtube_audio_media_proxy->stop(); - - youtube_downloader_task.cancel(); - youtube_video_media_proxy.reset(); - youtube_audio_media_proxy.reset(); - - struct MediaProxyMetadata { - std::unique_ptr *media_proxy; - std::string *url; - int64_t content_length; - }; - - MediaProxyMetadata media_proxies[2] = { - { &youtube_video_media_proxy, &v, youtube_video_content_length }, - { &youtube_audio_media_proxy, &a, youtube_audio_content_length } - }; - int num_proxied_media = 0; - for(int i = 0; i < 2; ++i) { - if(media_proxies[i].url->empty() || youtube_url_is_live_stream(*media_proxies[i].url)) - continue; - - *media_proxies[i].media_proxy = std::make_unique(); - if(!(*media_proxies[i].media_proxy)->start(*media_proxies[i].url, media_proxies[i].content_length)) { - show_notification("QuickMedia", "Failed to load start youtube media proxy", Urgency::CRITICAL); - current_page = previous_page; - go_to_previous_page = true; - return; - } - - std::string media_proxy_addr; - if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) { - show_notification("QuickMedia", "Failed to load start youtube media proxy", Urgency::CRITICAL); - current_page = previous_page; - go_to_previous_page = true; - return; - } - - *media_proxies[i].url = std::move(media_proxy_addr); - ++num_proxied_media; - } - - if(num_proxied_media > 0) { - youtube_downloader_task = AsyncTask([&youtube_video_media_proxy, &youtube_audio_media_proxy]() { - // TODO: Poll instead of sleep - while(!program_is_dead_in_current_thread()) { - if(youtube_video_media_proxy) - youtube_video_media_proxy->update(); - - if(youtube_audio_media_proxy) - youtube_audio_media_proxy->update(); - - usleep(1000); - } - }); - } + if(!v.empty() && !youtube_url_is_live_stream(v)) + v = "qm-yt://" + v; + if(!a.empty() && !youtube_url_is_live_stream(a)) + a = "qm-yt://" + a; } VideoPlayer::StartupArgs startup_args; @@ -3499,16 +3440,6 @@ namespace QuickMedia { video_player->get_time_in_file(&resume_start_time); video_player.reset(); page_changed = true; - - if(youtube_video_media_proxy) - youtube_video_media_proxy->stop(); - - if(youtube_audio_media_proxy) - youtube_audio_media_proxy->stop(); - - youtube_downloader_task.cancel(); - youtube_video_media_proxy.reset(); - youtube_audio_media_proxy.reset(); } }); @@ -3866,7 +3797,7 @@ namespace QuickMedia { return true; } } else { - size_t file_size = 0; + int64_t file_size = 0; if(download_to_file(url, image_filepath_tmp.data, extra_args, true, cloudflare_bypass) != DownloadResult::OK || (is_manganelo && file_get_size(image_filepath_tmp, &file_size) == 0 && file_size < 255)) { if(!image_download_cancel) show_notification("QuickMedia", "Failed to download image: " + url, Urgency::CRITICAL); return true; @@ -7763,7 +7694,7 @@ namespace QuickMedia { } const float progress_diff = progress - ui_progress; - const float progress_move = frame_timer.get_elapsed_time_seconds() * 500.0f * std::abs(progress_diff); + const float progress_move = frame_timer.get_elapsed_time_seconds() * 2500.0f * std::abs(progress_diff); if(std::abs(progress_diff) < progress_move) { ui_progress = progress; } else { diff --git a/src/Storage.cpp b/src/Storage.cpp index effa70b..7554b73 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -190,7 +190,7 @@ namespace QuickMedia { return 0; } - int file_get_size(const Path &path, size_t *size) { + int file_get_size(const Path &path, int64_t *size) { struct stat file_stat; memset(&file_stat, 0, sizeof(file_stat)); int ret; @@ -399,7 +399,7 @@ namespace QuickMedia { return false; } - std::string file_size_to_human_readable_string(size_t bytes) { + std::string file_size_to_human_readable_string(int64_t bytes) { double kb = (double)bytes / 1024.0; double mb = (double)bytes / 1024.0 / 1024.0; double gb = (double)bytes / 1024.0 / 1024.0 / 1024.0; diff --git a/src/plugins/FileManager.cpp b/src/plugins/FileManager.cpp index eb4a3ef..5fd6670 100644 --- a/src/plugins/FileManager.cpp +++ b/src/plugins/FileManager.cpp @@ -128,7 +128,7 @@ namespace QuickMedia { strftime(time_str, sizeof(time_str) - 1, "%Y %b %d, %a %H:%M", &last_modified_tm); std::error_code file_size_err; - size_t file_size = p.first.file_size(file_size_err); + int64_t file_size = p.first.file_size(file_size_err); if(file_size_err) file_size = 0; diff --git a/src/plugins/youtube/YoutubeMediaProxy.cpp b/src/plugins/youtube/YoutubeMediaProxy.cpp deleted file mode 100644 index e51e3d9..0000000 --- a/src/plugins/youtube/YoutubeMediaProxy.cpp +++ /dev/null @@ -1,623 +0,0 @@ -#include "../../../plugins/youtube/YoutubeMediaProxy.hpp" -#include "../../../include/NetUtils.hpp" -#include "../../../include/Utils.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -// TODO: What if the client sends a new header without reconnecting? is that even allowed by http standard? -// TODO: Detect when download has finished (and close connection). - -static ssize_t read_eintr(int fd, void *buffer, size_t size) { - while(true) { - ssize_t bytes_read = read(fd, buffer, size); - if(bytes_read == -1) { - if(errno != EINTR) - return -1; - } else { - return bytes_read; - } - } -} - -static ssize_t write_all(int fd, const void *buffer, size_t size) { - ssize_t bytes_written = 0; - while((size_t)bytes_written < size) { - ssize_t written = write(fd, (char*)buffer + bytes_written, size - bytes_written); - if(written == -1) { - if(errno != EINTR) - return -1; - } else { - bytes_written += written; - } - } - return bytes_written; -} - -static ssize_t write_all_blocking(int fd, const void *buffer, size_t size) { - ssize_t bytes_written = 0; - while((size_t)bytes_written < size) { - ssize_t written = write(fd, (char*)buffer + bytes_written, size - bytes_written); - if(written == -1) { - if(errno != EINTR && errno != EWOULDBLOCK) - return -1; - } else { - bytes_written += written; - } - } - return bytes_written; -} - -namespace QuickMedia { - static const int MAX_BUFFER_SIZE = 65536; - static const int64_t RANGE = 5242870; - static const char download_error_response_msg[] = - "HTTP/1.1 500 Internal Server Error\r\n" - "Content-Length: 0\r\n\r\n"; - static const char download_finished_response_msg[] = - "HTTP/1.1 200 OK\r\n" - "Content-Length: 0\r\n\r\n"; - - static bool set_non_blocking(int fd) { - const int flags = fcntl(fd, F_GETFL, 0); - if(fd == -1 || fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) - return false; - return true; - } - - // TODO: Restrict range end to remote file size (content-length which we have saved). - // TODO: Check if custom youtube redirect code is needed - bool YoutubeMediaProxy::start_download(const std::string &media_url, ReadProgram &read_program, int64_t range_start, int64_t end_range, bool include_header) { - std::string r = std::to_string(range_start) + "-" + std::to_string(end_range); - - std::string url = media_url;; - std::vector args = { "curl", - "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", - "--no-buffer", - "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", - "-g", "-s", "-L", "-f", "-r", r.c_str() }; - - if(include_header) - args.push_back("-i"); - - //fprintf(stderr, "url: %s\n", url.c_str()); - - args.insert(args.end(), { "--", url.c_str(), nullptr }); - - if(exec_program_pipe(args.data(), &read_program) != 0) - return false; - - if(!set_non_blocking(read_program.read_fd)) { - perror("start_download: failed to set curl pipe non blocking mode"); - close(read_program.read_fd); - kill(read_program.pid, SIGTERM); - read_program.read_fd = -1; - read_program.pid = -1; - return false; - } - - return true; - } - - YoutubeStaticMediaProxy::YoutubeStaticMediaProxy() { - - } - - YoutubeStaticMediaProxy::~YoutubeStaticMediaProxy() { - stop(); - } - - bool YoutubeStaticMediaProxy::start(const std::string &youtube_media_url, int64_t content_length) { - if(socket_fd != -1) - return false; - - socket_fd = socket(AF_INET, SOCK_STREAM, 0); - if(socket_fd == -1) { - perror("YoutubeStaticMediaProxy::start: socket failed"); - return false; - } - - socklen_t response_sock_addr_len; - struct sockaddr_in server_addr; - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - server_addr.sin_port = htons(0); - - next_range_length = RANGE; - - if(bind(socket_fd, (struct sockaddr*)&server_addr, sizeof(server_addr)) == -1) { - perror("YoutubeStaticMediaProxy::start: bind failed"); - goto err; - } - - if(listen(socket_fd, 2) == -1) { - perror("YoutubeStaticMediaProxy::start: listen failed"); - goto err; - } - - if(!set_non_blocking(socket_fd)) { - perror("YoutubeStaticMediaProxy::start: failed to set socket non blocking mode"); - goto err; - } - - struct sockaddr_in response_sock_addr; - response_sock_addr_len = sizeof(response_sock_addr); - if(getsockname(socket_fd, (struct sockaddr*)&response_sock_addr, &response_sock_addr_len) == -1) { - perror("YoutubeStaticMediaProxy::start: getsockname failed"); - goto err; - } - - port = ntohs(response_sock_addr.sin_port); - this->youtube_media_url = youtube_media_url; - this->content_length = content_length; - return true; - - err: - if(downloader_read_program.read_fd != -1) - close(downloader_read_program.read_fd); - if(downloader_read_program.pid != -1) - kill(downloader_read_program.pid, SIGTERM); - close(socket_fd); - socket_fd = -1; - return false; - } - - void YoutubeStaticMediaProxy::stop() { - if(downloader_read_program.read_fd != -1) { - close(downloader_read_program.read_fd); - downloader_read_program.read_fd = -1; - } - - if(downloader_read_program.pid != -1) { - kill(downloader_read_program.pid, SIGTERM); - wait_program(downloader_read_program.pid); - downloader_read_program.pid = -1; - } - - if(client_fd != -1) { - close(client_fd); - client_fd = -1; - } - - if(socket_fd != -1) { - close(socket_fd); - socket_fd = -1; - } - - clear_download_state(); - client_request_buffer.clear(); - client_request_finished = false; - } - - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update() { - if(socket_fd == -1) - return Error::OK; - - if(client_fd == -1) { - client_fd = accept_client(); - if(client_fd == -1) - return Error::OK; - } else { - const int new_client_fd = accept_client(); - if(new_client_fd != -1) { - on_client_disconnect(); - client_fd = new_client_fd; - } - } - - Error err = read_client_data(); - if(err != Error::OK || !client_request_finished) - return err; - if(downloader_read_program.pid == -1) - return Error::ERROR; - return handle_download(); - } - - bool YoutubeStaticMediaProxy::get_address(std::string &address) { - if(socket_fd == -1) - return false; - - address = "http://127.0.0.1:" + std::to_string(port); - return true; - } - - void YoutubeStaticMediaProxy::on_client_disconnect() { - client_request_buffer.clear(); - client_request_finished = false; - download_started = false; - - if(client_fd != -1) { - close(client_fd); - client_fd = -1; - } - - if(downloader_read_program.pid != -1) - kill(downloader_read_program.pid, SIGTERM); - - update_download_program_status(true); - } - - // Returns 0 if start range is not found - static int64_t header_extract_start_range(const std::string &header) { - std::string range = header_extract_value(header, "range"); - if(range.empty()) - return 0; - - int64_t start_range = 0; - if(sscanf(range.c_str(), " bytes=%" PRId64, &start_range) != 1) - return 0; - - return start_range; - } - - static int64_t header_extract_content_length(const std::string &header) { - std::string content_length_str = header_extract_value(header, "content-length"); - if(content_length_str.empty()) - return 0; - - errno = 0; - char *endptr; - int64_t content_length = strtoll(content_length_str.c_str(), &endptr, 10); - if(endptr == content_length_str.c_str() || errno != 0) - return 0; - - return content_length; - } - - // TODO: What about hls (live streams)? need to test with that. There may not be a need to use YoutubeMediaProxy for that case (in that case document it). - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::read_client_data() { - if(client_request_finished) - return Error::OK; - - char read_buffer[4096]; - const ssize_t num_bytes_read = read_eintr(client_fd, read_buffer, sizeof(read_buffer)); - if(num_bytes_read == -1) { - const int err = errno; - if(err == EAGAIN || err == EWOULDBLOCK) { - return Error::OK; - } else if(err == EPIPE || err == ECONNRESET) { - //fprintf(stderr, "YoutubeStaticMediaProxy::read_client_data: client disconnected\n"); - on_client_disconnect(); - return Error::ERROR; - } else { - perror("YoutubeStaticMediaProxy::read_client_data: read failed"); - return Error::ERROR; - } - } else if(num_bytes_read == 0) { - //fprintf(stderr, "YoutubeStaticMediaProxy::read_client_data: client disconnected\n"); - on_client_disconnect(); - return Error::ERROR; - } - - client_request_buffer.append(read_buffer, num_bytes_read); - const size_t header_end = client_request_buffer.find("\r\n\r\n"); - if(header_end != std::string::npos) { - client_request_buffer.erase(header_end + 4); - client_request_finished = true; - download_started = false; - download_read_buffer_offset = 0; - - const int64_t new_start_range = header_extract_start_range(client_request_buffer); - if(downloader_read_program.pid != -1) { - kill(downloader_read_program.pid, SIGTERM); - wait_program(downloader_read_program.pid); - downloader_read_program.pid = -1; - } - clear_download_state(); - return update_download_program_status(false, new_start_range, true); - } else { - if(client_request_buffer.size() > MAX_BUFFER_SIZE) { - client_request_finished = true; - fprintf(stderr, "YoutubeStaticMediaProxy::read_client_data: buffer is full (malicious client?)\n"); - return Error::ERROR; - } - } - return Error::OK; - } - - void YoutubeStaticMediaProxy::clear_download_state() { - download_header.clear(); - download_header_finished = false; - download_header_sent = false; - download_header_remaining_sent = false; - download_header_written_offset = 0; - download_read_buffer_offset = 0; - next_range_length = RANGE; - end_of_file = false; - } - - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::update_download_program_status(bool client_disconnected, int64_t new_range_start, bool restart_download) { - int program_status = 0; - if(downloader_read_program.pid != -1) { - if(client_disconnected) { - wait_program(downloader_read_program.pid); - } else { - if(wait_program_non_blocking(downloader_read_program.pid, &program_status) == 0 && program_status == 0) - return Error::OK; - } - downloader_read_program.pid = -1; - } - - if(downloader_read_program.read_fd != -1) { - close(downloader_read_program.read_fd); - downloader_read_program.read_fd = -1; - } - - if(program_status != 0) { - //fprintf(stderr, "YoutubeStaticMediaProxy::update_download_program_status: download failed, exit status: %d\n", program_status); - if(client_fd != -1) { - write_all_blocking(client_fd, download_finished_response_msg, sizeof(download_finished_response_msg) - 1); - close(client_fd); - client_fd = -1; - client_request_buffer.clear(); - client_request_finished = false; - } - return Error::ERROR; - } - - if(client_disconnected) { - current_download_range = 0; - } else { - current_download_range += next_range_length; - } - - if(new_range_start != -1) { - download_range_start = new_range_start; - current_download_range = download_range_start; - } - - if(new_range_start == -1) { - download_header_finished = true; - download_header_sent = true; - download_header_remaining_sent = true; - } else { - clear_download_state(); - } - - if(client_disconnected) { - clear_download_state(); - return Error::ERROR; - } - - if(!restart_download) - return Error::OK; - - if(new_range_start == -1) { - download_header_finished = true; - download_header_sent = true; - download_header_remaining_sent = true; - } - - int64_t end_range = current_download_range + RANGE; - if(end_range > content_length) { - end_range = content_length; - end_of_file = true; - } - - const bool start_download_success = start_download(youtube_media_url, downloader_read_program, current_download_range, end_range, new_range_start != -1); - if(!start_download_success) { - fprintf(stderr, "YoutubeStaticMediaProxy::update_download_program_status: failed to start download\n"); - if(client_fd != -1) { - write_all_blocking(client_fd, download_error_response_msg, sizeof(download_error_response_msg) - 1); - close(client_fd); - client_fd = -1; - client_request_buffer.clear(); - client_request_finished = false; - } - - clear_download_state(); - return Error::ERROR; - } - - return Error::OK; - } - - static void header_replace_content_length(std::string &header, size_t header_size, int64_t new_content_length) { - if(new_content_length < 0) - new_content_length = 0; - - const char *content_length_p = strcasestr(header.c_str(), "content-length:"); - if(!content_length_p) - return; - - const size_t content_length_start = (content_length_p + 15) - header.c_str(); - if(content_length_start >= header_size) - return; - const size_t line_end = header.find("\r\n", content_length_start); - header.replace(content_length_start, line_end - content_length_start, std::to_string(new_content_length)); - } - - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::continue_send(const char *buffer_start, size_t total_bytes_to_write, int &buffer_offset) { - int num_bytes_to_write = total_bytes_to_write - buffer_offset; - //assert(num_bytes_to_write >= 0); - if(num_bytes_to_write < 0) num_bytes_to_write = 0; - if(num_bytes_to_write == 0) { - buffer_offset = 0; - return Error::OK; - } - - const ssize_t num_bytes_written = write_all(client_fd, buffer_start + buffer_offset, num_bytes_to_write); - if(num_bytes_written == -1) { - const int err = errno; - if(err == EAGAIN || err == EWOULDBLOCK) { - return Error::OK; - } else if(err == EPIPE || err == ECONNRESET) { - //fprintf(stderr, "YoutubeStaticMediaProxy::continue_send: client disconnected\n"); - on_client_disconnect(); - return Error::ERROR; - } else { - perror("YoutubeStaticMediaProxy::continue_send: write failed"); - return Error::ERROR; - } - } else if(num_bytes_written == 0) { - //fprintf(stderr, "YoutubeStaticMediaProxy::continue_send: client disconnected\n"); - on_client_disconnect(); - return Error::ERROR; - } else if(num_bytes_written != num_bytes_to_write) { - buffer_offset += num_bytes_written; - } else { - buffer_offset = 0; - } - return Error::OK; - } - - static bool http_is_redirect(const char *header, size_t size) { - const void *end_of_first_line_p = memmem(header, size, "\r\n", 2); - if(!end_of_first_line_p) - return false; - return memmem(header, (const char*)end_of_first_line_p - header, " 30", 3) != nullptr; - } - - static size_t find_start_of_first_non_redirect_header(const char *headers, size_t size, size_t &header_end) { - const char *start = headers; - while(size > 0) { - const void *end_of_header = memmem(headers, size, "\r\n\r\n", 4); - if(!end_of_header) - return std::string::npos; - - const size_t offset_to_end_of_headers = ((const char*)end_of_header + 4) - headers; - if(!http_is_redirect(headers, offset_to_end_of_headers)) { - header_end = (headers - start) + offset_to_end_of_headers; - return headers - start; - } - - headers += offset_to_end_of_headers; - size -= offset_to_end_of_headers; - } - return std::string::npos; - } - - YoutubeStaticMediaProxy::Error YoutubeStaticMediaProxy::handle_download() { - // TODO: Maybe read even if write is being slow and failing? - if(download_read_buffer_offset == 0) { - downloader_num_read_bytes = read_eintr(downloader_read_program.read_fd, download_read_buffer, sizeof(download_read_buffer)); - if(downloader_num_read_bytes == -1) { - const int err = errno; - if(err == EAGAIN || err == EWOULDBLOCK) { - return Error::OK; - } else { - perror("YoutubeStaticMediaProxy::handle_download: curl read failed"); - return Error::ERROR; - } - } else if(downloader_num_read_bytes == 0) { - if(end_of_file) { - if(client_fd != -1) { - write_all_blocking(client_fd, download_finished_response_msg, sizeof(download_finished_response_msg) - 1); - close(client_fd); - client_fd = -1; - client_request_buffer.clear(); - client_request_finished = false; - } - return Error::OK; - } - - Error err = update_download_program_status(false, -1, true); - if(err != Error::OK) - return err; - } else { - if(!download_started) { - total_downloaded_bytes = 0; - download_started = true; - download_start_time = get_boottime_milliseconds(); - } - total_downloaded_bytes += downloader_num_read_bytes; - } - } - - // TODO: Remove this code and instead create the header ourselves and send it to the client. Then pipe the curl output directly to the client input. - if(!download_header_finished) { - download_header.append(download_read_buffer, downloader_num_read_bytes); - size_t header_end = std::string::npos; - const size_t offset_to_start_of_header = find_start_of_first_non_redirect_header(download_header.c_str(), download_header.size(), header_end); - if(header_end != std::string::npos) { - download_header.erase(0, offset_to_start_of_header); - header_end -= offset_to_start_of_header; - - download_header_finished = true; - download_header_sent = false; - download_header_remaining_sent = false; - download_header_written_offset = 0; - download_header_offset_to_end_of_header = header_end; - download_read_buffer_offset = -1; - - next_range_length = header_extract_content_length(download_header.substr(0, header_end)); - header_replace_content_length(download_header, header_end, content_length - download_range_start); - } else { - if(download_header.size() > MAX_BUFFER_SIZE) { - fprintf(stderr, "YoutubeStaticMediaProxy::handle_download: buffer is full (malicious server?)\n"); - if(downloader_read_program.pid != -1) { - kill(downloader_read_program.pid, SIGTERM); - wait_program(downloader_read_program.pid); - downloader_read_program.pid = -1; - } - clear_download_state(); - update_download_program_status(false, 0, false); - return Error::ERROR; - } - } - } - - if(download_header_finished && !download_header_sent) { - Error err = continue_send(download_header.data(), download_header_offset_to_end_of_header, download_header_written_offset); - if(err != Error::OK) - return err; - - if(download_header_written_offset == 0) { - download_header_sent = true; - download_header_written_offset = download_header_offset_to_end_of_header; - } - } - - if(download_header_finished && !download_header_remaining_sent) { - Error err = continue_send(download_header.data(), download_header.size(), download_header_written_offset); - if(err != Error::OK) - return err; - - if(download_header_written_offset == 0) { - download_header_remaining_sent = true; - download_read_buffer_offset = 0; - return Error::OK; - } - } - - if(download_header_remaining_sent) - return continue_send(download_read_buffer, downloader_num_read_bytes, download_read_buffer_offset); - - return Error::OK; - } - - int YoutubeStaticMediaProxy::accept_client() { - struct sockaddr_in client_addr; - socklen_t client_addr_len = sizeof(client_addr); - int new_client_fd = accept(socket_fd, (struct sockaddr*)&client_addr, &client_addr_len); - if(new_client_fd == -1) { - const int err = errno; - if(err == EAGAIN || err == EWOULDBLOCK) { - return -1; - } else { - perror("YoutubeStaticMediaProxy::accept_client accept failed"); - return -1; - } - } - - if(!set_non_blocking(new_client_fd)) { - perror("YoutubeStaticMediaProxy::accept_client: failed to set client socket non blocking mode"); - close(new_client_fd); - return -1; - } - - //fprintf(stderr, "YoutubeStaticMediaProxy::accept_client: client connected!\n"); - return new_client_fd; - } -} -- cgit v1.2.3