Youtube: remove dependency on youtube-dl for downloads (also fixes downloads of age restricted videos)

1 files changed, 475 insertions, 0 deletions

diff --git a/src/Downloader.cpp b/src/Downloader.cpp
new file mode 100644
index 0000000..ffb8675
--- /dev/null
+++ b/src/Downloader.cpp
@@ -0,0 +1,475 @@
+#include "../include/Downloader.hpp"
+#include "../include/Storage.hpp"
+#include "../include/NetUtils.hpp"
+#include "../include/Notification.hpp"
+#include <SFML/System/Clock.hpp>
+#include <unistd.h>
+#include <signal.h>
+
+namespace QuickMedia {
+    // TODO: Test with video that has hlsManifestUrl
+    static bool youtube_url_is_live_stream(const std::string &url) {
+        return url.find("yt_live_broadcast") != std::string::npos;
+    }
+
+    CurlDownloader::CurlDownloader(const std::string &url, const std::string &output_filepath, int64_t content_length) : Downloader(url, output_filepath), content_length(content_length) {
+        output_filepath_tmp = output_filepath;
+        output_filepath_tmp.append(".tmp");
+        read_program.pid = -1;
+        read_program.read_fd = -1;
+        progress_text = "0 bytes/Unknown";
+        download_speed_text = "Unknown/s";
+    }
+
+    bool CurlDownloader::start() {
+        remove(output_filepath_tmp.data.c_str());
+
+        if(content_length == -1) {
+            const char *args[] = { "curl",
+                "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed",
+                "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+                "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(),
+                "-D", "/dev/stdout",
+                "--", url.c_str(), nullptr };
+
+            if(exec_program_pipe(args, &read_program) != 0)
+                return false;
+        } else {
+            const char *args[] = { "curl",
+                "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed",
+                "-H", "user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+                "-g", "-s", "-L", "-f", "-o", output_filepath_tmp.data.c_str(),
+                "--", url.c_str(), nullptr };
+
+            if(exec_program_pipe(args, &read_program) != 0)
+                return false;
+        }
+
+        if(content_length != -1)
+            return true;
+
+        // TODO: Remove this async task and make the fd non blocking instead
+        header_reader = AsyncTask<bool>([this]{
+            char tmp_buf[1024];
+            while(true) {
+                ssize_t bytes_available = read(read_program.read_fd, tmp_buf, sizeof(tmp_buf));
+                if(bytes_available == -1) {
+                    return false;
+                } else if(bytes_available > 0 && content_length == -1) {
+                    header.append(tmp_buf, bytes_available);
+                    if(header.find("\r\n\r\n") != std::string::npos) {
+                        std::string content_length_str = header_extract_value(header, "content-length");
+                        if(!content_length_str.empty()) {
+                            errno = 0;
+                            char *endptr;
+                            const long content_length_tmp = strtol(content_length_str.c_str(), &endptr, 10);
+                            if(endptr != content_length_str.c_str() && errno == 0) {
+                                std::lock_guard<std::mutex> lock(content_length_mutex);
+                                content_length = content_length_tmp;
+                            }
+                        }
+                    }
+                }
+            }
+            return true;
+        });
+
+        return true;
+    }
+
+    bool CurlDownloader::stop(bool download_completed) {
+        if(read_program.read_fd != -1)
+            close(read_program.read_fd);
+        if(read_program.pid != -1)
+            kill(read_program.pid, SIGTERM);
+        if(!download_completed)
+            remove(output_filepath_tmp.data.c_str());
+        //header_reader.cancel();
+        finished = false;
+        return true;
+    }
+
+    DownloadUpdateStatus CurlDownloader::update() {
+        if(finished)
+            return DownloadUpdateStatus::FINISHED;
+        
+        if(read_program.pid == -1)
+            return DownloadUpdateStatus::ERROR;
+
+        int status = 0;
+        if(wait_program_non_blocking(read_program.pid, &status)) {
+            read_program.pid = -1;
+            if(status == 0 && rename_atomic(output_filepath_tmp.data.c_str(), output_filepath.c_str()) == 0) {
+                finished = true;
+                progress = 1.0f;
+                return DownloadUpdateStatus::FINISHED;
+            } else {
+                return DownloadUpdateStatus::ERROR;
+            }
+        }
+
+        if(header_reader.ready()) {
+            if(!header_reader.get())
+                return DownloadUpdateStatus::ERROR;
+        }
+
+        std::lock_guard<std::mutex> lock(content_length_mutex);
+        size_t output_file_size = 0;
+        file_get_size(output_filepath_tmp, &output_file_size);
+        size_t downloaded_size = std::min((int64_t)output_file_size, content_length);
+
+        if(content_length == -1) {
+            progress_text = std::to_string(output_file_size / 1024) + "/Unknown";
+        } else {
+            size_t percentage = 0;
+            if(output_file_size > 0)
+                percentage = (double)downloaded_size / (double)content_length * 100.0;
+            progress = (double)percentage / 100.0;
+            progress_text = file_size_to_human_readable_string(downloaded_size) + "/" + file_size_to_human_readable_string(content_length) + " (" + std::to_string(percentage) + "%)";
+        }
+
+        // TODO: Take into consideration time overflow?
+        size_t downloaded_diff = std::max(0lu, downloaded_size - downloaded_since_last_check);
+        download_speed_text = file_size_to_human_readable_string(downloaded_diff) + "/s";
+        downloaded_since_last_check = downloaded_size;
+
+        return DownloadUpdateStatus::DOWNLOADING;
+    }
+
+    float CurlDownloader::get_progress() {
+        return progress;
+    }
+
+    std::string CurlDownloader::get_progress_text() {
+        return progress_text;
+    }
+
+    std::string CurlDownloader::get_download_speed_text() {
+        return download_speed_text;
+    }
+
+    YoutubeDlDownloader::YoutubeDlDownloader(const std::string &url, const std::string &output_filepath, bool no_video) : Downloader(url, output_filepath), no_video(no_video) {
+        // youtube-dl requires a file extension for the file
+        if(this->output_filepath.find('.') == std::string::npos)
+            this->output_filepath += ".mkv";
+
+        read_program.pid = -1;
+        read_program.read_fd = -1;
+        progress_text = "0.0% of Unknown";
+        download_speed_text = "Unknown/s";
+    }
+
+    bool YoutubeDlDownloader::start() {
+        remove(output_filepath.c_str());
+
+        std::vector<const char*> args = { "youtube-dl", "--no-warnings", "--no-continue", "--output", output_filepath.c_str(), "--newline" };
+        if(no_video) {
+            args.push_back("-f");
+            args.push_back("bestaudio/best");
+            args.push_back("-x");
+        } else {
+            args.push_back("-f");
+            args.push_back("bestvideo+bestaudio/best");
+        }
+        args.insert(args.end(), { "--", url.c_str(), nullptr });
+
+        if(exec_program_pipe(args.data(), &read_program) != 0)
+            return false;
+
+        read_program_file = fdopen(read_program.read_fd, "rb");
+        if(!read_program_file) {
+            if(read_program.pid != -1)
+                wait_program(read_program.pid);
+            return false;
+        }
+
+        // TODO: Remove this async task and make the fd non blocking instead
+        youtube_dl_output_reader = AsyncTask<bool>([this]{
+            char line[128];
+            char progress_c[10];
+            char content_size_c[20];
+            char download_speed_c[20];
+
+            while(true) {
+                if(fgets(line, sizeof(line), read_program_file)) {
+                    int len = strlen(line);
+                    if(len > 0 && line[len - 1] == '\n') {
+                        line[len - 1] = '\0';
+                        --len;
+                    }
+
+                    if(sscanf(line, "[download] %10s of %20s at %20s", progress_c, content_size_c, download_speed_c) == 3) {
+                        std::lock_guard<std::mutex> lock(progress_update_mutex);
+
+                        if(strcmp(progress_c, "Unknown") != 0 && strcmp(content_size_c, "Unknown") != 0) {
+                            std::string progress_str = progress_c;
+                            progress_text = progress_str + " of " + content_size_c;
+                            if(progress_str.back() == '%') {
+                                errno = 0;
+                                char *endptr;
+                                const double progress_tmp = strtod(progress_str.c_str(), &endptr);
+                                if(endptr != progress_str.c_str() && errno == 0)
+                                    progress = progress_tmp / 100.0;
+                            }
+                        }
+
+                        if(strcmp(download_speed_c, "Unknown") == 0)
+                            download_speed_text = "Unknown/s";
+                        else
+                            download_speed_text = download_speed_c;
+                    }
+                } else {
+                    return false;
+                }
+            }
+
+            return true;
+        });
+
+        return true;
+    }
+
+    bool YoutubeDlDownloader::stop(bool) {
+        if(read_program_file)
+            fclose(read_program_file);
+        if(read_program.pid != -1)
+            kill(read_program.pid, SIGTERM);
+        // TODO: Remove the temporary files created by youtube-dl (if !download_completed)
+        //header_reader.cancel();
+        finished = false;
+        return true;
+    }
+
+    DownloadUpdateStatus YoutubeDlDownloader::update() {
+        if(finished)
+            return DownloadUpdateStatus::FINISHED;
+
+        if(read_program.pid == -1)
+            return DownloadUpdateStatus::ERROR;
+
+        int status = 0;
+        if(wait_program_non_blocking(read_program.pid, &status)) {
+            read_program.pid = -1;
+            if(status == 0) {
+                finished = true;
+                return DownloadUpdateStatus::FINISHED;
+            } else {
+                return DownloadUpdateStatus::ERROR;
+            }
+        }
+
+        if(youtube_dl_output_reader.ready()) {
+            if(!youtube_dl_output_reader.get())
+                return DownloadUpdateStatus::ERROR;
+        }
+
+        return DownloadUpdateStatus::DOWNLOADING;
+    }
+
+    float YoutubeDlDownloader::get_progress() {
+        std::lock_guard<std::mutex> lock(progress_update_mutex);
+        return progress;
+    }
+
+    std::string YoutubeDlDownloader::get_progress_text() {
+        std::lock_guard<std::mutex> lock(progress_update_mutex);
+        return progress_text;
+    }
+
+    std::string YoutubeDlDownloader::get_download_speed_text() {
+        std::lock_guard<std::mutex> lock(progress_update_mutex);
+        return download_speed_text;
+    }
+
+    YoutubeDownloader::YoutubeDownloader(const MediaMetadata &video_metadata, const MediaMetadata &audio_metadata, const std::string &output_filepath) :
+        Downloader("", output_filepath),
+        video_metadata(video_metadata),
+        audio_metadata(audio_metadata)
+    {
+        for(int i = 0; i < 2; ++i)
+            downloaders[i] = nullptr;
+    }
+
+    bool YoutubeDownloader::start() {
+        struct MediaProxyMetadata {
+            std::unique_ptr<YoutubeMediaProxy> *media_proxy;
+            MediaMetadata *media_metadata;
+            std::unique_ptr<CurlDownloader> *downloader;
+            std::string *output_filepath;
+            bool tmp_audio_file;
+        };
+
+        const bool has_video = !video_metadata.url.empty();
+        
+        MediaProxyMetadata media_proxies[2] = {
+            { &youtube_video_media_proxy, &video_metadata, &downloaders[0], &video_output_filepath, false },
+            { &youtube_audio_media_proxy, &audio_metadata, &downloaders[1], &audio_output_filepath, has_video }
+        };
+
+        int num_proxied_media = 0;
+        for(int i = 0; i < 2; ++i) {
+            media_proxies[i].output_filepath->clear();
+            if(media_proxies[i].media_metadata->url.empty())
+                continue;
+
+            if(youtube_url_is_live_stream(media_proxies[i].media_metadata->url))
+                *media_proxies[i].media_proxy = std::make_unique<YoutubeLiveStreamMediaProxy>();
+            else
+                *media_proxies[i].media_proxy = std::make_unique<YoutubeStaticMediaProxy>();
+
+            if(!(*media_proxies[i].media_proxy)->start(media_proxies[i].media_metadata->url, media_proxies[i].media_metadata->content_length)) {
+                fprintf(stderr, "Failed to load start youtube media proxy\n");
+                return false;
+            }
+
+            std::string media_proxy_addr;
+            if(!(*media_proxies[i].media_proxy)->get_address(media_proxy_addr)) {
+                fprintf(stderr, "Failed to load start youtube media proxy\n");
+                return false;
+            }
+
+            if(media_proxies[i].tmp_audio_file)
+                *media_proxies[i].output_filepath = output_filepath + ".audio";
+            else
+                *media_proxies[i].output_filepath = output_filepath;
+
+            remove(media_proxies[i].output_filepath->c_str());
+            fprintf(stderr, "Downloading %s to %s\n", media_proxy_addr.c_str(), media_proxies[i].output_filepath->c_str());
+            *media_proxies[i].downloader = std::make_unique<CurlDownloader>(media_proxy_addr, *media_proxies[i].output_filepath, media_proxies[i].media_metadata->content_length);
+            *media_proxies[i].output_filepath = (*media_proxies[i].downloader)->get_output_filepath();
+            if(!(*media_proxies[i].downloader)->start())
+                return false;
+            ++num_proxied_media;
+        }
+
+        if(num_proxied_media == 0) {
+            return false;
+        }
+
+        downloader_task = AsyncTask<void>([this]() {
+            sf::Clock timer;
+            const double sleep_time_millisec = 1;
+            while(!program_is_dead_in_current_thread()) {
+                if(youtube_video_media_proxy)
+                    youtube_video_media_proxy->update();
+
+                if(youtube_audio_media_proxy)
+                    youtube_audio_media_proxy->update();
+
+                const int sleep_time = sleep_time_millisec - timer.restart().asMilliseconds();
+                if(sleep_time > 0)
+                    std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time));
+            }
+        });
+
+        return true;
+    }
+
+    static bool ffmpeg_merge_audio_and_video(const std::string &video_filepath, const std::string &audio_filepath, const std::string &output_filepath) {
+        const char *args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", video_filepath.c_str(), "-i", audio_filepath.c_str(), "-c", "copy", "--", output_filepath.c_str(), nullptr };
+        return exec_program(args, nullptr, nullptr) == 0;
+    }
+
+    bool YoutubeDownloader::stop(bool download_completed) {
+        for(int i = 0; i < 2; ++i) {
+            if(downloaders[i]) {
+                downloaders[i]->stop(download_completed);
+                downloaders[i].reset();
+            }
+        }
+
+        if(youtube_video_media_proxy)
+            youtube_video_media_proxy->stop();
+
+        if(youtube_audio_media_proxy)
+            youtube_audio_media_proxy->stop();
+
+        if(downloader_task.valid())
+            downloader_task.cancel();
+
+        youtube_video_media_proxy.reset();
+        youtube_audio_media_proxy.reset();
+
+        if(!download_completed)
+            return false;
+
+        bool success = true;
+        if(!video_metadata.url.empty() && !audio_metadata.url.empty()) {
+            std::string tmp_file = video_output_filepath + ".tmp.mkv";
+            if(ffmpeg_merge_audio_and_video(video_output_filepath, audio_output_filepath, tmp_file)) {
+                success = rename_atomic(tmp_file.c_str(), output_filepath.c_str()) == 0;
+                if(success) {
+                    remove(audio_output_filepath.c_str());
+                } else {
+                    remove(tmp_file.c_str());
+                }
+            } else {
+                success = false;
+            }
+        }
+        return success;
+    }
+
+    DownloadUpdateStatus YoutubeDownloader::update() {
+        int num_downloading = 0;
+        int num_finished = 0;
+        int num_failed = 0;
+        int num_downloaders = 0;
+
+        for(int i = 0; i < 2; ++i) {
+            if(downloaders[i]) {
+                ++num_downloaders;
+                DownloadUpdateStatus update_status = downloaders[i]->update();
+                switch(update_status) {
+                    case DownloadUpdateStatus::DOWNLOADING:
+                        ++num_downloading;
+                        break;
+                    case DownloadUpdateStatus::FINISHED:
+                        ++num_finished;
+                        break;
+                    case DownloadUpdateStatus::ERROR:
+                        ++num_failed;
+                        break;
+                }
+            }
+        }
+
+        if(num_failed > 0)
+            return DownloadUpdateStatus::ERROR;
+        else if(num_finished == num_downloaders)
+            return DownloadUpdateStatus::FINISHED;
+        else
+            return DownloadUpdateStatus::DOWNLOADING;
+    }
+
+    float YoutubeDownloader::get_progress() {
+        return get_min_progress_downloader()->get_progress();
+    }
+
+    std::string YoutubeDownloader::get_progress_text() {
+        return get_min_progress_downloader()->get_progress_text();
+    }
+
+    std::string YoutubeDownloader::get_download_speed_text() {
+        return get_min_progress_downloader()->get_download_speed_text();
+    }
+
+    CurlDownloader* YoutubeDownloader::get_min_progress_downloader() {
+        int min_index = -1;
+        float progress = 999999.0f;
+        for(int i = 0; i < 2; ++i) {
+            if(downloaders[i]) {
+                float item_progress = downloaders[i]->get_progress();
+                if(item_progress < progress) {
+                    min_index = i;
+                    progress = item_progress;
+                }
+            }
+        }
+        //assert(min_index != -1);
+        // Shouldn't happen
+        if(min_index == -1)
+            min_index = 0;
+        return downloaders[min_index].get();
+    }
+}
\ No newline at end of file