Youtube: load english subtitles when available

7 files changed, 99 insertions, 7 deletions

diff --git a/TODO b/TODO
index f1d8ace..4e1ed73 100644
--- a/TODO
+++ b/TODO
@@ -153,7 +153,6 @@ Embedding elements in rich text: first byte should be an invalid utf8 character
 Do not set fps to monitor hz if no key is pressed and cursor is not moving (for example when the computer is sleeping). (Maybe just detect if any sfml event is received), but maybe that wouldn't work with accelerometer.
 Ctrl+arrow key to move to previous/next video.
 Add keybinding to view file-manager images in fullscreen to preview them. Also add keybinding to create/delete directories.
-Add loading of english subtitles for youtube.
 Update item height when it switches from not being merged with previous to being merged with previous. This happens when loading previous messages in matrix and the message is the top one.
 Reload youtube video url if the video is idle for too long. The video url is only valid for a specific amount of time (the valid duration is in the json).
 Improve live stream startup time by downloading the video formats in parts instead of the hls/dash manifest? (use YoutubeLiveStreamMediaProxy).
@@ -202,4 +201,5 @@ Add option to navigate studios/producers/author in AniList/MAL.
 Renable throttle detection after fixing it (it doesn't detect throttling well and it breaks for very long videos, such as 8 hour long videos).
 Show who deleted a message in matrix.
 Sync should replace all messages in the room (except for the selected room?) to reduce ram usage when in many rooms and when quickmedia has been running for a long time doing sync.
-Fix notifications not being marked as read correctly (they remain red!).
\ No newline at end of file
+Fix notifications not being marked as read correctly (they remain red!).
+Show youtube annotations.
\ No newline at end of file
diff --git a/include/VideoPlayer.hpp b/include/VideoPlayer.hpp
index cd1e979..4eab812 100644
--- a/include/VideoPlayer.hpp
+++ b/include/VideoPlayer.hpp
@@ -51,6 +51,8 @@ namespace QuickMedia {
         Error set_property(const std::string &property_name, const Json::Value &value);
         Error get_property(const std::string &property_name, Json::Value *result, Json::ValueType result_type);
 
+        Error add_subtitle(const std::string &url, const std::string &title, const std::string &lang);
+
         int exit_status;
     private:
         Error send_command(const char *cmd, size_t size);
diff --git a/plugins/Page.hpp b/plugins/Page.hpp
index f793c65..e60c752 100644
--- a/plugins/Page.hpp
+++ b/plugins/Page.hpp
@@ -111,6 +111,11 @@ namespace QuickMedia {
         const char* get_title() const override { return "Related videos"; }
     };
 
+    struct SubtitleData {
+        std::string url;
+        std::string title;
+    };
+
     class VideoPage : public Page {
     public:
         VideoPage(Program *program, std::string url) : Page(program), url(std::move(url)) {}
@@ -140,8 +145,11 @@ namespace QuickMedia {
         virtual std::string get_audio_url(std::string &ext) { (void)ext; return ""; }
         virtual std::string url_get_playable_url(const std::string &url) { return url; }
         virtual bool video_should_be_skipped(const std::string &url) { (void)url; return false; }
+        // This needs to be called before the other functions are called
         virtual PluginResult load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) { (void)title; (void)channel_url; (void)chapters; return PluginResult::OK; }
         virtual void mark_watched() {};
+        // Should not do any network request to not slow down video loading
+        virtual void get_subtitles(SubtitleData &subtitle_data) { (void)subtitle_data; }
     protected:
         std::string url;
     };
diff --git a/plugins/Youtube.hpp b/plugins/Youtube.hpp
index c3945de..e2bc2a4 100644
--- a/plugins/Youtube.hpp
+++ b/plugins/Youtube.hpp
@@ -153,6 +153,7 @@ namespace QuickMedia {
         std::string get_audio_url(std::string &ext) override;
         PluginResult load(std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) override;
         void mark_watched() override;
+        void get_subtitles(SubtitleData &subtitle_data) override;
     private:
         PluginResult parse_video_response(const Json::Value &json_root, std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters);
         void parse_format(const Json::Value &format_json, bool is_adaptive);
@@ -163,6 +164,7 @@ namespace QuickMedia {
         std::string livestream_url;
         std::vector<YoutubeVideoFormat> video_formats;
         std::vector<YoutubeAudioFormat> audio_formats;
+        std::map<std::string, SubtitleData> subtitle_urls_by_lang_code;
 
         std::string playback_url;
         std::string watchtime_url;
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index c85f031..1bc260c 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -2622,7 +2622,7 @@ namespace QuickMedia {
         }
 
         sf::WindowHandle video_player_window = None;
-        auto on_window_create = [this, &video_player_window, &video_loaded](sf::WindowHandle _video_player_window) mutable {
+        auto on_window_create = [this, &video_player_window, &video_loaded, &video_page](sf::WindowHandle _video_player_window) mutable {
             video_player_window = _video_player_window;
             XSelectInput(disp, video_player_window, KeyPressMask | PointerMotionMask);
             XSync(disp, False);
@@ -2633,6 +2633,11 @@ namespace QuickMedia {
             video_player->get_time_in_file(&time_in_file);
             if(time_in_file > 0.00001)
                 video_loaded = true;
+
+            SubtitleData subtitle_data;
+            video_page->get_subtitles(subtitle_data);
+            if(!subtitle_data.url.empty())
+                video_player->add_subtitle(subtitle_data.url, subtitle_data.title, "eng");
         };
 
         std::unique_ptr<YoutubeMediaProxy> youtube_video_media_proxy;
@@ -2658,7 +2663,7 @@ namespace QuickMedia {
             throttled = true;
         };
 
-        auto load_video_error_check = [this, &throttle_handler, &throttled, &youtube_downloader_task, &youtube_video_media_proxy, &youtube_audio_media_proxy, &youtube_video_content_length, &youtube_audio_content_length, &prev_start_time, &media_chapters, &video_url, &audio_url, &has_embedded_audio, &video_title, &video_tasks, &channel_url, previous_page, &go_to_previous_page, &video_loaded, video_page, &video_event_callback, &on_window_create, &video_player_window, is_youtube, is_matrix, download_if_streaming_fails](std::string start_time = "", bool reuse_media_source = false) mutable {
+        auto load_video_error_check = [&](std::string start_time = "", bool reuse_media_source = false) mutable {
             video_player.reset();
             channel_url.clear();
             video_loaded = false;
@@ -2677,7 +2682,7 @@ namespace QuickMedia {
                 bool load_successful = false;
                 for(int i = 0; i < num_retries; ++i) {
                     bool cancelled = false;
-                    TaskResult load_result = run_task_with_loading_screen([this, video_page, &cancelled, &new_title, &channel_url, &media_chapters, &youtube_video_content_length, &youtube_audio_content_length, largest_monitor_height, &has_embedded_audio, &video_url, &audio_url, &is_audio_only, &previous_page, is_youtube, download_if_streaming_fails]() {
+                    TaskResult load_result = run_task_with_loading_screen([&]() {
                         if(video_page->load(new_title, channel_url, media_chapters) != PluginResult::OK)
                             return false;
 
diff --git a/src/VideoPlayer.cpp b/src/VideoPlayer.cpp
index 3e22b8b..4b3ec92 100644
--- a/src/VideoPlayer.cpp
+++ b/src/VideoPlayer.cpp
@@ -409,7 +409,7 @@ namespace QuickMedia {
         command_data.append(property_name);
         command_data.append(value);
         Json::Value command(Json::objectValue);
-        command["command"] = command_data;
+        command["command"] = std::move(command_data);
 
         Json::StreamWriterBuilder builder;
         builder["commentStyle"] = "None";
@@ -429,7 +429,7 @@ namespace QuickMedia {
         command_data.append("get_property");
         command_data.append(property_name);
         Json::Value command(Json::objectValue);
-        command["command"] = command_data;
+        command["command"] = std::move(command_data);
         command["request_id"] = cmd_request_id;
 
         Json::StreamWriterBuilder builder;
@@ -472,6 +472,26 @@ namespace QuickMedia {
         return err;
     }
 
+    VideoPlayer::Error VideoPlayer::add_subtitle(const std::string &url, const std::string &title, const std::string &lang) {
+        Json::Value command_data(Json::arrayValue);
+        command_data.append("sub-add");
+        command_data.append(url);
+        command_data.append("auto");
+        if(!title.empty()) {
+            command_data.append(title);
+            if(!lang.empty())
+                command_data.append(lang);
+        }
+        Json::Value command(Json::objectValue);
+        command["command"] = std::move(command_data);
+
+        Json::StreamWriterBuilder builder;
+        builder["commentStyle"] = "None";
+        builder["indentation"] = "";
+        const std::string cmd_str = Json::writeString(builder, command) + "\n";
+        return send_command(cmd_str.c_str(), cmd_str.size());
+    }
+
     VideoPlayer::Error VideoPlayer::send_command(const char *cmd, size_t size) {
         if(!connected_to_ipc)
             return Error::FAIL_NOT_CONNECTED;
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index be405b1..2b2a7ee 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -2298,10 +2298,51 @@ namespace QuickMedia {
         return result;
     }
 
+    static void subtitle_url_set_vtt_format(std::string &subtitle_url) {
+        const size_t index = subtitle_url.find("&fmt=");
+        if(index == std::string::npos)
+            return;
+
+        size_t end_index = subtitle_url.find('&');
+        if(end_index == std::string::npos)
+            end_index = subtitle_url.size();
+
+        subtitle_url.replace(index, end_index - index, "&fmt=vtt");
+    }
+
+    static void parse_caption_tracks(const Json::Value &caption_tracks, std::map<std::string, SubtitleData> &subtitle_urls_by_lang_code) {
+        if(!caption_tracks.isArray())
+            return;
+
+        for(const Json::Value &caption_track : caption_tracks) {
+            if(!caption_track.isObject())
+                continue;
+            
+            const Json::Value &base_url_json = caption_track["baseUrl"];
+            const Json::Value &language_code_json = caption_track["languageCode"];
+            if(!base_url_json.isString() || !language_code_json.isString())
+                continue;
+
+            std::string base_url = base_url_json.asString();
+            subtitle_url_set_vtt_format(base_url);
+            std::string language_code = language_code_json.asString();
+
+            std::optional<std::string> title = yt_json_get_text(caption_track, "name");
+            SubtitleData subtitle_data;
+            subtitle_data.url = std::move(base_url);
+            if(title)
+                subtitle_data.title = std::move(title.value());
+            else
+                subtitle_data.title = language_code;
+            subtitle_urls_by_lang_code[std::move(language_code)] = std::move(subtitle_data);
+        }
+    }
+
     PluginResult YoutubeVideoPage::parse_video_response(const Json::Value &json_root, std::string &title, std::string &channel_url, std::vector<MediaChapter> &chapters) {
         livestream_url.clear();
         video_formats.clear();
         audio_formats.clear();
+        subtitle_urls_by_lang_code.clear();
         title.clear();
         channel_url.clear();
         chapters.clear();
@@ -2315,6 +2356,7 @@ namespace QuickMedia {
             if(status_json.isString() && (strcmp(status_json.asCString(), "UNPLAYABLE") == 0 || strcmp(status_json.asCString(), "LOGIN_REQUIRED") == 0)) {
                 const Json::Value &reason_json = playability_status_json["reason"];
                 fprintf(stderr, "Unable to play video, status: %s, reason: %s\n", status_json.asCString(), reason_json.isString() ? reason_json.asCString() : "Unknown");
+                return PluginResult::ERR;
             }
         }
 
@@ -2359,6 +2401,13 @@ namespace QuickMedia {
             }
         }
 
+        const Json::Value &captions_json = json_root["captions"];
+        if(captions_json.isObject()) {
+            const Json::Value &player_captions_tracklist_renderer_json = captions_json["playerCaptionsTracklistRenderer"];
+            if(player_captions_tracklist_renderer_json.isObject())
+                parse_caption_tracks(player_captions_tracklist_renderer_json["captionTracks"], subtitle_urls_by_lang_code);
+        }
+
         const Json::Value &playback_tracing_json = json_root["playbackTracking"];
         if(playback_tracing_json.isObject()) {
             if(playback_url.empty()) {
@@ -2472,6 +2521,12 @@ R"END(
         }
     }
 
+    void YoutubeVideoPage::get_subtitles(SubtitleData &subtitle_data) {
+        auto it = subtitle_urls_by_lang_code.find("en");
+        if(it != subtitle_urls_by_lang_code.end())
+            subtitle_data = it->second;
+    }
+
     static bool parse_cipher_format(const Json::Value &format, YoutubeFormat &youtube_format) {
         std::map<std::string, std::string> cipher_params;
         const Json::Value &cipher_json = format["cipher"];