path: root/src/plugins/Youtube.cpp

#include "../../plugins/Youtube.hpp"
#include "../../include/Storage.hpp"
#include "../../include/NetUtils.hpp"
#include "../../include/StringUtils.hpp"
#include "../../include/Scale.hpp"
#include <string.h>

namespace QuickMedia {
    // This is a common setup of text in the youtube json
    static std::optional<std::string> yt_json_get_text(const Json::Value &json, const char *root_name) {
        if(!json.isObject())
            return std::nullopt;

        const Json::Value &text_json = json[root_name];
        if(!text_json.isObject())
            return std::nullopt;

        const Json::Value &simple_text_json = text_json["simpleText"];
        if(simple_text_json.isString()) {
            return simple_text_json.asCString();
        } else {
            const Json::Value &runs_json = text_json["runs"];
            if(!runs_json.isArray() || runs_json.empty())
                return std::nullopt;

            std::string result;
            for(const Json::Value &first_runs_json : runs_json) {
                if(!first_runs_json.isObject())
                    continue;

                const Json::Value &text_json = first_runs_json["text"];
                if(text_json.isString())
                    result += text_json.asString();
            }
            if(!result.empty())
                return result;
        }

        return std::nullopt;
    }

    struct Thumbnail {
        const char *url;
        int width;
        int height;
    };

    // TODO: Use this in |parse_common_video_item| when QuickMedia supports webp
    static std::optional<Thumbnail> yt_json_get_largest_thumbnail(const Json::Value &thumbnail_json) {
        if(!thumbnail_json.isObject())
            return std::nullopt;

        const Json::Value &thumbnails_json = thumbnail_json["thumbnails"];
        if(!thumbnails_json.isArray())
            return std::nullopt;

        std::vector<Thumbnail> thumbnails;
        for(const Json::Value &thumbnail_data_json : thumbnails_json) {
            if(!thumbnail_data_json.isObject())
                continue;

            const Json::Value &url_json = thumbnail_data_json["url"];
            if(!url_json.isString())
                continue;

            const Json::Value &width_json = thumbnail_data_json["width"];
            if(!width_json.isInt())
                continue;

            const Json::Value &height_json = thumbnail_data_json["height"];
            if(!height_json.isInt())
                continue;

            thumbnails.push_back({ url_json.asCString(), width_json.asInt(), height_json.asInt() });
        }

        return *std::max_element(thumbnails.begin(), thumbnails.end(), [](const Thumbnail &thumbnail1, const Thumbnail &thumbnail2) {
            int size1 = thumbnail1.width * thumbnail1.height;
            int size2 = thumbnail2.width * thumbnail2.height;
            return size1 < size2;
        });
    }

    static std::shared_ptr<BodyItem> parse_common_video_item(const Json::Value &video_item_json, std::unordered_set<std::string> &added_videos) {
        const Json::Value &video_id_json = video_item_json["videoId"];
        if(!video_id_json.isString())
            return nullptr;

        std::string video_id_str = video_id_json.asString();
        if(added_videos.find(video_id_str) != added_videos.end())
            return nullptr;

        std::optional<std::string> title = yt_json_get_text(video_item_json, "title");
        if(!title)
            return nullptr;

        std::optional<std::string> date = yt_json_get_text(video_item_json, "publishedTimeText");
        std::optional<std::string> view_count_text = yt_json_get_text(video_item_json, "viewCountText");
        std::optional<std::string> owner_text = yt_json_get_text(video_item_json, "shortBylineText");
        std::optional<std::string> description_snippet = yt_json_get_text(video_item_json, "descriptionSnippet");
        std::optional<std::string> length = yt_json_get_text(video_item_json, "lengthText");
        if(!length) {
            const Json::Value &thumbnail_overlays_json = video_item_json["thumbnailOverlays"];
            if(thumbnail_overlays_json.isArray() && !thumbnail_overlays_json.empty()) {
                const Json::Value &thumbnail_overlay_json = thumbnail_overlays_json[0];
                if(thumbnail_overlay_json.isObject())
                    length = yt_json_get_text(thumbnail_overlay_json["thumbnailOverlayTimeStatusRenderer"], "text");
            }
        }

        std::string scheduled_text;
        const Json::Value &upcoming_event_data_json = video_item_json["upcomingEventData"];
        if(upcoming_event_data_json.isObject()) {
            const Json::Value &start_time_json = upcoming_event_data_json["startTime"];
            if(!start_time_json.isString())
                return nullptr;

            std::optional<std::string> upcoming_event_text = yt_json_get_text(upcoming_event_data_json, "upcomingEventText");
            if(!upcoming_event_text)
                return nullptr;

            time_t start_time = strtol(start_time_json.asCString(), nullptr, 10);
            struct tm *message_tm = localtime(&start_time);
            char time_str[128] = {0};
            strftime(time_str, sizeof(time_str) - 1, "%a %b %d %H:%M", message_tm);
            string_replace_all(upcoming_event_text.value(), "DATE_PLACEHOLDER", time_str);
            scheduled_text = std::move(upcoming_event_text.value());
        }
        
        auto body_item = BodyItem::create(title.value());
        std::string desc;
        if(view_count_text)
            desc += view_count_text.value();
        if(date) {
            if(!desc.empty())
                desc += " • ";
            desc += date.value();
        }
        if(!scheduled_text.empty()) {
            if(!desc.empty())
                desc += " • ";
            desc += scheduled_text;
        }
        if(length) {
            if(!desc.empty())
                desc += '\n';
            desc += length.value();
        }
        if(owner_text) {
            if(!desc.empty())
                desc += '\n';
            desc += owner_text.value();
        }
        if(description_snippet) {
            if(!desc.empty())
                desc += '\n';
            desc += '\n';
            desc += description_snippet.value();
        }
        body_item->set_description(std::move(desc));
        if(scheduled_text.empty())
            body_item->url = "https://www.youtube.com/watch?v=" + video_id_str;
        body_item->thumbnail_url = "https://img.youtube.com/vi/" + video_id_str + "/hqdefault.jpg";
        body_item->thumbnail_size = sf::Vector2i(175, 131);
        added_videos.insert(video_id_str);
        return body_item;
    }

    static std::shared_ptr<BodyItem> parse_content_video_renderer(const Json::Value &content_item_json, std::unordered_set<std::string> &added_videos) {
        if(!content_item_json.isObject())
            return nullptr;
        
        const Json::Value &video_renderer_json = content_item_json["videoRenderer"];
        if(!video_renderer_json.isObject())
            return nullptr;

        return parse_common_video_item(video_renderer_json, added_videos);
    }

    static std::shared_ptr<BodyItem> parse_channel_renderer(const Json::Value &channel_renderer_json) {
        if(!channel_renderer_json.isObject())
            return nullptr;

        const Json::Value &channel_id_json = channel_renderer_json["channelId"];
        if(!channel_id_json.isString())
            return nullptr;

        std::optional<std::string> title = yt_json_get_text(channel_renderer_json, "title");
        if(!title)
            return nullptr;

        std::optional<std::string> description = yt_json_get_text(channel_renderer_json, "descriptionSnippet");
        std::optional<std::string> video_count = yt_json_get_text(channel_renderer_json, "videoCountText");
        std::optional<std::string> subscribers = yt_json_get_text(channel_renderer_json, "subscriberCountText");

        const Json::Value &thumbnail_json = channel_renderer_json["thumbnail"];
        std::optional<Thumbnail> thumbnail = yt_json_get_largest_thumbnail(thumbnail_json);

        auto body_item = BodyItem::create(title.value());
        std::string desc;
        if(subscribers)
            desc += subscribers.value();
        if(video_count) {
            if(!desc.empty())
                desc += " • ";
            desc += video_count.value();
            if(strcmp(video_count.value().c_str(), "1") == 0)
                desc += " video";
            else
                desc += " videos";
        }
        if(description) {
            if(!desc.empty())
                desc += '\n';
            desc += '\n';
            desc += description.value();
        }
        body_item->set_description(std::move(desc));
        body_item->url = "https://www.youtube.com/channel/" + channel_id_json.asString();
        if(thumbnail) {
            body_item->thumbnail_url = std::string("https:") + thumbnail->url;
            body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE;
            body_item->thumbnail_size.x = thumbnail->width;
            body_item->thumbnail_size.y = thumbnail->height;
            body_item->thumbnail_size = clamp_to_size(body_item->thumbnail_size, sf::Vector2i(136, 136));
        }
        return body_item;
    }

    // Returns empty string if continuation token can't be found
    static std::string item_section_renderer_get_continuation_token(const Json::Value &item_section_renderer_json) {
        const Json::Value &continuation_item_renderer_json = item_section_renderer_json["continuationItemRenderer"];
        if(!continuation_item_renderer_json.isObject())
            return "";

        const Json::Value &continuation_endpoint_json = continuation_item_renderer_json["continuationEndpoint"];
        if(!continuation_endpoint_json.isObject())
            return "";

        const Json::Value &continuation_command_json = continuation_endpoint_json["continuationCommand"];
        if(!continuation_command_json.isObject())
            return "";

        const Json::Value &token_json = continuation_command_json["token"];
        if(!token_json.isString())
            return "";
        
        return token_json.asString();
    }

    static std::string grid_renderer_get_continuation_token(const Json::Value &grid_renderer_json) {
        const Json::Value &continuations_json = grid_renderer_json["continuations"];
        if(!continuations_json.isArray())
            return "";

        for(const Json::Value &continuation_json : continuations_json) {
            if(!continuation_json.isObject())
                continue;

            const Json::Value &next_continuation_data_json = continuation_json["nextContinuationData"];
            if(!next_continuation_data_json.isObject())
                continue;

            const Json::Value &continuation_item_json = next_continuation_data_json["continuation"];
            if(continuation_item_json.isString())
                return continuation_item_json.asString();
        }

        return "";
    }

    static void parse_item_section_renderer(const Json::Value &item_section_renderer_json, std::unordered_set<std::string> &added_videos, BodyItems &result_items) {
        const Json::Value &item_contents_json = item_section_renderer_json["contents"];
        if(!item_contents_json.isArray())
            return;

        for(const Json::Value &content_item_json : item_contents_json) {
            if(!content_item_json.isObject())
                continue;

            for(Json::Value::const_iterator it = content_item_json.begin(); it != content_item_json.end(); ++it) {
                Json::Value key = it.key();
                if(key.isString() && strcmp(key.asCString(), "shelfRenderer") == 0) {
                    const Json::Value &shelf_renderer_json = *it;
                    if(!shelf_renderer_json.isObject())
                        continue;

                    const Json::Value &item_content_json = shelf_renderer_json["content"];
                    if(!item_content_json.isObject())
                        continue;

                    const Json::Value &vertical_list_renderer_json = item_content_json["verticalListRenderer"];
                    if(!vertical_list_renderer_json.isObject())
                        continue;

                    const Json::Value &items_json = vertical_list_renderer_json["items"];
                    if(!items_json.isArray())
                        continue;
                    
                    for(const Json::Value &item_json : items_json) {
                        std::shared_ptr<BodyItem> body_item = parse_content_video_renderer(item_json, added_videos);
                        if(body_item)
                            result_items.push_back(std::move(body_item));
                    }
                } else if(key.isString() && strcmp(key.asCString(), "channelRenderer") == 0) {
                    std::shared_ptr<BodyItem> body_item = parse_channel_renderer(*it);
                    if(body_item)
                        result_items.push_back(std::move(body_item));
                } else {
                    std::shared_ptr<BodyItem> body_item = parse_content_video_renderer(content_item_json, added_videos);
                    if(body_item)
                        result_items.push_back(std::move(body_item));
                }
            }
        }
    }

    static std::string remove_index_from_playlist_url(const std::string &url) {
        std::string result = url;
        size_t index = result.rfind("&index=");
        if(index == std::string::npos)
            return result;
        return result.substr(0, index);
    }

    static std::shared_ptr<BodyItem> parse_compact_video_renderer_json(const Json::Value &item_json, std::unordered_set<std::string> &added_videos) {
        const Json::Value &compact_video_renderer_json = item_json["compactVideoRenderer"];
        if(!compact_video_renderer_json.isObject())
            return nullptr;

        return parse_common_video_item(compact_video_renderer_json, added_videos);
    }

    static BodyItems parse_channel_videos(const Json::Value &json_root, std::string &continuation_token, std::unordered_set<std::string> &added_videos) {
        BodyItems body_items;
        if(!json_root.isArray())
            return body_items;

        std::string new_continuation_token;
        for(const Json::Value &json_item : json_root) {
            if(!json_item.isObject())
                continue;

            const Json::Value &response_json = json_item["response"];
            if(!response_json.isObject())
                continue;

            const Json::Value &contents_json = response_json["contents"];
            if(!contents_json.isObject())
                continue;

            const Json::Value &tcbrr_json = contents_json["twoColumnBrowseResultsRenderer"];
            if(!tcbrr_json.isObject())
                continue;

            const Json::Value &tabs_json = tcbrr_json["tabs"];
            if(!tabs_json.isArray())
                continue;

            for(const Json::Value &tab_json : tabs_json) {
                if(!tab_json.isObject())
                    continue;

                const Json::Value &tab_renderer_json = tab_json["tabRenderer"];
                if(!tab_renderer_json.isObject())
                    continue;
                
                const Json::Value &content_json = tab_renderer_json["content"];
                if(!content_json.isObject())
                    continue;

                const Json::Value &section_list_renderer = content_json["sectionListRenderer"];
                if(!section_list_renderer.isObject())
                    continue;

                const Json::Value &contents2_json = section_list_renderer["contents"];
                if(!contents2_json.isArray())
                    continue;
                
                for(const Json::Value &content_item_json : contents2_json) {
                    if(!content_item_json.isObject())
                        continue;
                    
                    const Json::Value &item_section_renderer_json = content_item_json["itemSectionRenderer"];
                    if(!item_section_renderer_json.isObject())
                        continue;

                    const Json::Value &item_contents_json = item_section_renderer_json["contents"];
                    if(!item_contents_json.isArray())
                        continue;

                    for(const Json::Value &content_json : item_contents_json) {
                        if(!content_json.isObject())
                            continue;

                        const Json::Value &grid_renderer_json = content_json["gridRenderer"];
                        if(!grid_renderer_json.isObject())
                            continue;

                        if(new_continuation_token.empty())
                            new_continuation_token = grid_renderer_get_continuation_token(grid_renderer_json);

                        const Json::Value &items_json = grid_renderer_json["items"];
                        if(!items_json.isArray())
                            continue;

                        for(const Json::Value &item_json : items_json) {
                            if(!item_json.isObject())
                                continue;

                            const Json::Value &grid_video_renderer = item_json["gridVideoRenderer"];
                            if(!grid_video_renderer.isObject())
                                continue;

                            auto body_item = parse_common_video_item(grid_video_renderer, added_videos);
                            if(body_item)
                                body_items.push_back(std::move(body_item));
                        }
                    }
                }
            }
        }

        if(!new_continuation_token.empty())
            continuation_token = std::move(new_continuation_token);

        return body_items;
    }

    SearchResult YoutubeSearchPage::search(const std::string &str, BodyItems &result_items) {
        continuation_token.clear();
        current_page = 0;
        added_videos.clear();

        search_url = "https://youtube.com/results?search_query=";
        search_url += url_param_encode(str);

        std::vector<CommandArg> additional_args = {
            { "-H", "x-spf-referer: " + search_url },
            { "-H", "x-youtube-client-name: 1" },
            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
            { "-H", "referer: " + search_url }
        };

        //std::vector<CommandArg> cookies = get_cookies();
        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());

        Json::Value json_root;
        DownloadResult result = download_json(json_root, search_url + "&pbj=1", std::move(additional_args), true);
        if(result != DownloadResult::OK) return download_result_to_search_result(result);

        if(!json_root.isArray())
            return SearchResult::ERR;

        for(const Json::Value &json_item : json_root) {
            if(!json_item.isObject())
                continue;

            const Json::Value &response_json = json_item["response"];
            if(!response_json.isObject())
                continue;

            const Json::Value &contents_json = response_json["contents"];
            if(!contents_json.isObject())
                continue;

            const Json::Value &tcsrr_json = contents_json["twoColumnSearchResultsRenderer"];
            if(!tcsrr_json.isObject())
                continue;

            const Json::Value &primary_contents_json = tcsrr_json["primaryContents"];
            if(!primary_contents_json.isObject())
                continue;

            const Json::Value &section_list_renderer_json = primary_contents_json["sectionListRenderer"];
            if(!section_list_renderer_json.isObject())
                continue;
            
            const Json::Value &contents2_json = section_list_renderer_json["contents"];
            if(!contents2_json.isArray())
                continue;
            
            for(const Json::Value &item_json : contents2_json) {
                if(!item_json.isObject())
                    continue;

                if(continuation_token.empty())
                    continuation_token = item_section_renderer_get_continuation_token(item_json);
                
                const Json::Value &item_section_renderer_json = item_json["itemSectionRenderer"];
                if(!item_section_renderer_json.isObject())
                    continue;

                parse_item_section_renderer(item_section_renderer_json, added_videos, result_items);
            }
        }

        return SearchResult::OK;
    }

    PluginResult YoutubeSearchPage::get_page(const std::string&, int page, BodyItems &result_items) {
        while(current_page < page) {
            PluginResult plugin_result = search_get_continuation(search_url, continuation_token, result_items);
            if(plugin_result != PluginResult::OK) return plugin_result;
            ++current_page;
        }
        return PluginResult::OK;
    }

    PluginResult YoutubeSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
        if(url.empty())
            return PluginResult::OK;

        if(strncmp(url.c_str(), "https://www.youtube.com/channel/", 32) == 0) {
            // TODO: Make all pages (for all services) lazy fetch in a similar manner!
            result_tabs.push_back(Tab{create_body(), std::make_unique<YoutubeChannelPage>(program, url, "", title), create_search_bar("Search...", SEARCH_DELAY_FILTER)});
        } else {
            result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
        }
        return PluginResult::OK;
    }

    PluginResult YoutubeSearchPage::search_get_continuation(const std::string &url, const std::string &current_continuation_token, BodyItems &result_items) {
        std::string next_url = url + "&pbj=1&ctoken=" + current_continuation_token;

        std::vector<CommandArg> additional_args = {
            { "-H", "x-spf-referer: " + url },
            { "-H", "x-youtube-client-name: 1" },
            { "-H", "x-spf-previous: " + url },
            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
            { "-H", "referer: " + url }
        };

        //std::vector<CommandArg> cookies = get_cookies();
        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());

        Json::Value json_root;
        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);

        if(!json_root.isArray())
            return PluginResult::ERR;

        std::string new_continuation_token;
        for(const Json::Value &json_item : json_root) {
            if(!json_item.isObject())
                continue;

            const Json::Value &response_json = json_item["response"];
            if(!response_json.isObject())
                continue;

            const Json::Value &on_response_received_commands_json = response_json["onResponseReceivedCommands"];
            if(!on_response_received_commands_json.isArray())
                continue;

            for(const Json::Value &response_received_command : on_response_received_commands_json) {
                if(!response_received_command.isObject())
                    continue;

                const Json::Value &append_continuation_items_action_json = response_received_command["appendContinuationItemsAction"];
                if(!append_continuation_items_action_json.isObject())
                    continue;

                const Json::Value &continuation_items_json = append_continuation_items_action_json["continuationItems"];
                if(!continuation_items_json.isArray())
                    continue;

                for(const Json::Value &continuation_item : continuation_items_json) {
                    if(!continuation_item.isObject())
                        continue;

                    if(new_continuation_token.empty()) {
                        // Note: item_section_renderer is compatible with continuation_item
                        new_continuation_token = item_section_renderer_get_continuation_token(continuation_item);
                    }

                    const Json::Value &item_section_renderer_json = continuation_item["itemSectionRenderer"];
                    if(!item_section_renderer_json.isObject())
                        continue;

                    parse_item_section_renderer(item_section_renderer_json, added_videos, result_items);
                }
            }
        }

        if(!new_continuation_token.empty())
            continuation_token = std::move(new_continuation_token);

        return PluginResult::OK;
    }

    PluginResult YoutubeChannelPage::get_page(const std::string&, int page, BodyItems &result_items) {
        while(current_page < page) {
            PluginResult plugin_result = search_get_continuation(url, continuation_token, result_items);
            if(plugin_result != PluginResult::OK) return plugin_result;
            ++current_page;
        }
        return PluginResult::OK;
    }

    PluginResult YoutubeChannelPage::search_get_continuation(const std::string &url, const std::string &current_continuation_token, BodyItems &result_items) {
        std::string next_url = "https://www.youtube.com/browse_ajax?ctoken=" + current_continuation_token + "&continuation=" + current_continuation_token;

        std::vector<CommandArg> additional_args = {
            { "-H", "x-spf-referer: " + url },
            { "-H", "x-youtube-client-name: 1" },
            { "-H", "x-spf-previous: " + url },
            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
            { "-H", "referer: " + url }
        };

        //std::vector<CommandArg> cookies = get_cookies();
        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());

        Json::Value json_root;
        DownloadResult result = download_json(json_root, next_url, std::move(additional_args), true);
        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);

        if(!json_root.isArray())
            return PluginResult::ERR;

        std::string new_continuation_token;
        for(const Json::Value &json_item : json_root) {
            if(!json_item.isObject())
                continue;

            const Json::Value &response_json = json_item["response"];
            if(!response_json.isObject())
                continue;

            const Json::Value &continuation_contents_json = response_json["continuationContents"];
            if(!continuation_contents_json.isObject())
                continue;

            const Json::Value &grid_continuation_json = continuation_contents_json["gridContinuation"];
            if(!grid_continuation_json.isObject())
                continue;

            if(new_continuation_token.empty()) {
                // grid_continuation_json is compatible with grid_renderer
                new_continuation_token = grid_renderer_get_continuation_token(grid_continuation_json);
            }

            const Json::Value &items_json = grid_continuation_json["items"];
            if(!items_json.isArray())
                continue;

            for(const Json::Value &item_json : items_json) {
                if(!item_json.isObject())
                    continue;

                const Json::Value &grid_video_renderer = item_json["gridVideoRenderer"];
                if(!grid_video_renderer.isObject())
                    continue;

                auto body_item = parse_common_video_item(grid_video_renderer, added_videos);
                if(body_item)
                    result_items.push_back(std::move(body_item));
            }
        }

        if(!new_continuation_token.empty())
            continuation_token = std::move(new_continuation_token);

        return PluginResult::OK;
    }

    PluginResult YoutubeChannelPage::submit(const std::string&, const std::string &url, std::vector<Tab> &result_tabs) {
        if(url.empty())
            return PluginResult::OK;
        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
        return PluginResult::OK;
    }

    PluginResult YoutubeChannelPage::lazy_fetch(BodyItems &result_items) {
        std::vector<CommandArg> additional_args = {
            { "-H", "x-spf-referer: " + url },
            { "-H", "x-youtube-client-name: 1" },
            { "-H", "x-spf-previous: " + url },
            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
            { "-H", "referer: " + url }
        };

        //std::vector<CommandArg> cookies = get_cookies();
        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());

        Json::Value json_root;
        DownloadResult result = download_json(json_root, url + "/videos?pbj=1", std::move(additional_args), true);
        if(result != DownloadResult::OK) return download_result_to_plugin_result(result);
        result_items = parse_channel_videos(json_root, continuation_token, added_videos);
        return PluginResult::OK;
    }

    PluginResult YoutubeRelatedVideosPage::submit(const std::string&, const std::string&, std::vector<Tab> &result_tabs) {
        result_tabs.push_back(Tab{nullptr, std::make_unique<YoutubeVideoPage>(program), nullptr});
        return PluginResult::OK;
    }

    // TODO: Make this faster by using string search instead of parsing html.
    // TODO: If the result is a play
    BodyItems YoutubeVideoPage::get_related_media(const std::string &url, std::string &channel_url) {
        BodyItems result_items;

        std::string modified_url = remove_index_from_playlist_url(url);
        std::vector<CommandArg> additional_args = {
            { "-H", "x-spf-referer: " + url },
            { "-H", "x-youtube-client-name: 1" },
            { "-H", "x-spf-previous: " + url },
            { "-H", "x-youtube-client-version: 2.20200626.03.00" },
            { "-H", "referer: " + url }
        };

        //std::vector<CommandArg> cookies = get_cookies();
        //additional_args.insert(additional_args.end(), cookies.begin(), cookies.end());

        Json::Value json_root;
        DownloadResult result = download_json(json_root, modified_url + "&pbj=1", std::move(additional_args), true);
        if(result != DownloadResult::OK) return result_items;

        if(!json_root.isArray())
            return result_items;

        std::unordered_set<std::string> added_videos;

        for(const Json::Value &json_item : json_root) {
            if(!json_item.isObject())
                continue;

            if(channel_url.empty()) {
                const Json::Value &player_response_json = json_item["playerResponse"];
                if(player_response_json.isObject()) {
                    const Json::Value &video_details_json = player_response_json["videoDetails"];
                    if(video_details_json.isObject()) {
                        const Json::Value &channel_id_json = video_details_json["channelId"];
                        if(channel_id_json.isString())
                            channel_url = "https://www.youtube.com/channel/" + channel_id_json.asString();
                    }
                }
            }

            const Json::Value &response_json = json_item["response"];
            if(!response_json.isObject())
                continue;

            const Json::Value &contents_json = response_json["contents"];
            if(!contents_json.isObject())
                return result_items;

            const Json::Value &tcwnr_json = contents_json["twoColumnWatchNextResults"];
            if(!tcwnr_json.isObject())
                return result_items;

            const Json::Value &secondary_results_json = tcwnr_json["secondaryResults"];
            if(!secondary_results_json.isObject())
                return result_items;

            const Json::Value &secondary_results2_json = secondary_results_json["secondaryResults"];
            if(!secondary_results2_json.isObject())
                return result_items;
            
            const Json::Value &results_json = secondary_results2_json["results"];
            if(!results_json.isArray())
                return result_items;

            for(const Json::Value &item_json : results_json) {
                if(!item_json.isObject())
                    continue;

                auto body_item = parse_compact_video_renderer_json(item_json, added_videos);
                if(body_item)
                    result_items.push_back(std::move(body_item));
                
                const Json::Value &compact_autoplay_renderer_json = item_json["compactAutoplayRenderer"];
                if(!compact_autoplay_renderer_json.isObject())
                    continue;
                
                const Json::Value &item_contents_json = compact_autoplay_renderer_json["contents"];
                if(!item_contents_json.isArray())
                    continue;
                
                for(const Json::Value &content_item_json : item_contents_json) {
                    if(!content_item_json.isObject())
                        continue;
                    
                    auto body_item = parse_compact_video_renderer_json(content_item_json, added_videos);
                    if(body_item)
                        result_items.push_back(std::move(body_item));
                }
            }
        }

        return result_items;
    }

    std::unique_ptr<Page> YoutubeVideoPage::create_search_page(Program *program, int &search_delay) {
        search_delay = 350;
        return std::make_unique<YoutubeSearchPage>(program);
    }

    std::unique_ptr<RelatedVideosPage> YoutubeVideoPage::create_related_videos_page(Program *program, const std::string&, const std::string&) {
        return std::make_unique<YoutubeRelatedVideosPage>(program);
    }

    std::unique_ptr<LazyFetchPage> YoutubeVideoPage::create_channels_page(Program *program, const std::string &channel_url) {
        return std::make_unique<YoutubeChannelPage>(program, channel_url, "", "Channel videos");
    }
}