Ph: fix related videos images and titles missing or incorrect. Use message type to open in video player instead of using http head

6 files changed, 135 insertions, 146 deletions

diff --git a/.gitmodules b/.gitmodules
index 97f7a6d..474a748 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "depends/rapidjson"]
 	path = depends/rapidjson
 	url = https://git.dec05eba.com/rapidjson
+[submodule "depends/html-parser"]
+	path = depends/html-parser
+	url = https://git.dec05eba.com/html-parser
diff --git a/depends/html-parser b/depends/html-parser
new file mode 160000
+Subproject fdfdf20d085a7c705477d878c11ed208577facb
diff --git a/src/Body.cpp b/src/Body.cpp
index 8224087..9a811a9 100644
--- a/src/Body.cpp
+++ b/src/Body.cpp
@@ -281,7 +281,6 @@ namespace QuickMedia {
         //item_background.setFillColor(front_color);
         //item_background.setOutlineThickness(1.0f);
         //item_background.setOutlineColor(sf::Color(13, 15, 17));
-        image_fallback.setSize(thumbnail_fallback_size);
         item_background_shadow.setFillColor(line_seperator_color);
         num_visible_items = 0;
         last_item_fully_visible = true;
@@ -579,11 +578,16 @@ namespace QuickMedia {
                 // We want the next image fallback to have the same size as the successful image rendering, because its likely the image fallback will have the same size (for example thumbnails on youtube)
                 //image_fallback.setSize(sf::Vector2f(width_ratio * image_size.x, height_ratio * image_size.y));
             } else if(!item->thumbnail_url.empty()) {
+                sf::Vector2f content_size(thumbnail_resize_target_size.x, thumbnail_resize_target_size.y);
+                if(item->thumbnail_size.x > 0 && item->thumbnail_size.y > 0)
+                    content_size = sf::Vector2f(item->thumbnail_size.x, item->thumbnail_size.y);
+                image_fallback.setSize(content_size);
+
                 if(thumbnail_mask_shader && item->thumbnail_mask_type == ThumbnailMaskType::CIRCLE) {
                     // TODO: Use the mask shader instead, but a vertex shader is also needed for that to pass the vertex coordinates since
                     // shapes dont have texture coordinates.
                     // TODO: Cache circle shape
-                    sf::CircleShape circle_shape(image_fallback.getSize().x * 0.5f);
+                    sf::CircleShape circle_shape(content_size.x * 0.5f);
                     circle_shape.setFillColor(sf::Color::White);
                     circle_shape.setPosition(item_pos + sf::Vector2f(image_padding_x, padding_y));
                     window.draw(circle_shape);
@@ -591,10 +595,7 @@ namespace QuickMedia {
                     image_fallback.setPosition(item_pos + sf::Vector2f(image_padding_x, padding_y));
                     window.draw(image_fallback);
                 }
-                float image_width = image_fallback.getSize().x;
-                if(item->thumbnail_size.x > 0 && item->thumbnail_size.y > 0)
-                    image_width = item->thumbnail_size.x;
-                text_offset_x += image_padding_x + image_width;
+                text_offset_x += image_padding_x + content_size.x;
             }
         }
 
@@ -699,7 +700,7 @@ namespace QuickMedia {
             item_height += item->description_text->getHeight() - 2.0f;
         }
         if(draw_thumbnails && !item->thumbnail_url.empty()) {
-            float image_height = image_fallback.getSize().y;
+            float image_height = thumbnail_resize_target_size.y;
             if(item->thumbnail_size.x > 0 && item->thumbnail_size.y > 0)
                 image_height = item->thumbnail_size.y;
 
@@ -718,7 +719,10 @@ namespace QuickMedia {
                 item_thumbnail->referenced = true;
 
                 if(!item->thumbnail_url.empty() && item_thumbnail->loading_state == LoadingState::NOT_LOADED) {
-                    async_image_loader.load_thumbnail(item->thumbnail_url, item->thumbnail_is_local, thumbnail_resize_target_size, program->is_tor_enabled(), item_thumbnail);
+                    sf::Vector2i resize_size = thumbnail_resize_target_size;
+                    if(item->thumbnail_size.x > 0 && item->thumbnail_size.y > 0)
+                        resize_size = item->thumbnail_size;
+                    async_image_loader.load_thumbnail(item->thumbnail_url, item->thumbnail_is_local, resize_size, program->is_tor_enabled(), item_thumbnail);
                 }
 
                 if(item_thumbnail->loading_state == LoadingState::FINISHED_LOADING && item_thumbnail->image->getSize().x > 0 && item_thumbnail->image->getSize().y > 0) {
diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp
index 614fad3..0ab78a1 100644
--- a/src/QuickMedia.cpp
+++ b/src/QuickMedia.cpp
@@ -3263,41 +3263,6 @@ namespace QuickMedia {
             }
         };
 
-        // TODO: Remove this? it adds additional delay to launch.
-        // TODO: Make async
-        auto link_get_content_type = [this](const std::string &url) -> const char* {
-            std::vector<CommandArg> additional_args = {
-                { "-I", "" } // HEAD request, to get content-type
-            };
-
-            std::string program_result;
-            if(download_to_string(url, program_result, std::move(additional_args), use_tor, true) != DownloadResult::OK) {
-                fprintf(stderr, "HEAD request to %s failed\n", url.c_str());
-                return nullptr;
-            }
-
-            const char *result = nullptr;
-            string_split(program_result, '\n', [&result](const char *str, size_t size) mutable -> bool {
-                if(size > 13 && strncasecmp(str, "content-type:", 13) == 0) {
-                    std::string content_type_value(str + 13, size - 13);
-                    content_type_value = strip(content_type_value);
-                    if(strncasecmp(content_type_value.c_str(), "audio", 5) == 0) {
-                        result = "audio";
-                        return false;
-                    } else if(strncasecmp(content_type_value.c_str(), "video", 5) == 0) {
-                        result = "video";
-                        return false;
-                    } else if(strncasecmp(content_type_value.c_str(), "image", 5) == 0) {
-                        result = "image";
-                        return false;
-                    }
-                }
-                return true;
-            });
-
-            return result;
-        };
-
         auto add_new_messages_to_current_room = [this, &tabs, &current_room](Messages &messages) {
             int num_items = tabs[MESSAGES_TAB_INDEX].body->items.size();
             bool scroll_to_end = (num_items == 0 || tabs[MESSAGES_TAB_INDEX].body->is_selected_item_last_visible_item());
@@ -3327,6 +3292,7 @@ namespace QuickMedia {
                 body_item->thumbnail_url = room->avatar_url;
                 body_item->userdata = room.get(); // Note: this has to be valid as long as the room list is valid!
                 body_item->thumbnail_mask_type = ThumbnailMaskType::CIRCLE;
+                body_item->thumbnail_size = sf::Vector2i(32, 32);
                 tabs[ROOMS_TAB_INDEX].body->items.push_back(body_item);
                 body_items_by_room[room] = { body_item, true, 0 };
             }
@@ -3478,16 +3444,18 @@ namespace QuickMedia {
                     if(tabs[selected_tab].type == ChatTabType::MESSAGES && event.key.code == sf::Keyboard::Enter) {
                         BodyItem *selected = tabs[selected_tab].body->get_selected();
                         if(selected) {
+                            MessageType message_type = static_cast<Message*>(selected->userdata)->type;
                             std::string selected_url = selected->url;
-                            if(selected_url.empty() && selected->embedded_item)
+                            if(selected_url.empty() && selected->embedded_item) {
                                 selected_url = selected->embedded_item->url;
+                                message_type = static_cast<Message*>(selected->embedded_item->userdata)->type;
+                            }
                             if(!selected_url.empty()) {
-                                const char *content_type = link_get_content_type(selected_url);
-                                if(content_type && (strcmp(content_type, "audio") == 0 || strcmp(content_type, "video") == 0 || strcmp(content_type, "image") == 0)) {
+                                if(message_type == MessageType::VIDEO || message_type == MessageType::IMAGE || message_type == MessageType::AUDIO) {
                                     page_stack.push(PageType::CHAT);
                                     watched_videos.clear();
                                     current_page = PageType::VIDEO_CONTENT;
-                                    bool is_audio = strcmp(content_type, "audio") == 0;
+                                    bool is_audio = (message_type == MessageType::AUDIO);
                                     bool prev_no_video = no_video;
                                     no_video = is_audio;
                                     // TODO: Add title
diff --git a/src/plugins/Pornhub.cpp b/src/plugins/Pornhub.cpp
index e8df9d7..b063a32 100644
--- a/src/plugins/Pornhub.cpp
+++ b/src/plugins/Pornhub.cpp
@@ -1,7 +1,9 @@
 #include "../../plugins/Pornhub.hpp"
 #include "../../include/StringUtils.hpp"
 #include "../../include/NetUtils.hpp"
-#include <quickmedia/HtmlSearch.h>
+extern "C" {
+#include <HtmlParser.h>
+}
 #include <string.h>
 
 namespace QuickMedia {
@@ -9,64 +11,124 @@ namespace QuickMedia {
         return strncmp(str, begin_with, strlen(begin_with)) == 0;
     }
 
-    static bool contains(const char *str, const char *substr) {
-        return strstr(str, substr);
+    // TODO: Optimize by using HtmlStringView instead of std::string
+    struct HtmlElement {
+        std::string tag_name;
+        std::map<std::string, std::string> attributes;
+        std::vector<HtmlElement*> children;
+        HtmlElement *parent = nullptr; // ref
+    };
+
+    static void html_cleanup(HtmlElement *html_element_root) {
+        for(HtmlElement *child_html_element : html_element_root->children) {
+            html_cleanup(child_html_element);
+        }
+        delete html_element_root;
     }
 
-    SearchResult PornhubSearchPage::search(const std::string &str, BodyItems &result_items) {
-        std::string url = "https://www.pornhub.com/video/search?search=";
-        url += url_param_encode(str);
+    static const std::string& html_get_attribute_or(HtmlElement *html_element, const std::string &attr_key, const std::string &default_value) {
+        auto it = html_element->attributes.find(attr_key);
+        if(it != html_element->attributes.end())
+            return it->second;
+        else
+            return default_value;
+    }
+
+    struct HtmlParseUserdata {
+        HtmlElement *current_html_element;
+    };
+
+    static void html_page_callback(HtmlParser *html_parser, HtmlParseType parse_type, void *userdata) {
+        HtmlParseUserdata *parse_userdata = (HtmlParseUserdata*)userdata;
+        if(parse_type == HTML_PARSE_TAG_START) {
+            auto new_html_element = new HtmlElement();
+            new_html_element->tag_name.assign(html_parser->tag_name.data, html_parser->tag_name.size);
+            new_html_element->parent = parse_userdata->current_html_element;
+
+            parse_userdata->current_html_element->children.push_back(new_html_element);
+            parse_userdata->current_html_element = new_html_element;
+        } else if(parse_type == HTML_PARSE_TAG_END) {
+            if(parse_userdata->current_html_element->parent)
+                parse_userdata->current_html_element = parse_userdata->current_html_element->parent;
+        } else if(parse_type == HTML_PARSE_ATTRIBUTE) {
+            std::string attr_key(html_parser->attribute_key.data, html_parser->attribute_key.size);
+            std::string attr_value(html_parser->attribute_value.data, html_parser->attribute_value.size);
+            parse_userdata->current_html_element->attributes.insert(std::make_pair(std::move(attr_key), std::move(attr_value)));
+        }
+    }
+
+    static HtmlElement* html_parse(char *source, size_t size) {
+        HtmlElement *html_element_root = new HtmlElement();
+        HtmlParseUserdata parse_userdata;
+        parse_userdata.current_html_element = html_element_root;
+        HtmlParser html_parser;
+        html_parser_init(&html_parser, source, size, html_page_callback, &parse_userdata);
+        html_parser_parse(&html_parser);
+        html_parser_deinit(&html_parser);
+        return html_element_root;
+    }
+
+    using HtmlFindTagsCallback = std::function<void(HtmlElement *html_element)>;
+    static void html_find_tags_with_class(HtmlElement *html_element, const std::string &tag_name, const std::string &class_value, const HtmlFindTagsCallback &callback) {
+        if(html_element->tag_name == tag_name) {
+            if(html_get_attribute_or(html_element, "class", "") == class_value)
+                callback(html_element);
+        }
+        for(HtmlElement *child_html_element : html_element->children) {
+            html_find_tags_with_class(child_html_element, tag_name, class_value, callback);
+        }
+    }
+
+    static void html_find_tags(HtmlElement *html_element, const std::string &tag_name, const HtmlFindTagsCallback &callback) {
+        if(html_element->tag_name == tag_name)
+            callback(html_element);
+        for(HtmlElement *child_html_element : html_element->children) {
+            html_find_tags(child_html_element, tag_name, callback);
+        }
+    }
 
+    static SearchResult get_videos_in_page(const std::string &url, bool use_tor, BodyItems &result_items) {
         std::string website_data;
-        if(download_to_string(url, website_data, {}, is_tor_enabled()) != DownloadResult::OK)
+        if(download_to_string(url, website_data, {}, use_tor) != DownloadResult::OK)
             return SearchResult::NET_ERR;
 
-        struct ItemData {
-            BodyItems *result_items;
-            size_t index;
-        };
-        ItemData item_data = { &result_items, 0 };
-
-        QuickMediaHtmlSearch html_search;
-        int result = quickmedia_html_search_init(&html_search, website_data.c_str());
-        if(result != 0)
-            goto cleanup;
-
-        result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='phimage']//a",
-            [](QuickMediaHtmlNode *node, void *userdata) {
-                auto *result_items = (BodyItems*)userdata;
-                const char *href = quickmedia_html_node_get_attribute_value(node, "href");
-                const char *title = quickmedia_html_node_get_attribute_value(node, "title");
-                if(href && title && begins_with(href, "/view_video.php?viewkey")) {
-                    auto item = BodyItem::create(strip(title));
-                    item->url = std::string("https://www.pornhub.com") + href;
-                    result_items->push_back(std::move(item));
-                }
-            }, &result_items);
-        if(result != 0)
-            goto cleanup;
-
-        result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='phimage']//img",
-            [](QuickMediaHtmlNode *node, void *userdata) {
-                ItemData *item_data = (ItemData*)userdata;
-                if(item_data->index >= item_data->result_items->size())
-                    return;
-
-                const char *data_src = quickmedia_html_node_get_attribute_value(node, "data-src");
-                if(data_src && contains(data_src, "phncdn.com/videos")) {
-                    (*item_data->result_items)[item_data->index]->thumbnail_url = data_src;
-                    ++item_data->index;
-                }
-            }, &item_data);
+        HtmlElement *html_root = html_parse(website_data.data(), website_data.size());
+        html_find_tags_with_class(html_root, "div", "phimage", [&result_items](HtmlElement *html_element) {
+            auto it = html_element->attributes.find("data-entrycode");
+            if(it == html_element->attributes.end() || it->second != "VidPg-premVid-videoPage") {
+                html_find_tags(html_element, "a", [&result_items](HtmlElement *html_element) {
+                    const std::string &href = html_get_attribute_or(html_element, "href", "");
+                    const std::string &title = html_get_attribute_or(html_element, "title", "");
+                    if(!href.empty() && !title.empty() && begins_with(href.c_str(), "/view_video.php?viewkey")) {
+                        std::string title_fixed = strip(title);
+                        html_unescape_sequences(title_fixed);
+                        auto item = BodyItem::create(std::move(title_fixed));
+                        item->url = std::string("https://www.pornhub.com") + href;
+                        item->thumbnail_size = sf::Vector2i(192, 108);
+                        result_items.push_back(std::move(item));
+
+                        html_find_tags(html_element, "img", [&result_items](HtmlElement *html_element) {
+                            const std::string &src = html_get_attribute_or(html_element, "data-src", "");
+                            if(src.find("phncdn.com/videos") != std::string::npos)
+                                result_items.back()->thumbnail_url = src;
+                        });
+                    }
+                });
+            }
+        });
+        html_cleanup(html_root);
 
         // Attempt to skip promoted videos (that are not related to the search term)
-        if(result_items.size() >= 4) {
+        if(result_items.size() >= 4)
             result_items.erase(result_items.begin(), result_items.begin() + 4);
-        }
 
-        cleanup:
-        quickmedia_html_search_deinit(&html_search);
-        return result == 0 ? SearchResult::OK : SearchResult::ERR;
+        return SearchResult::OK;
+    }
+
+    SearchResult PornhubSearchPage::search(const std::string &str, BodyItems &result_items) {
+        std::string url = "https://www.pornhub.com/video/search?search=";
+        url += url_param_encode(str);
+        return get_videos_in_page(url, is_tor_enabled(), result_items);
     }
 
     PluginResult PornhubSearchPage::submit(const std::string &title, const std::string &url, std::vector<Tab> &result_tabs) {
@@ -78,56 +140,7 @@ namespace QuickMedia {
 
     BodyItems PornhubVideoPage::get_related_media(const std::string &url) {
         BodyItems result_items;
-
-        std::string website_data;
-        if(download_to_string(url, website_data, {}, is_tor_enabled()) != DownloadResult::OK)
-            return result_items;
-
-        struct ItemData {
-            BodyItems *result_items;
-            size_t index;
-        };
-        ItemData item_data = { &result_items, 0 };
-
-        QuickMediaHtmlSearch html_search;
-        int result = quickmedia_html_search_init(&html_search, website_data.c_str());
-        if(result != 0)
-            goto cleanup;
-
-        result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='phimage']//a",
-            [](QuickMediaHtmlNode *node, void *userdata) {
-                auto *result_items = (BodyItems*)userdata;
-                const char *href = quickmedia_html_node_get_attribute_value(node, "href");
-                const char *title = quickmedia_html_node_get_attribute_value(node, "title");
-                if(href && title && begins_with(href, "/view_video.php?viewkey")) {
-                    auto item = BodyItem::create(strip(title));
-                    item->url = std::string("https://www.pornhub.com") + href;
-                    result_items->push_back(std::move(item));
-                }
-            }, &result_items);
-        if(result != 0)
-            goto cleanup;
-
-        result = quickmedia_html_find_nodes_xpath(&html_search, "//div[class='phimage']//img",
-            [](QuickMediaHtmlNode *node, void *userdata) {
-                ItemData *item_data = (ItemData*)userdata;
-                if(item_data->index >= item_data->result_items->size())
-                    return;
-
-                const char *src = quickmedia_html_node_get_attribute_value(node, "src");
-                if(src && contains(src, "phncdn.com/videos")) {
-                    (*item_data->result_items)[item_data->index]->thumbnail_url = src;
-                    ++item_data->index;
-                }
-            }, &item_data);
-
-        // Attempt to skip promoted videos (that are not related to the search term)
-        if(result_items.size() >= 4) {
-            result_items.erase(result_items.begin(), result_items.begin() + 4);
-        }
-
-        cleanup:
-        quickmedia_html_search_deinit(&html_search);
+        get_videos_in_page(url, is_tor_enabled(), result_items);
         return result_items;
     }
 }
\ No newline at end of file
diff --git a/src/plugins/Youtube.cpp b/src/plugins/Youtube.cpp
index 424a8d2..54b020f 100644
--- a/src/plugins/Youtube.cpp
+++ b/src/plugins/Youtube.cpp
@@ -98,6 +98,7 @@ namespace QuickMedia {
         body_item->set_description(std::move(desc));
         body_item->url = "https://www.youtube.com/watch?v=" + video_id_str;
         body_item->thumbnail_url = std::move(thumbnail_url);
+        body_item->thumbnail_size = sf::Vector2i(158, 119);
         added_videos.insert(video_id_str);
         return body_item;
     }