Implement html sync, fix rss sync

1 files changed, 211 insertions, 23 deletions

diff --git a/src/html.c b/src/html.c
index 02982d6..3159500 100644
--- a/src/html.c
+++ b/src/html.c
@@ -4,6 +4,7 @@
 #include "buffer.h"
 #include "stringutils.h"
 #include "rss_html_common.h"
+#include "main.h"
 #include "json.h"
 #include <limits.h>
 #include <string.h>
@@ -11,6 +12,8 @@
 #include <libgen.h>
 #include <signal.h>
 #include <time.h>
+#include <dirent.h>
+#include <unistd.h>
 #include <assert.h>
 
 static int str_starts_with(const char *str, int len, const char *substr, int substr_len) {
@@ -48,9 +51,38 @@ static int url_extract_domain(const char *url, char *domain, int domain_len) {
     return 0;
 }
 
+/*
+    The plugin should print the names and urls of each item (chapter for manga) and the output list should stop when an
+    item matches any item in the input. The output should be sorted from newest to oldest.
+
+    The input should be in this format:
+      [
+        {
+          "title": "Example name",
+          "url": "https://example.com"
+        },
+        {
+          "title": "Another item",
+          "url": "https://another.url.com"
+        }
+      ]
+
+    And the output should be in this format:
+      [
+        {
+          "name": "Example name",
+          "url": "https://example.com"
+        },
+        {
+          "name": "Another item",
+          "url": "https://another.url.com"
+        }
+      ]
+
+    TODO: Rename input "title" to "url", to make input and output match (easier to test with).
+*/
 typedef int (*PluginListCallback)(const char *name, const char *url, void *userdata);
-static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, const char *latest, PluginListCallback callback, void *userdata) {
-    (void)latest;
+static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, struct json_array_s *downloaded_items, PluginListCallback callback, void *userdata) {
     int result;
     Buffer buffer;
     buffer_init(&buffer);
@@ -65,6 +97,34 @@ static struct json_value_s* plugin_list(char *plugin_filepath, const char *url,
         goto err_cleanup;
     }
 
+    if(downloaded_items) {
+        struct json_value_s downloaded_items_value;
+        downloaded_items_value.payload = downloaded_items;
+        downloaded_items_value.type = json_type_array;
+        size_t json_output_len = 0;
+        void *json_body_str = json_write_minified(&downloaded_items_value, &json_output_len);
+        if(!json_body_str) {
+            fprintf(stderr, "Failed to convert downloaded items to json\n");
+            if(process_id != -1)
+                kill(process_id, SIGKILL);
+            close(stdin_file);
+            close(stdout_file);
+            goto err_cleanup;
+        }
+
+        /* This is a bug in the json library */
+        json_output_len = strlen(json_body_str);
+        if(write(stdin_file, json_body_str, json_output_len) != (ssize_t)json_output_len) {
+            fprintf(stderr, "Failed to write all bytes to plugin list\n");
+            if(process_id != -1)
+                kill(process_id, SIGKILL);
+            close(stdin_file);
+            close(stdout_file);
+            goto err_cleanup;
+        }
+        free(json_body_str);
+    }
+
     result = program_wait_until_exit(process_id, stdin_file, stdout_file, program_buffer_write_callback, &buffer);
     if(result != 0) {
         fprintf(stderr, "Failed to launch plugin list for plugin %s\n", basename(plugin_filepath));
@@ -76,6 +136,7 @@ static struct json_value_s* plugin_list(char *plugin_filepath, const char *url,
         fprintf(stderr, "Failed to load plugin %s list output as json\n", basename(plugin_filepath));
         goto err_cleanup;
     }
+    buffer_deinit(&buffer);
 
     struct json_array_s *json_root_array = json_value_as_array(json_root);
     if(!json_root_array) {
@@ -108,7 +169,6 @@ static struct json_value_s* plugin_list(char *plugin_filepath, const char *url,
             break;
     }
 
-    buffer_deinit(&buffer);
     return json_root;
 
     err_cleanup:
@@ -123,7 +183,6 @@ typedef struct {
 } PluginListUserdata;
 
 static int plugin_list_callback(const char *name, const char *url, void *userdata) {
-    /*fprintf(stderr, "name: |%s|, url: |%s|\n", name, url);*/
     PluginListUserdata *plugin_list_userdata = userdata;
     if(plugin_list_userdata->start_after && strcmp(plugin_list_userdata->start_after, name) == 0) {
         plugin_list_userdata->found_start_after = 1;
@@ -133,6 +192,31 @@ static int plugin_list_callback(const char *name, const char *url, void *userdat
     return 0;
 }
 
+/* Store result in @plugin_filepath */
+static int get_plugin_filepath(const char *program_dir, const char *plugin_name, char *plugin_filepath) {
+    const char *path_components[] = { program_dir, "plugins" };
+    path_join(plugin_filepath, path_components, 2);
+    if(file_exists(plugin_filepath) != 0) {
+        strcpy(plugin_filepath, "/usr/share/automedia/plugins");
+        if(file_exists(plugin_filepath) != 0) {
+            fprintf(stderr, "Failed to find plugins directory\n");
+            return -1;
+        }
+    }
+
+    strcat(plugin_filepath, "/");
+    strcat(plugin_filepath, plugin_name);
+    if(file_exists(plugin_filepath) != 0) {
+        strcat(plugin_filepath, ".py");
+        if(file_exists(plugin_filepath) != 0) {
+            fprintf(stderr, "Plugin doesn't exist: %s\n", plugin_name);
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
 int add_html(const char *name, const char *url, char *html_config_dir, char *program_dir, const char *start_after) {
     int result = 0;
 
@@ -152,26 +236,9 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro
         return -1;
     }
 
-    const char *path_components[] = { program_dir, "plugins" };
     char domain_plugin_path[PATH_MAX];
-    path_join(domain_plugin_path, path_components, 2);
-    if(file_exists(domain_plugin_path) != 0) {
-        strcpy(domain_plugin_path, "/usr/share/automedia/plugins");
-        if(file_exists(domain_plugin_path) != 0) {
-            fprintf(stderr, "Failed to find plugins directory\n");
-            return -1;
-        }
-    }
-
-    strcat(domain_plugin_path, "/");
-    strcat(domain_plugin_path, domain);
-    if(file_exists(domain_plugin_path) != 0) {
-        strcat(domain_plugin_path, ".py");
-        if(file_exists(domain_plugin_path) != 0) {
-            fprintf(stderr, "Plugin doesn't exist: %s\n", domain);
-            return -1;
-        }
-    }
+    if(get_plugin_filepath(program_dir, domain, domain_plugin_path) != 0)
+        return -1;
 
     PluginListUserdata plugin_list_userdata;
     plugin_list_userdata.start_after = start_after;
@@ -255,3 +322,124 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro
     free(json_root);
     return result;
 }
+
+static int plugin_list_sync_callback(const char *name, const char *url, void *userdata) {
+    Buffer *download_items_buffer = userdata;
+    DownloadItemsData download_items_data;
+    download_items_data.title = name;
+    download_items_data.link = url;
+    buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data));
+    return 0;
+}
+
+static int download_html_items_in_reverse(const char *plugin_filepath, Buffer *download_items_buffer, TrackedHtml *tracked_html, char *html_tracked_dir, const char *download_dir) {
+    int result = 0;
+    DownloadItemsData *added_download_items[MAX_UPDATE_ITEMS];
+
+    char item_dir[PATH_MAX];
+    const char *path_components[] = { download_dir, tracked_html->title };
+    int item_dir_len = path_join(item_dir, path_components, 2);
+
+    Buffer json_element_buffer;
+    buffer_init(&json_element_buffer);
+
+    DownloadItemsData *download_items_it = buffer_end(download_items_buffer);
+    DownloadItemsData *download_items_end = buffer_begin(download_items_buffer);
+    download_items_it--;
+    download_items_end--;
+    int download_item_index = 0;
+    for(; download_items_it != download_items_end && download_item_index < MAX_UPDATE_ITEMS && is_program_running(); --download_items_it) {
+        item_dir[item_dir_len] = '/';
+        strcpy(item_dir + item_dir_len + 1, download_items_it->title);
+        if(create_directory_recursive(item_dir) != 0) {
+            fprintf(stderr, "Failed to create directory for html item: %s\n", download_items_it->title);
+            result = -1;
+            break;
+        }
+
+        /* TODO: Make asynchronous */
+        const char *args[] = { plugin_filepath, "download", download_items_it->link, NULL };
+        result = program_exec(args, NULL, NULL);
+        if(result != 0)
+            fprintf(stderr, "Failed while downloading html, url: %s\n", download_items_it->link);
+        
+        char notify_msg[PATH_MAX];
+        const char *path_components[] = { tracked_html->title, download_items_it->title };
+        path_join(notify_msg, path_components, 2);
+        const char *notify_args[] = { "notify-send", "-u", result == 0 ? "normal" : "critical", "--", result == 0 ? "Download finished" : "Download failed", notify_msg, NULL };
+        program_exec(notify_args, NULL, NULL);
+
+        if(result != 0)
+            break;
+
+        added_download_items[download_item_index] = download_items_it;
+        ++download_item_index;
+    }
+
+    TrackedItem tracked_item;
+    tracked_item.title = tracked_html->title;
+    tracked_item.link = tracked_html->link;
+    tracked_item.json_data = tracked_html->json_data;
+    result = tracked_item_update_latest(&tracked_item, html_tracked_dir, added_download_items, NULL, download_item_index);
+
+    buffer_deinit(&json_element_buffer);
+    return result;
+}
+
+/* TODO: Make asynchronous. Right now this will only complete when the whole chapter download completes */
+int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download_dir, char *html_config_dir) {
+    /* TODO: This can be cached */
+    int html_config_dir_len = strlen(html_config_dir);
+
+    fprintf(stderr, "Syncing %s\n", tracked_html->title);
+
+    char plugin_filepath[PATH_MAX];
+    /* This will check with ${tracked_html->plugin}.py as well, but that is fine */
+    if(get_plugin_filepath(program_dir, tracked_html->plugin, plugin_filepath) != 0)
+        return -1;
+
+    struct json_value_s *downloaded_items = json_object_get_field_by_name(tracked_html->json_data, "downloaded");
+    struct json_array_s *downloaded_items_array = NULL;
+    if(downloaded_items) {
+        downloaded_items_array = json_value_as_array(downloaded_items);
+        if(!downloaded_items_array) {
+            fprintf(stderr, "Corrupt json for html item: %s\n", tracked_html->title);
+            return -1;
+        }
+    }
+
+    Buffer download_items_buffer;
+    buffer_init(&download_items_buffer);
+
+    int result = 0;
+
+    struct json_value_s *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items_array, plugin_list_sync_callback, &download_items_buffer);
+    if(!json_root) {
+        result = -1;
+        goto cleanup;
+    }
+
+    char *html_tracked_dir = html_config_dir;
+    strcat(html_tracked_dir, "/tracked/");
+
+    result = download_html_items_in_reverse(plugin_filepath, &download_items_buffer, tracked_html, html_tracked_dir, download_dir);
+    if(result != 0) {
+        fprintf(stderr, "Failed while download html item for url: %s\n", tracked_html->link);
+        goto cleanup;
+    }
+
+    char updated[32];
+    sprintf(updated, "%ld", time(NULL));
+    strcat(html_tracked_dir, tracked_html->title);
+    result = file_overwrite_in_dir(html_tracked_dir, "synced", updated, strlen(updated));
+    if(result != 0) {
+        fprintf(stderr, "Failed to update %s/synced\n", html_tracked_dir);
+        goto cleanup;
+    }
+
+    cleanup:
+    free(json_root);
+    buffer_deinit(&download_items_buffer);
+    html_config_dir[html_config_dir_len] = '\0';
+    return result;
+}