Add all items starting at start-after to the download list in the data json file. This makes downloading more robust if title/url is changes for any item

7 files changed, 116 insertions, 101 deletions

diff --git a/src/html.c b/src/html.c
index 572a396..6e27d13 100644
--- a/src/html.c
+++ b/src/html.c
@@ -166,19 +166,12 @@ static cJSON* plugin_list(char *plugin_filepath, const char *url, cJSON *downloa
     return NULL;
 }
 
-typedef struct {
-    const char *start_after;
-    int found_start_after;
-    const char *start_after_url;
-} PluginListUserdata;
-
-static int plugin_list_callback(const char *name, const char *url, void *userdata) {
-    PluginListUserdata *plugin_list_userdata = userdata;
-    if(plugin_list_userdata->start_after && strcmp(plugin_list_userdata->start_after, name) == 0) {
-        plugin_list_userdata->found_start_after = 1;
-        plugin_list_userdata->start_after_url = url;
-        return 1;
-    }
+static int plugin_list_append_item_callback(const char *name, const char *url, void *userdata) {
+    Buffer *download_items_buffer = userdata;
+    DownloadItemsData download_items_data;
+    download_items_data.title = name;
+    download_items_data.link = url;
+    buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data));
     return 0;
 }
 
@@ -230,21 +223,6 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro
     if(get_plugin_filepath(program_dir, domain, domain_plugin_path) != 0)
         return -1;
 
-    PluginListUserdata plugin_list_userdata;
-    plugin_list_userdata.start_after = start_after;
-    plugin_list_userdata.found_start_after = 0;
-    plugin_list_userdata.start_after_url = NULL;
-
-    cJSON *json_root = plugin_list(domain_plugin_path, url, NULL, plugin_list_callback, &plugin_list_userdata);
-    if(!json_root)
-        return -1;
-
-    if(start_after && !plugin_list_userdata.found_start_after) {
-        fprintf(stderr, "Failed to find %s in html %s\n", start_after, url);
-        result = -1;
-        goto cleanup;
-    }
-
     char *html_tracked_dir = html_config_dir;
     strcat(html_tracked_dir, "/tracked/");
     strcat(html_tracked_dir, name);
@@ -253,12 +231,40 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro
     strcpy(in_progress_filepath, html_tracked_dir);
     strcat(in_progress_filepath, "/.in_progress");
 
+    Buffer download_items_buffer;
+    buffer_init(&download_items_buffer);
+    cJSON *json_root = NULL;
+
     if(file_exists(html_tracked_dir) == 0 && file_exists(in_progress_filepath) != 0) {
         fprintf(stderr, "You are already tracking %s\n", url);
         result = -1;
         goto cleanup;
     }
 
+    json_root = plugin_list(domain_plugin_path, url, NULL, plugin_list_append_item_callback, &download_items_buffer);
+    if(!json_root) {
+        result = -1;
+        goto cleanup;
+    }
+
+    DownloadItemsData *download_items_start = NULL;
+    if(start_after) {
+        DownloadItemsData *download_items_it = buffer_begin(&download_items_buffer);
+        DownloadItemsData *download_items_end = buffer_end(&download_items_buffer);
+        for(; download_items_it != download_items_end; ++download_items_it) {
+            if(strcmp(start_after, download_items_it->title) == 0) {
+                download_items_start = download_items_it;
+                break;
+            }
+        }
+
+        if(!download_items_start) {
+            fprintf(stderr, "Failed to find %s in html %s\n", start_after, url);
+            result = -1;
+            goto cleanup;
+        }
+    }
+
     result = create_directory_recursive(html_tracked_dir);
     if(result != 0) {
         fprintf(stderr, "Failed to create %s, error: %s\n", html_tracked_dir, strerror(result));
@@ -273,14 +279,14 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro
     result = create_lock_file(in_progress_filepath);
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/.in_progress\n", html_tracked_dir);
-        remove(html_tracked_dir);
+        remove_recursive(html_tracked_dir);
         goto cleanup;
     }
 
     result = file_overwrite_in_dir(html_tracked_dir, "link", url, strlen(url));
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir);
-        remove(html_tracked_dir);
+        remove_recursive(html_tracked_dir);
         goto cleanup;
     }
 
@@ -288,43 +294,35 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro
     result = file_overwrite_in_dir(html_tracked_dir, "plugin", plugin_name, strlen(plugin_name));
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir);
-        remove(html_tracked_dir);
+        remove_recursive(html_tracked_dir);
         goto cleanup;
     }
 
     char updated[32];
     assert(sizeof(time_t) == sizeof(long));
-    sprintf(updated, "%ld", time(NULL));
+    snprintf(updated, sizeof(updated), "%ld", time(NULL));
     result = file_overwrite_in_dir(html_tracked_dir, "updated", updated, strlen(updated));
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/updated\n", html_tracked_dir);
-        remove(html_tracked_dir);
+        remove_recursive(html_tracked_dir);
         goto cleanup;
     }
 
-    result = write_plugin_json_to_file(html_tracked_dir, "data", url, updated, start_after, plugin_list_userdata.start_after_url, plugin_name);
+    size_t num_download_items = download_items_start ? (((DownloadItemsData*)buffer_end(&download_items_buffer)) - download_items_start) : 0;
+    result = write_plugin_json_to_file(html_tracked_dir, "data", url, updated, download_items_start, num_download_items, plugin_name);
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/data\n", html_tracked_dir);
-        remove(html_tracked_dir);
+        remove_recursive(html_tracked_dir);
         goto cleanup;
     }
 
-    remove(in_progress_filepath);
-
     cleanup:
+    remove(in_progress_filepath);
+    buffer_deinit(&download_items_buffer);
     cJSON_Delete(json_root);
     return result;
 }
 
-static int plugin_list_sync_callback(const char *name, const char *url, void *userdata) {
-    Buffer *download_items_buffer = userdata;
-    DownloadItemsData download_items_data;
-    download_items_data.title = name;
-    download_items_data.link = url;
-    buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data));
-    return 0;
-}
-
 static int download_html_items_in_reverse(const char *plugin_filepath, Buffer *download_items_buffer, TrackedHtml *tracked_html, char *html_tracked_dir, const char *download_dir) {
     int result = 0;
     DownloadItemsData *added_download_items[MAX_UPDATE_ITEMS];
@@ -410,7 +408,7 @@ int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download
 
     int result = 0;
 
-    cJSON *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items, plugin_list_sync_callback, &download_items_buffer);
+    cJSON *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items, plugin_list_append_item_callback, &download_items_buffer);
     if(!json_root) {
         result = -1;
         goto cleanup;
@@ -426,7 +424,7 @@ int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download
     }
 
     char updated[32];
-    sprintf(updated, "%ld", time(NULL));
+    snprintf(updated, sizeof(updated), "%ld", time(NULL));
     strcat(html_tracked_dir, tracked_html->title);
     result = file_overwrite_in_dir(html_tracked_dir, "synced", updated, strlen(updated));
     if(result != 0) {
diff --git a/src/html.h b/src/html.h
index 327a39e..b604d48 100644
--- a/src/html.h
+++ b/src/html.h
@@ -10,6 +10,7 @@ typedef struct {
     cJSON *json_data;
 } TrackedHtml;
 
+/* Modifies @html_config_dir */
 int add_html(const char *name, const char *url, char *html_config_dir, char *program_dir, const char *start_after);
 int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download_dir, char *html_config_dir);
 
diff --git a/src/main.c b/src/main.c
index 04ae6fe..b47e24a 100644
--- a/src/main.c
+++ b/src/main.c
@@ -566,8 +566,8 @@ static int cmdline_contains_str(const char *cmdline, int cmdline_len, const char
 static int proc_read_cmdline(const char *pid_str, char *cmdline_data, int cmdline_data_size, int *cmdline_data_size_output) {
     assert(cmdline_data_size > 0);
 
-    char cmdline_file_path[128];
-    sprintf(cmdline_file_path, "/proc/%s/cmdline", pid_str);
+    char cmdline_file_path[256];
+    snprintf(cmdline_file_path, sizeof(cmdline_file_path), "/proc/%s/cmdline", pid_str);
 
     int cmdline_fd = open(cmdline_file_path, O_RDONLY);
     if(cmdline_fd == -1) {
@@ -648,7 +648,7 @@ static void command_sync(int argc, char **argv, char *rss_config_dir, char *html
     signal(SIGINT, automedia_pid_signal_handler);
 
     char process_pid_str[32];
-    sprintf(process_pid_str, "%d", getpid());
+    snprintf(process_pid_str, sizeof(process_pid_str), "%d", getpid());
     int process_pid_str_len = strlen(process_pid_str);
     if(write(pid_file, process_pid_str, process_pid_str_len) != process_pid_str_len) {
         fprintf(stderr, "Failed to write pid to %s\n", automedia_pid_path);
diff --git a/src/rss.c b/src/rss.c
index 8e6cfe2..dadbacd 100644
--- a/src/rss.c
+++ b/src/rss.c
@@ -79,7 +79,7 @@ static char* string_substr_before_tag_end(char *str, const char *tag) {
     return tag_p;
 }
 
-typedef int (*RssParseCallback)(char *title, char *link, void *userdata);
+typedef int (*RssParseCallback)(const char *title, const char *link, void *userdata);
 
 static int parse_rss(char *str, char **rss_title_str, RssParseCallback parse_callback, void *userdata) {
     *rss_title_str = NULL;
@@ -150,19 +150,12 @@ static int parse_rss(char *str, char **rss_title_str, RssParseCallback parse_cal
     }
 }
 
-typedef struct {
-    const char *start_after;
-    int found_start_after;
-    const char *start_after_url;
-} RssParseUserdata;
-
-static int rss_parse_add_callback(char *title, char *link, void *userdata) {
-    RssParseUserdata *rss_parse_userdata = userdata;
-    if(rss_parse_userdata->start_after && strcmp(rss_parse_userdata->start_after, title) == 0) {
-        rss_parse_userdata->found_start_after = 1;
-        rss_parse_userdata->start_after_url = link;
-        return 1;
-    }
+static int rss_parse_add_callback(const char *title, const char *link, void *userdata) {
+    Buffer *download_items_buffer = userdata;
+    DownloadItemsData download_items_data;
+    download_items_data.title = title;
+    download_items_data.link = link;
+    buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data));
     return 0;
 }
 
@@ -303,8 +296,6 @@ static int get_rss_url_from_episode_info(const char *episode_name, EpisodeInfo *
     return 0;
 }
 
-/* TODO: Fix the remove() calls. They wont work since they are not recursive and the directories has files in them */
-/* Same for add_html */
 int add_rss(const char *name, char *url, char *rss_config_dir, const char *start_after) {
     int result = 0;
     char rss_url[4096];
@@ -341,22 +332,32 @@ int add_rss(const char *name, char *url, char *rss_config_dir, const char *start
         }
     }
 
-    RssParseUserdata rss_parse_userdata;
-    rss_parse_userdata.start_after = start_after;
-    rss_parse_userdata.found_start_after = 0;
-    rss_parse_userdata.start_after_url = NULL;
+    Buffer download_items_buffer;
+    buffer_init(&download_items_buffer);
 
     char *rss_title = NULL;
-    result = parse_rss(buffer.data, &rss_title, rss_parse_add_callback, &rss_parse_userdata);
+    result = parse_rss(buffer.data, &rss_title, rss_parse_add_callback, &download_items_buffer);
     if(result != 0) {
         fprintf(stderr, "Failed to parse rss for url: %s\n", url);
         goto cleanup;
     }
 
-    if(start_after && !rss_parse_userdata.found_start_after) {
-        fprintf(stderr, "Failed to find %s in rss %s", start_after, url);
-        result = -1;
-        goto cleanup;
+    DownloadItemsData *download_items_start = NULL;
+    if(start_after) {
+        DownloadItemsData *download_items_it = buffer_begin(&download_items_buffer);
+        DownloadItemsData *download_items_end = buffer_end(&download_items_buffer);
+        for(; download_items_it != download_items_end; ++download_items_it) {
+            if(strcmp(start_after, download_items_it->title) == 0) {
+                download_items_start = download_items_it;
+                break;
+            }
+        }
+
+        if(!download_items_start) {
+            fprintf(stderr, "Failed to find %s in rss %s\n", start_after, url);
+            result = -1;
+            goto cleanup;
+        }
     }
 
     if(!name) {
@@ -399,36 +400,37 @@ int add_rss(const char *name, char *url, char *rss_config_dir, const char *start
     result = create_lock_file(in_progress_filepath);
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/.in_progress\n", rss_tracked_dir);
-        remove(rss_tracked_dir);
+        remove_recursive(rss_tracked_dir);
         goto cleanup;
     }
 
     result = file_overwrite_in_dir(rss_tracked_dir, "link", url, strlen(url));
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/link\n", rss_tracked_dir);
-        remove(rss_tracked_dir);
+        remove_recursive(rss_tracked_dir);
         goto cleanup;
     }
 
     char updated[32];
-    sprintf(updated, "%ld", time(NULL));
+    snprintf(updated, sizeof(updated), "%ld", time(NULL));
     result = file_overwrite_in_dir(rss_tracked_dir, "updated", updated, strlen(updated));
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/updated\n", rss_tracked_dir);
-        remove(rss_tracked_dir);
+        remove_recursive(rss_tracked_dir);
         goto cleanup;
     }
 
-    result = write_plugin_json_to_file(rss_tracked_dir, "data", url, updated, start_after, rss_parse_userdata.start_after_url, NULL);
+    size_t num_download_items = download_items_start ? (((DownloadItemsData*)buffer_end(&download_items_buffer)) - download_items_start) : 0;
+    result = write_plugin_json_to_file(rss_tracked_dir, "data", url, updated, download_items_start, num_download_items, NULL);
     if(result != 0) {
         fprintf(stderr, "Failed to create %s/data\n", rss_tracked_dir);
-        remove(rss_tracked_dir);
+        remove_recursive(rss_tracked_dir);
         goto cleanup;
     }
 
-    remove(in_progress_filepath);
-
     cleanup:
+    remove(in_progress_filepath);
+    buffer_deinit(&download_items_buffer);
     buffer_deinit(&buffer);
     return result;
 }
@@ -460,7 +462,7 @@ typedef struct {
     Buffer *download_items_buffer;
 } RssParseSyncData;
 
-static int rss_parse_sync_callback(char *title, char *link, void *userdata) {
+static int rss_parse_sync_callback(const char *title, const char *link, void *userdata) {
     RssParseSyncData *rss_parse_sync_data = userdata;
     if(is_item_already_downloaded(title, link, rss_parse_sync_data->tracked_rss))
         return 1;
@@ -551,7 +553,7 @@ int sync_rss(TrackedRss *tracked_rss, TransmissionSession *transmission_session,
     }
 
     char updated[32];
-    sprintf(updated, "%ld", time(NULL));
+    snprintf(updated, sizeof(updated), "%ld", time(NULL));
     strcat(rss_tracked_dir, tracked_rss->title);
     result = file_overwrite_in_dir(rss_tracked_dir, "synced", updated, strlen(updated));
     if(result != 0) {
diff --git a/src/rss.h b/src/rss.h
index 87d064f..a9cd65b 100644
--- a/src/rss.h
+++ b/src/rss.h
@@ -10,6 +10,7 @@ typedef struct {
     cJSON *json_data;
 } TrackedRss;
 
+/* Modifies @rss_config_dir */
 int add_rss(const char *name, char *url, char *rss_config_dir, const char *start_after);
 int sync_rss(TrackedRss *tracked_rss, struct TransmissionSession *transmission_session, char *rss_config_dir);
 
diff --git a/src/rss_html_common.c b/src/rss_html_common.c
index c72273c..567671e 100644
--- a/src/rss_html_common.c
+++ b/src/rss_html_common.c
@@ -7,7 +7,7 @@
 #include <time.h>
 #include <assert.h>
 
-int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, const char *start_after, const char *start_after_url, const char *plugin_name) {
+int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, DownloadItemsData *prev_download_items, size_t num_prev_download_items, const char *plugin_name) {
     int result = 0;
 
     cJSON *json_body = cJSON_CreateObject();
@@ -27,17 +27,20 @@ int write_plugin_json_to_file(const char *dir, const char *filename, const char
         goto cleanup;
     }
 
-    if(start_after) {
+    time_t time_now = time(NULL);
+    for(size_t i = 0; i < num_prev_download_items; ++i) {
         cJSON *downloaded_item_json = cJSON_CreateObject();
         if(!downloaded_item_json) {
             result = -1;
             goto cleanup;
         }
 
-        cJSON_AddStringToObject(downloaded_item_json, "title", start_after);
-        cJSON_AddStringToObject(downloaded_item_json, "time", updated);
-        if(start_after_url)
-            cJSON_AddStringToObject(downloaded_item_json, "url", start_after_url);
+        char item_created_timestamp_fake[32];
+        snprintf(item_created_timestamp_fake, sizeof(item_created_timestamp_fake), "%ld", time_now - i);
+
+        cJSON_AddStringToObject(downloaded_item_json, "title", prev_download_items[i].title);
+        cJSON_AddStringToObject(downloaded_item_json, "time", item_created_timestamp_fake);
+        cJSON_AddStringToObject(downloaded_item_json, "url", prev_download_items[i].link);
 
         cJSON_AddItemToArray(downloaded_json, downloaded_item_json);
     }
@@ -57,12 +60,22 @@ int write_plugin_json_to_file(const char *dir, const char *filename, const char
     return result;
 }
 
+static long timestamps_get_max(long *timestamps, size_t num_timestamps) {
+    long max_timestamp = 0;
+    for(size_t i = 0; i < num_timestamps; ++i) {
+        long timestamp = timestamps[i];
+        if(timestamp > max_timestamp)
+            max_timestamp = timestamp;
+    }
+    return max_timestamp;
+}
+
 /* TODO: If this fails in the middle, recover and update the next time somehow */
 int tracked_item_update_latest(TrackedItem *tracked_item, char *tracked_dir, DownloadItemsData **download_items, char **filenames, long *timestamps, int num_download_items) {
     if(num_download_items == 0)
         return 0;
 
-    assert(num_download_items <= MAX_UPDATE_ITEMS);
+    assert(download_items);
     assert(timestamps);
     int tracked_dir_len = strlen(tracked_dir);
     int result = 0;
@@ -72,7 +85,7 @@ int tracked_item_update_latest(TrackedItem *tracked_item, char *tracked_dir, Dow
     
     char updated[32];
     assert(sizeof(time_t) == sizeof(long));
-    sprintf(updated, "%ld", timestamps[num_download_items - 1]);
+    snprintf(updated, sizeof(updated), "%ld", timestamps_get_max(timestamps, num_download_items));
     int updated_len = strlen(updated);
     result = file_overwrite_in_dir(item_filepath, "updated", updated, updated_len);
     if(result != 0) {
diff --git a/src/rss_html_common.h b/src/rss_html_common.h
index 085ddc1..cc22b24 100644
--- a/src/rss_html_common.h
+++ b/src/rss_html_common.h
@@ -1,6 +1,8 @@
 #ifndef RSS_HTML_COMMON_H
 #define RSS_HTML_COMMON_H
 
+#include <stddef.h>
+
 #define MAX_UPDATE_ITEMS 10
 
 typedef struct cJSON cJSON;
@@ -11,20 +13,18 @@ typedef struct {
 } DownloadItemsData;
 
 typedef struct {
-    char *title;
-    char *link;
+    const char *title;
+    const char *link;
     cJSON *json_data;
 } TrackedItem;
 
-int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, const char *start_after, const char *start_after_url, const char *plugin_name);
+int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, DownloadItemsData *prev_download_items, size_t num_prev_download_items, const char *plugin_name);
 
 /*
-    @num_download_items can't be more than MAX_UPDATE_ITEMS.
-    @filenames can be NULL, in which case filenames are not stored for items. This is the case for html items.
-*/
-/*
     Note: tracked_item.json_data becomes invalid after this call.
+    @filenames can be NULL, in which case filenames are not stored for items. This is the case for html items.
     @tracked_dir is also modified and then restored at the end.
+    @download_items and @timestamps both need to be @num_download_items long. If @filenames is not NULL, then it also has to be @num_download_items long.
 */
 int tracked_item_update_latest(TrackedItem *tracked_item, char *tracked_dir, DownloadItemsData **download_items, char **filenames, long *timestamps, int num_download_items);