From fee89c4afdde4dacee51a763bc4d931320a9d69d Mon Sep 17 00:00:00 2001 From: dec05eba Date: Tue, 15 Jun 2021 09:28:35 +0200 Subject: Add all items starting at start-after to the download list in the data json file. This makes downloading more robust if title/url is changes for any item --- src/html.c | 94 +++++++++++++++++++++++++-------------------------- src/html.h | 1 + src/main.c | 6 ++-- src/rss.c | 72 ++++++++++++++++++++------------------- src/rss.h | 1 + src/rss_html_common.c | 29 +++++++++++----- src/rss_html_common.h | 14 ++++---- 7 files changed, 116 insertions(+), 101 deletions(-) (limited to 'src') diff --git a/src/html.c b/src/html.c index 572a396..6e27d13 100644 --- a/src/html.c +++ b/src/html.c @@ -166,19 +166,12 @@ static cJSON* plugin_list(char *plugin_filepath, const char *url, cJSON *downloa return NULL; } -typedef struct { - const char *start_after; - int found_start_after; - const char *start_after_url; -} PluginListUserdata; - -static int plugin_list_callback(const char *name, const char *url, void *userdata) { - PluginListUserdata *plugin_list_userdata = userdata; - if(plugin_list_userdata->start_after && strcmp(plugin_list_userdata->start_after, name) == 0) { - plugin_list_userdata->found_start_after = 1; - plugin_list_userdata->start_after_url = url; - return 1; - } +static int plugin_list_append_item_callback(const char *name, const char *url, void *userdata) { + Buffer *download_items_buffer = userdata; + DownloadItemsData download_items_data; + download_items_data.title = name; + download_items_data.link = url; + buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data)); return 0; } @@ -230,21 +223,6 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro if(get_plugin_filepath(program_dir, domain, domain_plugin_path) != 0) return -1; - PluginListUserdata plugin_list_userdata; - plugin_list_userdata.start_after = start_after; - plugin_list_userdata.found_start_after = 0; - plugin_list_userdata.start_after_url = NULL; - - cJSON *json_root = plugin_list(domain_plugin_path, url, NULL, plugin_list_callback, &plugin_list_userdata); - if(!json_root) - return -1; - - if(start_after && !plugin_list_userdata.found_start_after) { - fprintf(stderr, "Failed to find %s in html %s\n", start_after, url); - result = -1; - goto cleanup; - } - char *html_tracked_dir = html_config_dir; strcat(html_tracked_dir, "/tracked/"); strcat(html_tracked_dir, name); @@ -253,12 +231,40 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro strcpy(in_progress_filepath, html_tracked_dir); strcat(in_progress_filepath, "/.in_progress"); + Buffer download_items_buffer; + buffer_init(&download_items_buffer); + cJSON *json_root = NULL; + if(file_exists(html_tracked_dir) == 0 && file_exists(in_progress_filepath) != 0) { fprintf(stderr, "You are already tracking %s\n", url); result = -1; goto cleanup; } + json_root = plugin_list(domain_plugin_path, url, NULL, plugin_list_append_item_callback, &download_items_buffer); + if(!json_root) { + result = -1; + goto cleanup; + } + + DownloadItemsData *download_items_start = NULL; + if(start_after) { + DownloadItemsData *download_items_it = buffer_begin(&download_items_buffer); + DownloadItemsData *download_items_end = buffer_end(&download_items_buffer); + for(; download_items_it != download_items_end; ++download_items_it) { + if(strcmp(start_after, download_items_it->title) == 0) { + download_items_start = download_items_it; + break; + } + } + + if(!download_items_start) { + fprintf(stderr, "Failed to find %s in html %s\n", start_after, url); + result = -1; + goto cleanup; + } + } + result = create_directory_recursive(html_tracked_dir); if(result != 0) { fprintf(stderr, "Failed to create %s, error: %s\n", html_tracked_dir, strerror(result)); @@ -273,14 +279,14 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro result = create_lock_file(in_progress_filepath); if(result != 0) { fprintf(stderr, "Failed to create %s/.in_progress\n", html_tracked_dir); - remove(html_tracked_dir); + remove_recursive(html_tracked_dir); goto cleanup; } result = file_overwrite_in_dir(html_tracked_dir, "link", url, strlen(url)); if(result != 0) { fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir); - remove(html_tracked_dir); + remove_recursive(html_tracked_dir); goto cleanup; } @@ -288,43 +294,35 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro result = file_overwrite_in_dir(html_tracked_dir, "plugin", plugin_name, strlen(plugin_name)); if(result != 0) { fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir); - remove(html_tracked_dir); + remove_recursive(html_tracked_dir); goto cleanup; } char updated[32]; assert(sizeof(time_t) == sizeof(long)); - sprintf(updated, "%ld", time(NULL)); + snprintf(updated, sizeof(updated), "%ld", time(NULL)); result = file_overwrite_in_dir(html_tracked_dir, "updated", updated, strlen(updated)); if(result != 0) { fprintf(stderr, "Failed to create %s/updated\n", html_tracked_dir); - remove(html_tracked_dir); + remove_recursive(html_tracked_dir); goto cleanup; } - result = write_plugin_json_to_file(html_tracked_dir, "data", url, updated, start_after, plugin_list_userdata.start_after_url, plugin_name); + size_t num_download_items = download_items_start ? (((DownloadItemsData*)buffer_end(&download_items_buffer)) - download_items_start) : 0; + result = write_plugin_json_to_file(html_tracked_dir, "data", url, updated, download_items_start, num_download_items, plugin_name); if(result != 0) { fprintf(stderr, "Failed to create %s/data\n", html_tracked_dir); - remove(html_tracked_dir); + remove_recursive(html_tracked_dir); goto cleanup; } - remove(in_progress_filepath); - cleanup: + remove(in_progress_filepath); + buffer_deinit(&download_items_buffer); cJSON_Delete(json_root); return result; } -static int plugin_list_sync_callback(const char *name, const char *url, void *userdata) { - Buffer *download_items_buffer = userdata; - DownloadItemsData download_items_data; - download_items_data.title = name; - download_items_data.link = url; - buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data)); - return 0; -} - static int download_html_items_in_reverse(const char *plugin_filepath, Buffer *download_items_buffer, TrackedHtml *tracked_html, char *html_tracked_dir, const char *download_dir) { int result = 0; DownloadItemsData *added_download_items[MAX_UPDATE_ITEMS]; @@ -410,7 +408,7 @@ int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download int result = 0; - cJSON *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items, plugin_list_sync_callback, &download_items_buffer); + cJSON *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items, plugin_list_append_item_callback, &download_items_buffer); if(!json_root) { result = -1; goto cleanup; @@ -426,7 +424,7 @@ int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download } char updated[32]; - sprintf(updated, "%ld", time(NULL)); + snprintf(updated, sizeof(updated), "%ld", time(NULL)); strcat(html_tracked_dir, tracked_html->title); result = file_overwrite_in_dir(html_tracked_dir, "synced", updated, strlen(updated)); if(result != 0) { diff --git a/src/html.h b/src/html.h index 327a39e..b604d48 100644 --- a/src/html.h +++ b/src/html.h @@ -10,6 +10,7 @@ typedef struct { cJSON *json_data; } TrackedHtml; +/* Modifies @html_config_dir */ int add_html(const char *name, const char *url, char *html_config_dir, char *program_dir, const char *start_after); int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download_dir, char *html_config_dir); diff --git a/src/main.c b/src/main.c index 04ae6fe..b47e24a 100644 --- a/src/main.c +++ b/src/main.c @@ -566,8 +566,8 @@ static int cmdline_contains_str(const char *cmdline, int cmdline_len, const char static int proc_read_cmdline(const char *pid_str, char *cmdline_data, int cmdline_data_size, int *cmdline_data_size_output) { assert(cmdline_data_size > 0); - char cmdline_file_path[128]; - sprintf(cmdline_file_path, "/proc/%s/cmdline", pid_str); + char cmdline_file_path[256]; + snprintf(cmdline_file_path, sizeof(cmdline_file_path), "/proc/%s/cmdline", pid_str); int cmdline_fd = open(cmdline_file_path, O_RDONLY); if(cmdline_fd == -1) { @@ -648,7 +648,7 @@ static void command_sync(int argc, char **argv, char *rss_config_dir, char *html signal(SIGINT, automedia_pid_signal_handler); char process_pid_str[32]; - sprintf(process_pid_str, "%d", getpid()); + snprintf(process_pid_str, sizeof(process_pid_str), "%d", getpid()); int process_pid_str_len = strlen(process_pid_str); if(write(pid_file, process_pid_str, process_pid_str_len) != process_pid_str_len) { fprintf(stderr, "Failed to write pid to %s\n", automedia_pid_path); diff --git a/src/rss.c b/src/rss.c index 8e6cfe2..dadbacd 100644 --- a/src/rss.c +++ b/src/rss.c @@ -79,7 +79,7 @@ static char* string_substr_before_tag_end(char *str, const char *tag) { return tag_p; } -typedef int (*RssParseCallback)(char *title, char *link, void *userdata); +typedef int (*RssParseCallback)(const char *title, const char *link, void *userdata); static int parse_rss(char *str, char **rss_title_str, RssParseCallback parse_callback, void *userdata) { *rss_title_str = NULL; @@ -150,19 +150,12 @@ static int parse_rss(char *str, char **rss_title_str, RssParseCallback parse_cal } } -typedef struct { - const char *start_after; - int found_start_after; - const char *start_after_url; -} RssParseUserdata; - -static int rss_parse_add_callback(char *title, char *link, void *userdata) { - RssParseUserdata *rss_parse_userdata = userdata; - if(rss_parse_userdata->start_after && strcmp(rss_parse_userdata->start_after, title) == 0) { - rss_parse_userdata->found_start_after = 1; - rss_parse_userdata->start_after_url = link; - return 1; - } +static int rss_parse_add_callback(const char *title, const char *link, void *userdata) { + Buffer *download_items_buffer = userdata; + DownloadItemsData download_items_data; + download_items_data.title = title; + download_items_data.link = link; + buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data)); return 0; } @@ -303,8 +296,6 @@ static int get_rss_url_from_episode_info(const char *episode_name, EpisodeInfo * return 0; } -/* TODO: Fix the remove() calls. They wont work since they are not recursive and the directories has files in them */ -/* Same for add_html */ int add_rss(const char *name, char *url, char *rss_config_dir, const char *start_after) { int result = 0; char rss_url[4096]; @@ -341,22 +332,32 @@ int add_rss(const char *name, char *url, char *rss_config_dir, const char *start } } - RssParseUserdata rss_parse_userdata; - rss_parse_userdata.start_after = start_after; - rss_parse_userdata.found_start_after = 0; - rss_parse_userdata.start_after_url = NULL; + Buffer download_items_buffer; + buffer_init(&download_items_buffer); char *rss_title = NULL; - result = parse_rss(buffer.data, &rss_title, rss_parse_add_callback, &rss_parse_userdata); + result = parse_rss(buffer.data, &rss_title, rss_parse_add_callback, &download_items_buffer); if(result != 0) { fprintf(stderr, "Failed to parse rss for url: %s\n", url); goto cleanup; } - if(start_after && !rss_parse_userdata.found_start_after) { - fprintf(stderr, "Failed to find %s in rss %s", start_after, url); - result = -1; - goto cleanup; + DownloadItemsData *download_items_start = NULL; + if(start_after) { + DownloadItemsData *download_items_it = buffer_begin(&download_items_buffer); + DownloadItemsData *download_items_end = buffer_end(&download_items_buffer); + for(; download_items_it != download_items_end; ++download_items_it) { + if(strcmp(start_after, download_items_it->title) == 0) { + download_items_start = download_items_it; + break; + } + } + + if(!download_items_start) { + fprintf(stderr, "Failed to find %s in rss %s\n", start_after, url); + result = -1; + goto cleanup; + } } if(!name) { @@ -399,36 +400,37 @@ int add_rss(const char *name, char *url, char *rss_config_dir, const char *start result = create_lock_file(in_progress_filepath); if(result != 0) { fprintf(stderr, "Failed to create %s/.in_progress\n", rss_tracked_dir); - remove(rss_tracked_dir); + remove_recursive(rss_tracked_dir); goto cleanup; } result = file_overwrite_in_dir(rss_tracked_dir, "link", url, strlen(url)); if(result != 0) { fprintf(stderr, "Failed to create %s/link\n", rss_tracked_dir); - remove(rss_tracked_dir); + remove_recursive(rss_tracked_dir); goto cleanup; } char updated[32]; - sprintf(updated, "%ld", time(NULL)); + snprintf(updated, sizeof(updated), "%ld", time(NULL)); result = file_overwrite_in_dir(rss_tracked_dir, "updated", updated, strlen(updated)); if(result != 0) { fprintf(stderr, "Failed to create %s/updated\n", rss_tracked_dir); - remove(rss_tracked_dir); + remove_recursive(rss_tracked_dir); goto cleanup; } - result = write_plugin_json_to_file(rss_tracked_dir, "data", url, updated, start_after, rss_parse_userdata.start_after_url, NULL); + size_t num_download_items = download_items_start ? (((DownloadItemsData*)buffer_end(&download_items_buffer)) - download_items_start) : 0; + result = write_plugin_json_to_file(rss_tracked_dir, "data", url, updated, download_items_start, num_download_items, NULL); if(result != 0) { fprintf(stderr, "Failed to create %s/data\n", rss_tracked_dir); - remove(rss_tracked_dir); + remove_recursive(rss_tracked_dir); goto cleanup; } - remove(in_progress_filepath); - cleanup: + remove(in_progress_filepath); + buffer_deinit(&download_items_buffer); buffer_deinit(&buffer); return result; } @@ -460,7 +462,7 @@ typedef struct { Buffer *download_items_buffer; } RssParseSyncData; -static int rss_parse_sync_callback(char *title, char *link, void *userdata) { +static int rss_parse_sync_callback(const char *title, const char *link, void *userdata) { RssParseSyncData *rss_parse_sync_data = userdata; if(is_item_already_downloaded(title, link, rss_parse_sync_data->tracked_rss)) return 1; @@ -551,7 +553,7 @@ int sync_rss(TrackedRss *tracked_rss, TransmissionSession *transmission_session, } char updated[32]; - sprintf(updated, "%ld", time(NULL)); + snprintf(updated, sizeof(updated), "%ld", time(NULL)); strcat(rss_tracked_dir, tracked_rss->title); result = file_overwrite_in_dir(rss_tracked_dir, "synced", updated, strlen(updated)); if(result != 0) { diff --git a/src/rss.h b/src/rss.h index 87d064f..a9cd65b 100644 --- a/src/rss.h +++ b/src/rss.h @@ -10,6 +10,7 @@ typedef struct { cJSON *json_data; } TrackedRss; +/* Modifies @rss_config_dir */ int add_rss(const char *name, char *url, char *rss_config_dir, const char *start_after); int sync_rss(TrackedRss *tracked_rss, struct TransmissionSession *transmission_session, char *rss_config_dir); diff --git a/src/rss_html_common.c b/src/rss_html_common.c index c72273c..567671e 100644 --- a/src/rss_html_common.c +++ b/src/rss_html_common.c @@ -7,7 +7,7 @@ #include #include -int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, const char *start_after, const char *start_after_url, const char *plugin_name) { +int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, DownloadItemsData *prev_download_items, size_t num_prev_download_items, const char *plugin_name) { int result = 0; cJSON *json_body = cJSON_CreateObject(); @@ -27,17 +27,20 @@ int write_plugin_json_to_file(const char *dir, const char *filename, const char goto cleanup; } - if(start_after) { + time_t time_now = time(NULL); + for(size_t i = 0; i < num_prev_download_items; ++i) { cJSON *downloaded_item_json = cJSON_CreateObject(); if(!downloaded_item_json) { result = -1; goto cleanup; } - cJSON_AddStringToObject(downloaded_item_json, "title", start_after); - cJSON_AddStringToObject(downloaded_item_json, "time", updated); - if(start_after_url) - cJSON_AddStringToObject(downloaded_item_json, "url", start_after_url); + char item_created_timestamp_fake[32]; + snprintf(item_created_timestamp_fake, sizeof(item_created_timestamp_fake), "%ld", time_now - i); + + cJSON_AddStringToObject(downloaded_item_json, "title", prev_download_items[i].title); + cJSON_AddStringToObject(downloaded_item_json, "time", item_created_timestamp_fake); + cJSON_AddStringToObject(downloaded_item_json, "url", prev_download_items[i].link); cJSON_AddItemToArray(downloaded_json, downloaded_item_json); } @@ -57,12 +60,22 @@ int write_plugin_json_to_file(const char *dir, const char *filename, const char return result; } +static long timestamps_get_max(long *timestamps, size_t num_timestamps) { + long max_timestamp = 0; + for(size_t i = 0; i < num_timestamps; ++i) { + long timestamp = timestamps[i]; + if(timestamp > max_timestamp) + max_timestamp = timestamp; + } + return max_timestamp; +} + /* TODO: If this fails in the middle, recover and update the next time somehow */ int tracked_item_update_latest(TrackedItem *tracked_item, char *tracked_dir, DownloadItemsData **download_items, char **filenames, long *timestamps, int num_download_items) { if(num_download_items == 0) return 0; - assert(num_download_items <= MAX_UPDATE_ITEMS); + assert(download_items); assert(timestamps); int tracked_dir_len = strlen(tracked_dir); int result = 0; @@ -72,7 +85,7 @@ int tracked_item_update_latest(TrackedItem *tracked_item, char *tracked_dir, Dow char updated[32]; assert(sizeof(time_t) == sizeof(long)); - sprintf(updated, "%ld", timestamps[num_download_items - 1]); + snprintf(updated, sizeof(updated), "%ld", timestamps_get_max(timestamps, num_download_items)); int updated_len = strlen(updated); result = file_overwrite_in_dir(item_filepath, "updated", updated, updated_len); if(result != 0) { diff --git a/src/rss_html_common.h b/src/rss_html_common.h index 085ddc1..cc22b24 100644 --- a/src/rss_html_common.h +++ b/src/rss_html_common.h @@ -1,6 +1,8 @@ #ifndef RSS_HTML_COMMON_H #define RSS_HTML_COMMON_H +#include + #define MAX_UPDATE_ITEMS 10 typedef struct cJSON cJSON; @@ -11,20 +13,18 @@ typedef struct { } DownloadItemsData; typedef struct { - char *title; - char *link; + const char *title; + const char *link; cJSON *json_data; } TrackedItem; -int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, const char *start_after, const char *start_after_url, const char *plugin_name); +int write_plugin_json_to_file(const char *dir, const char *filename, const char *url, const char *updated, DownloadItemsData *prev_download_items, size_t num_prev_download_items, const char *plugin_name); -/* - @num_download_items can't be more than MAX_UPDATE_ITEMS. - @filenames can be NULL, in which case filenames are not stored for items. This is the case for html items. -*/ /* Note: tracked_item.json_data becomes invalid after this call. + @filenames can be NULL, in which case filenames are not stored for items. This is the case for html items. @tracked_dir is also modified and then restored at the end. + @download_items and @timestamps both need to be @num_download_items long. If @filenames is not NULL, then it also has to be @num_download_items long. */ int tracked_item_update_latest(TrackedItem *tracked_item, char *tracked_dir, DownloadItemsData **download_items, char **filenames, long *timestamps, int num_download_items); -- cgit v1.2.3