From 9946c0363648b44d396b07d8a1a4557c568edc88 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Wed, 15 Jul 2020 16:49:36 +0200 Subject: Implement html sync, fix rss sync --- src/html.c | 234 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 211 insertions(+), 23 deletions(-) (limited to 'src/html.c') diff --git a/src/html.c b/src/html.c index 02982d6..3159500 100644 --- a/src/html.c +++ b/src/html.c @@ -4,6 +4,7 @@ #include "buffer.h" #include "stringutils.h" #include "rss_html_common.h" +#include "main.h" #include "json.h" #include #include @@ -11,6 +12,8 @@ #include #include #include +#include +#include #include static int str_starts_with(const char *str, int len, const char *substr, int substr_len) { @@ -48,9 +51,38 @@ static int url_extract_domain(const char *url, char *domain, int domain_len) { return 0; } +/* + The plugin should print the names and urls of each item (chapter for manga) and the output list should stop when an + item matches any item in the input. The output should be sorted from newest to oldest. + + The input should be in this format: + [ + { + "title": "Example name", + "url": "https://example.com" + }, + { + "title": "Another item", + "url": "https://another.url.com" + } + ] + + And the output should be in this format: + [ + { + "name": "Example name", + "url": "https://example.com" + }, + { + "name": "Another item", + "url": "https://another.url.com" + } + ] + + TODO: Rename input "title" to "url", to make input and output match (easier to test with). +*/ typedef int (*PluginListCallback)(const char *name, const char *url, void *userdata); -static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, const char *latest, PluginListCallback callback, void *userdata) { - (void)latest; +static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, struct json_array_s *downloaded_items, PluginListCallback callback, void *userdata) { int result; Buffer buffer; buffer_init(&buffer); @@ -65,6 +97,34 @@ static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, goto err_cleanup; } + if(downloaded_items) { + struct json_value_s downloaded_items_value; + downloaded_items_value.payload = downloaded_items; + downloaded_items_value.type = json_type_array; + size_t json_output_len = 0; + void *json_body_str = json_write_minified(&downloaded_items_value, &json_output_len); + if(!json_body_str) { + fprintf(stderr, "Failed to convert downloaded items to json\n"); + if(process_id != -1) + kill(process_id, SIGKILL); + close(stdin_file); + close(stdout_file); + goto err_cleanup; + } + + /* This is a bug in the json library */ + json_output_len = strlen(json_body_str); + if(write(stdin_file, json_body_str, json_output_len) != (ssize_t)json_output_len) { + fprintf(stderr, "Failed to write all bytes to plugin list\n"); + if(process_id != -1) + kill(process_id, SIGKILL); + close(stdin_file); + close(stdout_file); + goto err_cleanup; + } + free(json_body_str); + } + result = program_wait_until_exit(process_id, stdin_file, stdout_file, program_buffer_write_callback, &buffer); if(result != 0) { fprintf(stderr, "Failed to launch plugin list for plugin %s\n", basename(plugin_filepath)); @@ -76,6 +136,7 @@ static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, fprintf(stderr, "Failed to load plugin %s list output as json\n", basename(plugin_filepath)); goto err_cleanup; } + buffer_deinit(&buffer); struct json_array_s *json_root_array = json_value_as_array(json_root); if(!json_root_array) { @@ -108,7 +169,6 @@ static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, break; } - buffer_deinit(&buffer); return json_root; err_cleanup: @@ -123,7 +183,6 @@ typedef struct { } PluginListUserdata; static int plugin_list_callback(const char *name, const char *url, void *userdata) { - /*fprintf(stderr, "name: |%s|, url: |%s|\n", name, url);*/ PluginListUserdata *plugin_list_userdata = userdata; if(plugin_list_userdata->start_after && strcmp(plugin_list_userdata->start_after, name) == 0) { plugin_list_userdata->found_start_after = 1; @@ -133,6 +192,31 @@ static int plugin_list_callback(const char *name, const char *url, void *userdat return 0; } +/* Store result in @plugin_filepath */ +static int get_plugin_filepath(const char *program_dir, const char *plugin_name, char *plugin_filepath) { + const char *path_components[] = { program_dir, "plugins" }; + path_join(plugin_filepath, path_components, 2); + if(file_exists(plugin_filepath) != 0) { + strcpy(plugin_filepath, "/usr/share/automedia/plugins"); + if(file_exists(plugin_filepath) != 0) { + fprintf(stderr, "Failed to find plugins directory\n"); + return -1; + } + } + + strcat(plugin_filepath, "/"); + strcat(plugin_filepath, plugin_name); + if(file_exists(plugin_filepath) != 0) { + strcat(plugin_filepath, ".py"); + if(file_exists(plugin_filepath) != 0) { + fprintf(stderr, "Plugin doesn't exist: %s\n", plugin_name); + return -1; + } + } + + return 0; +} + int add_html(const char *name, const char *url, char *html_config_dir, char *program_dir, const char *start_after) { int result = 0; @@ -152,26 +236,9 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro return -1; } - const char *path_components[] = { program_dir, "plugins" }; char domain_plugin_path[PATH_MAX]; - path_join(domain_plugin_path, path_components, 2); - if(file_exists(domain_plugin_path) != 0) { - strcpy(domain_plugin_path, "/usr/share/automedia/plugins"); - if(file_exists(domain_plugin_path) != 0) { - fprintf(stderr, "Failed to find plugins directory\n"); - return -1; - } - } - - strcat(domain_plugin_path, "/"); - strcat(domain_plugin_path, domain); - if(file_exists(domain_plugin_path) != 0) { - strcat(domain_plugin_path, ".py"); - if(file_exists(domain_plugin_path) != 0) { - fprintf(stderr, "Plugin doesn't exist: %s\n", domain); - return -1; - } - } + if(get_plugin_filepath(program_dir, domain, domain_plugin_path) != 0) + return -1; PluginListUserdata plugin_list_userdata; plugin_list_userdata.start_after = start_after; @@ -255,3 +322,124 @@ int add_html(const char *name, const char *url, char *html_config_dir, char *pro free(json_root); return result; } + +static int plugin_list_sync_callback(const char *name, const char *url, void *userdata) { + Buffer *download_items_buffer = userdata; + DownloadItemsData download_items_data; + download_items_data.title = name; + download_items_data.link = url; + buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data)); + return 0; +} + +static int download_html_items_in_reverse(const char *plugin_filepath, Buffer *download_items_buffer, TrackedHtml *tracked_html, char *html_tracked_dir, const char *download_dir) { + int result = 0; + DownloadItemsData *added_download_items[MAX_UPDATE_ITEMS]; + + char item_dir[PATH_MAX]; + const char *path_components[] = { download_dir, tracked_html->title }; + int item_dir_len = path_join(item_dir, path_components, 2); + + Buffer json_element_buffer; + buffer_init(&json_element_buffer); + + DownloadItemsData *download_items_it = buffer_end(download_items_buffer); + DownloadItemsData *download_items_end = buffer_begin(download_items_buffer); + download_items_it--; + download_items_end--; + int download_item_index = 0; + for(; download_items_it != download_items_end && download_item_index < MAX_UPDATE_ITEMS && is_program_running(); --download_items_it) { + item_dir[item_dir_len] = '/'; + strcpy(item_dir + item_dir_len + 1, download_items_it->title); + if(create_directory_recursive(item_dir) != 0) { + fprintf(stderr, "Failed to create directory for html item: %s\n", download_items_it->title); + result = -1; + break; + } + + /* TODO: Make asynchronous */ + const char *args[] = { plugin_filepath, "download", download_items_it->link, NULL }; + result = program_exec(args, NULL, NULL); + if(result != 0) + fprintf(stderr, "Failed while downloading html, url: %s\n", download_items_it->link); + + char notify_msg[PATH_MAX]; + const char *path_components[] = { tracked_html->title, download_items_it->title }; + path_join(notify_msg, path_components, 2); + const char *notify_args[] = { "notify-send", "-u", result == 0 ? "normal" : "critical", "--", result == 0 ? "Download finished" : "Download failed", notify_msg, NULL }; + program_exec(notify_args, NULL, NULL); + + if(result != 0) + break; + + added_download_items[download_item_index] = download_items_it; + ++download_item_index; + } + + TrackedItem tracked_item; + tracked_item.title = tracked_html->title; + tracked_item.link = tracked_html->link; + tracked_item.json_data = tracked_html->json_data; + result = tracked_item_update_latest(&tracked_item, html_tracked_dir, added_download_items, NULL, download_item_index); + + buffer_deinit(&json_element_buffer); + return result; +} + +/* TODO: Make asynchronous. Right now this will only complete when the whole chapter download completes */ +int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download_dir, char *html_config_dir) { + /* TODO: This can be cached */ + int html_config_dir_len = strlen(html_config_dir); + + fprintf(stderr, "Syncing %s\n", tracked_html->title); + + char plugin_filepath[PATH_MAX]; + /* This will check with ${tracked_html->plugin}.py as well, but that is fine */ + if(get_plugin_filepath(program_dir, tracked_html->plugin, plugin_filepath) != 0) + return -1; + + struct json_value_s *downloaded_items = json_object_get_field_by_name(tracked_html->json_data, "downloaded"); + struct json_array_s *downloaded_items_array = NULL; + if(downloaded_items) { + downloaded_items_array = json_value_as_array(downloaded_items); + if(!downloaded_items_array) { + fprintf(stderr, "Corrupt json for html item: %s\n", tracked_html->title); + return -1; + } + } + + Buffer download_items_buffer; + buffer_init(&download_items_buffer); + + int result = 0; + + struct json_value_s *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items_array, plugin_list_sync_callback, &download_items_buffer); + if(!json_root) { + result = -1; + goto cleanup; + } + + char *html_tracked_dir = html_config_dir; + strcat(html_tracked_dir, "/tracked/"); + + result = download_html_items_in_reverse(plugin_filepath, &download_items_buffer, tracked_html, html_tracked_dir, download_dir); + if(result != 0) { + fprintf(stderr, "Failed while download html item for url: %s\n", tracked_html->link); + goto cleanup; + } + + char updated[32]; + sprintf(updated, "%ld", time(NULL)); + strcat(html_tracked_dir, tracked_html->title); + result = file_overwrite_in_dir(html_tracked_dir, "synced", updated, strlen(updated)); + if(result != 0) { + fprintf(stderr, "Failed to update %s/synced\n", html_tracked_dir); + goto cleanup; + } + + cleanup: + free(json_root); + buffer_deinit(&download_items_buffer); + html_config_dir[html_config_dir_len] = '\0'; + return result; +} -- cgit v1.2.3