Implement add_html

1 files changed, 264 insertions, 0 deletions

diff --git a/src/html.c b/src/html.c
new file mode 100644
index 0000000..de888a0
--- /dev/null
+++ b/src/html.c
@@ -0,0 +1,264 @@
+#include "html.h"
+#include "fileutils.h"
+#include "program.h"
+#include "buffer.h"
+#include "stringutils.h"
+#include "rss_html_common.h"
+#include "json.h"
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+#include <libgen.h>
+#include <signal.h>
+#include <time.h>
+
+static int str_starts_with(const char *str, int len, const char *substr, int substr_len) {
+    return len >= substr_len && memcmp(str, substr, substr_len) == 0;
+}
+
+static int url_extract_domain(const char *url, char *domain, int domain_len) {
+    int url_len = strlen(url);
+    if(str_starts_with(url, url_len, "http://", 7)) {
+        url += 7;
+        url_len -= 7;
+    } else if(str_starts_with(url, url_len, "https://", 8)) {
+        url += 8;
+        url_len -= 8;
+    }
+
+    if(str_starts_with(url, url_len, "www.", 4)) {
+        url += 4;
+        url_len -= 4;
+    }
+
+    char *end = strchr(url, '.');
+    if(end) {
+        int len = end - url;
+        if(len >= domain_len)
+            return -1;
+        memcpy(domain, url, len);
+        domain[len] = '\0';
+    } else {
+        if(url_len >= domain_len)
+            return -1;
+        memcpy(domain, url, url_len);
+        domain[url_len] = '\0';
+    }
+    return 0;
+}
+
+static struct json_value_s* json_object_get_field_by_name(struct json_object_s *json_obj, const char *name) {
+    struct json_object_element_s *obj_element = json_obj->start;
+    while(obj_element) {
+        if(strcmp(obj_element->name->string, name) == 0)
+            return obj_element->value;
+        obj_element = obj_element->next;
+    }
+    return NULL;
+}
+
+typedef int (*PluginListCallback)(const char *name, const char *url, void *userdata);
+static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, const char *latest, PluginListCallback callback, void *userdata) {
+    (void)latest;
+    int result;
+    Buffer buffer;
+    buffer_init(&buffer);
+
+    const char *args[] = { plugin_filepath, "list", url, NULL };
+    int process_id = -1;
+    int stdin_file = -1;
+    int stdout_file = -1;
+    result = program_exec_async(args, &process_id, &stdin_file, &stdout_file);
+    if(result != 0) {
+        fprintf(stderr, "Failed to launch plugin list for plugin %s\n", basename(plugin_filepath));
+        goto err_cleanup;
+    }
+
+    result = program_wait_until_exit(process_id, stdin_file, stdout_file, program_buffer_write_callback, &buffer);
+    if(result != 0) {
+        fprintf(stderr, "Failed to launch plugin list for plugin %s\n", basename(plugin_filepath));
+        goto err_cleanup;
+    }
+
+    /*fprintf(stderr, "plugin list: %s\n", (char*)buffer.data);*/
+
+    struct json_value_s *json_root = json_parse(buffer.data, buffer.size);
+    if(!json_root) {
+        fprintf(stderr, "Failed to load plugin %s list output as json\n", basename(plugin_filepath));
+        goto err_cleanup;
+    }
+
+    struct json_array_s *json_root_array = json_value_as_array(json_root);
+    if(!json_root_array) {
+        fprintf(stderr, "Failed to load plugin %s list output as json\n", basename(plugin_filepath));
+        free(json_root);
+        goto err_cleanup;
+    }
+
+    struct json_array_element_s *json_array_element = json_root_array->start;
+    for(; json_array_element; json_array_element = json_array_element->next) {
+        struct json_object_s *array_element_value = json_value_as_object(json_array_element->value);
+        if(!array_element_value)
+            continue;
+        
+        struct json_value_s *name_json = json_object_get_field_by_name(array_element_value, "name");
+        struct json_value_s *url_json = json_object_get_field_by_name(array_element_value, "url");
+        struct json_string_s *name_json_str = json_value_as_string(name_json);
+        struct json_string_s *url_json_str = json_value_as_string(url_json);
+        if(!name_json_str || !url_json_str)
+            continue;
+        
+        /* Cast from const to mutable variable because I can */
+        char *name = (char*)name_json_str->string;
+        string_replace(name, '/', '_');
+        name = strip(name);
+        if(callback(name, url_json_str->string, userdata) != 0)
+            break;
+    }
+
+    buffer_deinit(&buffer);
+    return json_root;
+
+    err_cleanup:
+    buffer_deinit(&buffer);
+    return NULL;
+}
+
+typedef struct {
+    const char *start_after;
+    int found_start_after;
+    const char *start_after_url;
+} PluginListUserdata;
+
+static int plugin_list_callback(const char *name, const char *url, void *userdata) {
+    /*fprintf(stderr, "name: |%s|, url: |%s|\n", name, url);*/
+    PluginListUserdata *plugin_list_userdata = userdata;
+    if(plugin_list_userdata->start_after && strcmp(plugin_list_userdata->start_after, name) == 0) {
+        plugin_list_userdata->found_start_after = 1;
+        plugin_list_userdata->start_after_url = url;
+        return 1;
+    }
+    return 0;
+}
+
+int add_html(const char *name, const char *url, char *html_config_dir, char *program_dir, const char *start_after) {
+    int result = 0;
+
+    if(!name || name[0] == '\0') {
+        fprintf(stderr, "Name not provided or empty\n");
+        return -1;
+    }
+
+    char domain[2086];
+    if(url_extract_domain(url, domain, sizeof(domain)) != 0) {
+        fprintf(stderr, "Url %s is too long\n", url);
+        return -1;
+    }
+
+    if(domain[0] == '\0') {
+        fprintf(stderr, "Invalid url: %s\n", url);
+        return -1;
+    }
+
+    const char *path_components[] = { program_dir, "plugins" };
+    char domain_plugin_path[PATH_MAX];
+    path_join(domain_plugin_path, path_components, 2);
+    if(file_exists(domain_plugin_path) != 0) {
+        strcpy(domain_plugin_path, "/usr/share/automedia/plugins");
+        if(file_exists(domain_plugin_path) != 0) {
+            fprintf(stderr, "Failed to find plugins directory\n");
+            return -1;
+        }
+    }
+
+    strcat(domain_plugin_path, "/");
+    strcat(domain_plugin_path, domain);
+    if(file_exists(domain_plugin_path) != 0) {
+        strcat(domain_plugin_path, ".py");
+        if(file_exists(domain_plugin_path) != 0) {
+            fprintf(stderr, "Plugin doesn't exist: %s\n", domain);
+            return -1;
+        }
+    }
+
+    PluginListUserdata plugin_list_userdata;
+    plugin_list_userdata.start_after = start_after;
+    plugin_list_userdata.found_start_after = 0;
+    plugin_list_userdata.start_after_url = NULL;
+
+    struct json_value_s *json_root = plugin_list(domain_plugin_path, url, NULL, plugin_list_callback, &plugin_list_userdata);
+    if(!json_root)
+        return -1;
+
+    if(start_after && !plugin_list_userdata.found_start_after) {
+        fprintf(stderr, "Failed to find %s in html %s\n", start_after, url);
+        result = -1;
+        goto cleanup;
+    }
+
+    char *html_tracked_dir = html_config_dir;
+    strcat(html_tracked_dir, "/tracked/");
+    strcat(html_tracked_dir, name);
+
+    if(file_exists(html_tracked_dir) == 0) {
+        fprintf(stderr, "You are already tracking %s\n", url);
+        result = -1;
+        goto cleanup;
+    }
+
+    result = create_directory_recursive(html_tracked_dir);
+    if(result != 0) {
+        fprintf(stderr, "Failed to create %s, error: %s\n", html_tracked_dir, strerror(result));
+        goto cleanup;
+    }
+
+    /*
+        Create an ".in_progress" file to prevent periodic sync from reading rss data
+        before we have finished adding all the data.
+    */
+    char in_progress_filepath[PATH_MAX];
+    strcpy(in_progress_filepath, html_tracked_dir);
+    strcat(in_progress_filepath, "/.in_progress");
+    result = create_lock_file(in_progress_filepath);
+    if(result != 0) {
+        fprintf(stderr, "Failed to create %s/.in_progress\n", html_tracked_dir);
+        goto cleanup;
+    }
+
+    result = file_overwrite_in_dir(html_tracked_dir, "link", url, strlen(url));
+    if(result != 0) {
+        fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir);
+        remove(html_tracked_dir);
+        goto cleanup;
+    }
+
+    char *plugin_name = basename(domain_plugin_path);
+    result = file_overwrite_in_dir(html_tracked_dir, "plugin", plugin_name, strlen(plugin_name));
+    if(result != 0) {
+        fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir);
+        remove(html_tracked_dir);
+        goto cleanup;
+    }
+
+    char updated[32];
+    sprintf(updated, "%ld", time(NULL));
+    result = file_overwrite_in_dir(html_tracked_dir, "updated", updated, strlen(updated));
+    if(result != 0) {
+        fprintf(stderr, "Failed to create %s/updated\n", html_tracked_dir);
+        remove(html_tracked_dir);
+        goto cleanup;
+    }
+
+    result = write_plugin_json_to_file(html_tracked_dir, "data", url, updated, start_after, plugin_list_userdata.start_after_url, plugin_name);
+    if(result != 0) {
+        fprintf(stderr, "Failed to create %s/data\n", html_tracked_dir);
+        remove(html_tracked_dir);
+        goto cleanup;
+    }
+
+    remove(in_progress_filepath);
+
+    cleanup:
+    free(json_root);
+    return result;
+}