path: root/src/html.c

#include "html.h"
#include "fileutils.h"
#include "program.h"
#include "buffer.h"
#include "stringutils.h"
#include "rss_html_common.h"
#include "main.h"
#include "json.h"
#include <limits.h>
#include <string.h>
#include <stdio.h>
#include <libgen.h>
#include <signal.h>
#include <time.h>
#include <dirent.h>
#include <unistd.h>
#include <assert.h>

static int str_starts_with(const char *str, int len, const char *substr, int substr_len) {
    return len >= substr_len && memcmp(str, substr, substr_len) == 0;
}

static int url_extract_domain(const char *url, char *domain, int domain_len) {
    int url_len = strlen(url);
    if(str_starts_with(url, url_len, "http://", 7)) {
        url += 7;
        url_len -= 7;
    } else if(str_starts_with(url, url_len, "https://", 8)) {
        url += 8;
        url_len -= 8;
    }

    if(str_starts_with(url, url_len, "www.", 4)) {
        url += 4;
        url_len -= 4;
    }

    char *end = strchr(url, '.');
    if(end) {
        int len = end - url;
        if(len >= domain_len)
            return -1;
        memcpy(domain, url, len);
        domain[len] = '\0';
    } else {
        if(url_len >= domain_len)
            return -1;
        memcpy(domain, url, url_len);
        domain[url_len] = '\0';
    }
    return 0;
}

/*
    The plugin should print the names and urls of each item (chapter for manga) and the output list should stop when an
    item matches any item in the input. The output should be sorted from newest to oldest.

    The input should be in this format:
      [
        {
          "title": "Example name",
          "url": "https://example.com"
        },
        {
          "title": "Another item",
          "url": "https://another.url.com"
        }
      ]

    And the output should be in this format:
      [
        {
          "name": "Example name",
          "url": "https://example.com"
        },
        {
          "name": "Another item",
          "url": "https://another.url.com"
        }
      ]

    TODO: Rename input "title" to "url", to make input and output match (easier to test with).
*/
typedef int (*PluginListCallback)(const char *name, const char *url, void *userdata);
static struct json_value_s* plugin_list(char *plugin_filepath, const char *url, struct json_array_s *downloaded_items, PluginListCallback callback, void *userdata) {
    int result;
    Buffer buffer;
    buffer_init(&buffer);

    const char *args[] = { plugin_filepath, "list", url, NULL };
    int process_id = -1;
    int stdin_file = -1;
    int stdout_file = -1;
    result = program_exec_async(args, &process_id, &stdin_file, &stdout_file);
    if(result != 0) {
        fprintf(stderr, "Failed to launch plugin list for plugin %s\n", basename(plugin_filepath));
        goto err_cleanup;
    }

    if(downloaded_items) {
        struct json_value_s downloaded_items_value;
        downloaded_items_value.payload = downloaded_items;
        downloaded_items_value.type = json_type_array;
        size_t json_output_len = 0;
        void *json_body_str = json_write_minified(&downloaded_items_value, &json_output_len);
        if(!json_body_str) {
            fprintf(stderr, "Failed to convert downloaded items to json\n");
            if(process_id != -1)
                kill(process_id, SIGKILL);
            close(stdin_file);
            close(stdout_file);
            goto err_cleanup;
        }

        /* This is a bug in the json library */
        json_output_len = strlen(json_body_str);
        if(write(stdin_file, json_body_str, json_output_len) != (ssize_t)json_output_len) {
            fprintf(stderr, "Failed to write all bytes to plugin list\n");
            if(process_id != -1)
                kill(process_id, SIGKILL);
            close(stdin_file);
            close(stdout_file);
            goto err_cleanup;
        }
        free(json_body_str);
    }

    result = program_wait_until_exit(process_id, stdin_file, stdout_file, program_buffer_write_callback, &buffer);
    if(result != 0) {
        fprintf(stderr, "Failed to launch plugin list for plugin %s\n", basename(plugin_filepath));
        goto err_cleanup;
    }

    struct json_value_s *json_root = json_parse(buffer.data, buffer.size);
    if(!json_root) {
        fprintf(stderr, "Failed to load plugin %s list output as json\n", basename(plugin_filepath));
        goto err_cleanup;
    }
    buffer_deinit(&buffer);

    struct json_array_s *json_root_array = json_value_as_array(json_root);
    if(!json_root_array) {
        fprintf(stderr, "Failed to load plugin %s list output as json\n", basename(plugin_filepath));
        free(json_root);
        goto err_cleanup;
    }

    struct json_array_element_s *json_array_element = json_root_array->start;
    for(; json_array_element; json_array_element = json_array_element->next) {
        struct json_object_s *array_element_value = json_value_as_object(json_array_element->value);
        if(!array_element_value)
            continue;
        
        struct json_value_s *name_json = json_object_get_field_by_name(array_element_value, "name");
        struct json_value_s *url_json = json_object_get_field_by_name(array_element_value, "url");
        if(!name_json || !url_json)
            continue;

        struct json_string_s *name_json_str = json_value_as_string(name_json);
        struct json_string_s *url_json_str = json_value_as_string(url_json);
        if(!name_json_str || !url_json_str)
            continue;
        
        /* Cast from const to mutable variable because I can */
        char *name = (char*)name_json_str->string;
        string_replace(name, '/', '_');
        name = strip(name);
        if(callback(name, url_json_str->string, userdata) != 0)
            break;
    }

    return json_root;

    err_cleanup:
    buffer_deinit(&buffer);
    return NULL;
}

typedef struct {
    const char *start_after;
    int found_start_after;
    const char *start_after_url;
} PluginListUserdata;

static int plugin_list_callback(const char *name, const char *url, void *userdata) {
    PluginListUserdata *plugin_list_userdata = userdata;
    if(plugin_list_userdata->start_after && strcmp(plugin_list_userdata->start_after, name) == 0) {
        plugin_list_userdata->found_start_after = 1;
        plugin_list_userdata->start_after_url = url;
        return 1;
    }
    return 0;
}

/* Store result in @plugin_filepath */
static int get_plugin_filepath(const char *program_dir, const char *plugin_name, char *plugin_filepath) {
    const char *path_components[] = { program_dir, "plugins" };
    path_join(plugin_filepath, path_components, 2);
    if(file_exists(plugin_filepath) != 0) {
        strcpy(plugin_filepath, "/usr/share/automedia/plugins");
        if(file_exists(plugin_filepath) != 0) {
            fprintf(stderr, "Failed to find plugins directory\n");
            return -1;
        }
    }

    strcat(plugin_filepath, "/");
    strcat(plugin_filepath, plugin_name);
    if(file_exists(plugin_filepath) != 0) {
        strcat(plugin_filepath, ".py");
        if(file_exists(plugin_filepath) != 0) {
            fprintf(stderr, "Plugin doesn't exist: %s\n", plugin_name);
            return -1;
        }
    }

    return 0;
}

int add_html(const char *name, const char *url, char *html_config_dir, char *program_dir, const char *start_after) {
    int result = 0;

    if(!name || name[0] == '\0') {
        fprintf(stderr, "Name not provided or empty\n");
        return -1;
    }

    char domain[2086];
    if(url_extract_domain(url, domain, sizeof(domain)) != 0) {
        fprintf(stderr, "Url %s is too long\n", url);
        return -1;
    }

    if(domain[0] == '\0') {
        fprintf(stderr, "Invalid url: %s\n", url);
        return -1;
    }

    char domain_plugin_path[PATH_MAX];
    if(get_plugin_filepath(program_dir, domain, domain_plugin_path) != 0)
        return -1;

    PluginListUserdata plugin_list_userdata;
    plugin_list_userdata.start_after = start_after;
    plugin_list_userdata.found_start_after = 0;
    plugin_list_userdata.start_after_url = NULL;

    struct json_value_s *json_root = plugin_list(domain_plugin_path, url, NULL, plugin_list_callback, &plugin_list_userdata);
    if(!json_root)
        return -1;

    if(start_after && !plugin_list_userdata.found_start_after) {
        fprintf(stderr, "Failed to find %s in html %s\n", start_after, url);
        result = -1;
        goto cleanup;
    }

    char *html_tracked_dir = html_config_dir;
    strcat(html_tracked_dir, "/tracked/");
    strcat(html_tracked_dir, name);

    if(file_exists(html_tracked_dir) == 0) {
        fprintf(stderr, "You are already tracking %s\n", url);
        result = -1;
        goto cleanup;
    }

    result = create_directory_recursive(html_tracked_dir);
    if(result != 0) {
        fprintf(stderr, "Failed to create %s, error: %s\n", html_tracked_dir, strerror(result));
        goto cleanup;
    }

    /*
        Create an ".in_progress" file to prevent periodic sync from reading rss data
        before we have finished adding all the data.
    */
    char in_progress_filepath[PATH_MAX];
    strcpy(in_progress_filepath, html_tracked_dir);
    strcat(in_progress_filepath, "/.in_progress");
    result = create_lock_file(in_progress_filepath);
    if(result != 0) {
        fprintf(stderr, "Failed to create %s/.in_progress\n", html_tracked_dir);
        goto cleanup;
    }

    result = file_overwrite_in_dir(html_tracked_dir, "link", url, strlen(url));
    if(result != 0) {
        fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir);
        remove(html_tracked_dir);
        goto cleanup;
    }

    char *plugin_name = basename(domain_plugin_path);
    result = file_overwrite_in_dir(html_tracked_dir, "plugin", plugin_name, strlen(plugin_name));
    if(result != 0) {
        fprintf(stderr, "Failed to create %s/link\n", html_tracked_dir);
        remove(html_tracked_dir);
        goto cleanup;
    }

    char updated[32];
    assert(sizeof(time_t) == 4);
    sprintf(updated, "%ld", time(NULL));
    result = file_overwrite_in_dir(html_tracked_dir, "updated", updated, strlen(updated));
    if(result != 0) {
        fprintf(stderr, "Failed to create %s/updated\n", html_tracked_dir);
        remove(html_tracked_dir);
        goto cleanup;
    }

    result = write_plugin_json_to_file(html_tracked_dir, "data", url, updated, start_after, plugin_list_userdata.start_after_url, plugin_name);
    if(result != 0) {
        fprintf(stderr, "Failed to create %s/data\n", html_tracked_dir);
        remove(html_tracked_dir);
        goto cleanup;
    }

    remove(in_progress_filepath);

    cleanup:
    free(json_root);
    return result;
}

static int plugin_list_sync_callback(const char *name, const char *url, void *userdata) {
    Buffer *download_items_buffer = userdata;
    DownloadItemsData download_items_data;
    download_items_data.title = name;
    download_items_data.link = url;
    buffer_append(download_items_buffer, &download_items_data, sizeof(download_items_data));
    return 0;
}

static int download_html_items_in_reverse(const char *plugin_filepath, Buffer *download_items_buffer, TrackedHtml *tracked_html, char *html_tracked_dir, const char *download_dir) {
    int result = 0;
    DownloadItemsData *added_download_items[MAX_UPDATE_ITEMS];

    char item_dir[PATH_MAX];
    const char *path_components[] = { download_dir, tracked_html->title };
    int item_dir_len = path_join(item_dir, path_components, 2);

    Buffer json_element_buffer;
    buffer_init(&json_element_buffer);

    DownloadItemsData *download_items_it = buffer_end(download_items_buffer);
    DownloadItemsData *download_items_end = buffer_begin(download_items_buffer);
    download_items_it--;
    download_items_end--;
    int download_item_index = 0;
    for(; download_items_it != download_items_end && download_item_index < MAX_UPDATE_ITEMS && is_program_running(); --download_items_it) {
        char notify_msg[PATH_MAX];
        const char *path_components[] = { tracked_html->title, download_items_it->title };
        path_join(notify_msg, path_components, 2);

        item_dir[item_dir_len] = '/';
        strcpy(item_dir + item_dir_len + 1, download_items_it->title);
        fprintf(stderr, "Starting download of html item: %s (title: %s)\n", download_items_it->link, notify_msg);
        if(create_directory_recursive(item_dir) != 0) {
            fprintf(stderr, "Failed to create directory for html item: %s\n", download_items_it->title);
            result = -1;
            break;
        }

        /* TODO: Make asynchronous */
        const char *args[] = { plugin_filepath, "download", download_items_it->link, NULL };
        result = program_exec(args, NULL, NULL);
        if(result != 0)
            fprintf(stderr, "Failed while downloading html, url: %s\n", download_items_it->link);
        
        const char *notify_args[] = { "notify-send", "-u", result == 0 ? "normal" : "critical", "--", result == 0 ? "Download finished" : "Download failed", notify_msg, NULL };
        program_exec(notify_args, NULL, NULL);

        if(result != 0)
            break;

        added_download_items[download_item_index] = download_items_it;
        ++download_item_index;
    }

    TrackedItem tracked_item;
    tracked_item.title = tracked_html->title;
    tracked_item.link = tracked_html->link;
    tracked_item.json_data = tracked_html->json_data;
    result = tracked_item_update_latest(&tracked_item, html_tracked_dir, added_download_items, NULL, download_item_index);

    buffer_deinit(&json_element_buffer);
    return result;
}

/* TODO: Make asynchronous. Right now this will only complete when the whole chapter download completes */
int sync_html(TrackedHtml *tracked_html, char *program_dir, const char *download_dir, char *html_config_dir) {
    /* TODO: This can be cached */
    int html_config_dir_len = strlen(html_config_dir);

    fprintf(stderr, "Syncing %s\n", tracked_html->title);

    char plugin_filepath[PATH_MAX];
    /* This will check with ${tracked_html->plugin}.py as well, but that is fine */
    if(get_plugin_filepath(program_dir, tracked_html->plugin, plugin_filepath) != 0)
        return -1;

    struct json_value_s *downloaded_items = json_object_get_field_by_name(tracked_html->json_data, "downloaded");
    struct json_array_s *downloaded_items_array = NULL;
    if(downloaded_items) {
        downloaded_items_array = json_value_as_array(downloaded_items);
        if(!downloaded_items_array) {
            fprintf(stderr, "Corrupt json for html item: %s\n", tracked_html->title);
            return -1;
        }
    }

    Buffer download_items_buffer;
    buffer_init(&download_items_buffer);

    int result = 0;

    struct json_value_s *json_root = plugin_list(plugin_filepath, tracked_html->link, downloaded_items_array, plugin_list_sync_callback, &download_items_buffer);
    if(!json_root) {
        result = -1;
        goto cleanup;
    }

    char *html_tracked_dir = html_config_dir;
    strcat(html_tracked_dir, "/tracked/");

    result = download_html_items_in_reverse(plugin_filepath, &download_items_buffer, tracked_html, html_tracked_dir, download_dir);
    if(result != 0) {
        fprintf(stderr, "Failed while download html item for url: %s\n", tracked_html->link);
        goto cleanup;
    }

    char updated[32];
    sprintf(updated, "%ld", time(NULL));
    strcat(html_tracked_dir, tracked_html->title);
    result = file_overwrite_in_dir(html_tracked_dir, "synced", updated, strlen(updated));
    if(result != 0) {
        fprintf(stderr, "Failed to update %s/synced\n", html_tracked_dir);
        goto cleanup;
    }

    cleanup:
    free(json_root);
    buffer_deinit(&download_items_buffer);
    html_config_dir[html_config_dir_len] = '\0';
    return result;
}