Matrix: stream download to rapidjson parser

3 files changed, 101 insertions, 96 deletions

diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp
index c44bed5..4f8b5b9 100644
--- a/src/DownloadUtils.cpp
+++ b/src/DownloadUtils.cpp
@@ -3,6 +3,8 @@
 #include "../include/Storage.hpp"
 #include "../include/base64_url.hpp"
 #include <SFML/System/Clock.hpp>
+#include <rapidjson/filereadstream.h>
+#include <unistd.h>
 
 static const bool debug_download = false;
 
@@ -77,4 +79,47 @@ namespace QuickMedia {
             return download_result;
         }
     }
+
+    // TODO: Add timeout
+    DownloadResult download_to_json(const std::string &url, rapidjson::Document &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent, bool fail_on_error) {
+        sf::Clock timer;
+        std::vector<const char*> args;
+        if(use_tor)
+            args.push_back("torsocks");
+        args.insert(args.end(), { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", "-s", "-L" });
+        if(fail_on_error)
+            args.push_back("-f");
+        for(const CommandArg &arg : additional_args) {
+            args.push_back(arg.option.c_str());
+            args.push_back(arg.value.c_str());
+        }
+        if(use_browser_useragent) {
+            args.push_back("-H");
+            args.push_back(useragent_str);
+        }
+        args.push_back("--");
+        args.push_back(url.c_str());
+        args.push_back(nullptr);
+        if(debug_download) {
+            for(const char *arg : args) {
+                if(arg)
+                    fprintf(stderr, "'%s' ", arg);
+            }
+            fprintf(stderr, "\n");
+        }
+        ReadProgram read_program;
+        if(exec_program_pipe(args.data(), &read_program) != 0)
+            return DownloadResult::NET_ERR;
+        
+        FILE *file = fdopen(read_program.read_fd, "rb");
+        char read_buffer[8192];
+        rapidjson::FileReadStream is(file, read_buffer, sizeof(read_buffer));
+        rapidjson::ParseResult parse_result = result.ParseStream(is);
+
+        wait_program(read_program.pid);
+        fclose(file);
+        fprintf(stderr, "Download duration for %s: %d ms\n", url.c_str(), timer.getElapsedTime().asMilliseconds());
+        
+        return parse_result.IsError() ? DownloadResult::ERR : DownloadResult::OK;
+    }
 }
\ No newline at end of file
diff --git a/src/Program.c b/src/Program.c
index 2307798..fc80e5e 100644
--- a/src/Program.c
+++ b/src/Program.c
@@ -11,7 +11,7 @@
 #define READ_END 0
 #define WRITE_END 1
 
-int exec_program(const char **args, ProgramOutputCallback output_callback, void *userdata) {
+int exec_program_pipe(const char **args, ReadProgram *read_program) {
     /* 1 arguments */
     if(args[0] == NULL)
         return -1;
@@ -49,58 +49,68 @@ int exec_program(const char **args, ProgramOutputCallback output_callback, void
         _exit(127);
     } else { /* parent */
         close(fd[WRITE_END]);
+        read_program->pid = pid;
+        read_program->read_fd = fd[READ_END];
+        return 0;
+    }
+}
 
-        int result = 0;
-        int status;
-
-        char buffer[4097];
+int exec_program(const char **args, ProgramOutputCallback output_callback, void *userdata) {
+    ReadProgram read_program;
+    int res = exec_program_pipe(args, &read_program);
+    if(res != 0)
+        return res;
 
-        for(;;) {
-            ssize_t bytes_read = read(fd[READ_END], buffer, sizeof(buffer) - 1);
-            if(bytes_read == 0) {
-                break;
-            } else if(bytes_read == -1) {
-                int err = errno;
-                fprintf(stderr, "Failed to read from pipe to program %s, error: %s\n", args[0], strerror(err));
-                result = -err;
-                goto cleanup;
-            }
+    int result = 0;
+    int status;
 
-            buffer[bytes_read] = '\0';
-            if(output_callback && output_callback(buffer, bytes_read, userdata) != 0)
-                break;
-        }
+    char buffer[4097];
 
-        if(waitpid(pid, &status, 0) == -1) {
-            perror("waitpid failed");
-            result = -5;
+    for(;;) {
+        ssize_t bytes_read = read(read_program.read_fd, buffer, sizeof(buffer) - 1);
+        if(bytes_read == 0) {
+            break;
+        } else if(bytes_read == -1) {
+            int err = errno;
+            fprintf(stderr, "Failed to read from pipe to program %s, error: %s\n", args[0], strerror(err));
+            result = -err;
             goto cleanup;
         }
 
-        if(!WIFEXITED(status)) {
-            result = -4;
-            goto cleanup;
-        }
+        buffer[bytes_read] = '\0';
+        if(output_callback && output_callback(buffer, bytes_read, userdata) != 0)
+            break;
+    }
 
-        int exit_status = WEXITSTATUS(status);
-        if(exit_status != 0) {
-            fprintf(stderr, "Failed to execute program (");
-            const char **arg = args;
-            while(*arg) {
-                if(arg != args)
-                    fputc(' ', stderr);
-                fprintf(stderr, "'%s'", *arg);
-                ++arg;
-            }
-            fprintf(stderr, "), exit status %d\n", exit_status);
-            result = -exit_status;
-            goto cleanup;
-        }
+    if(waitpid(read_program.pid, &status, 0) == -1) {
+        perror("waitpid failed");
+        result = -5;
+        goto cleanup;
+    }
 
-        cleanup:
-        close(fd[READ_END]);
-        return result;
+    if(!WIFEXITED(status)) {
+        result = -4;
+        goto cleanup;
+    }
+
+    int exit_status = WEXITSTATUS(status);
+    if(exit_status != 0) {
+        fprintf(stderr, "Failed to execute program (");
+        const char **arg = args;
+        while(*arg) {
+            if(arg != args)
+                fputc(' ', stderr);
+            fprintf(stderr, "'%s'", *arg);
+            ++arg;
+        }
+        fprintf(stderr, "), exit status %d\n", exit_status);
+        result = -exit_status;
+        goto cleanup;
     }
+
+    cleanup:
+    close(read_program.read_fd);
+    return result;
 }
 
 int wait_program(pid_t process_id) {
@@ -186,43 +196,3 @@ int exec_program_async(const char **args, pid_t *result_process_id) {
     }
     return 0;
 }
-
-#if 0
-int program_pipe_write(ProgramPipe *self, const char *data, size_t size) {
-    ssize_t bytes_written = write(self->write_fd, data, size);
-    if(bytes_written == -1) {
-        int err = errno;
-        perror("Failed to write to pipe to program");
-        return -err;
-    }
-    return 0;
-}
-
-int program_pipe_read(ProgramPipe *self, ProgramOutputCallback output_callback, void *userdata) {
-    char buffer[2048];
-
-    for(;;) {
-        ssize_t bytes_read = read(self->read_fd, buffer, sizeof(buffer) - 1);
-        if(bytes_read == 0) {
-            break;
-        } else if(bytes_read == -1) {
-            int err = errno;
-            perror("Failed to read from pipe to program");
-            return -err;
-        }
-
-        buffer[bytes_read] = '\0';
-        if(output_callback && output_callback(buffer, bytes_read, userdata) != 0)
-            break;
-    }
-
-    return 0;
-}
-
-void program_pipe_close(ProgramPipe *self) {
-    close(self->read_fd);
-    close(self->write_fd);
-    self->read_fd = -1;
-    self->write_fd = -1;
-}
-#endif
diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp
index 3637d41..a0d8568 100644
--- a/src/plugins/Matrix.cpp
+++ b/src/plugins/Matrix.cpp
@@ -1730,22 +1730,12 @@ namespace QuickMedia {
     }
 
     DownloadResult Matrix::download_json(rapidjson::Document &result, const std::string &url, std::vector<CommandArg> additional_args, bool use_browser_useragent, std::string *err_msg) const {
-        std::string server_response;
-        if(download_to_string(url, server_response, std::move(additional_args), use_tor, use_browser_useragent, err_msg == nullptr) != DownloadResult::OK) {
+        if(download_to_json(url, result, std::move(additional_args), use_tor, use_browser_useragent, err_msg == nullptr) != DownloadResult::OK) {
+            // Cant get error since we parse directory to json. TODO: Make this work somehow?
             if(err_msg)
-                *err_msg = server_response;
+                *err_msg = "Failed to download/parse json";
             return DownloadResult::NET_ERR;
         }
-
-        rapidjson::ParseResult parse_result = result.Parse(server_response.c_str(), server_response.size());
-        if(parse_result.IsError()) {
-            std::string error_code_str = std::to_string(parse_result.Code());
-            fprintf(stderr, "download_json error: %s\n", error_code_str.c_str());
-            if(err_msg)
-                *err_msg = "Json parse error: " + std::move(error_code_str);
-            return DownloadResult::ERR;
-        }
-
         return DownloadResult::OK;
     }
 }
\ No newline at end of file