Matrix: stream download to rapidjson parser

1 files changed, 45 insertions, 0 deletions

diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp
index c44bed5..4f8b5b9 100644
--- a/src/DownloadUtils.cpp
+++ b/src/DownloadUtils.cpp
@@ -3,6 +3,8 @@
 #include "../include/Storage.hpp"
 #include "../include/base64_url.hpp"
 #include <SFML/System/Clock.hpp>
+#include <rapidjson/filereadstream.h>
+#include <unistd.h>
 
 static const bool debug_download = false;
 
@@ -77,4 +79,47 @@ namespace QuickMedia {
             return download_result;
         }
     }
+
+    // TODO: Add timeout
+    DownloadResult download_to_json(const std::string &url, rapidjson::Document &result, const std::vector<CommandArg> &additional_args, bool use_tor, bool use_browser_useragent, bool fail_on_error) {
+        sf::Clock timer;
+        std::vector<const char*> args;
+        if(use_tor)
+            args.push_back("torsocks");
+        args.insert(args.end(), { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", "-s", "-L" });
+        if(fail_on_error)
+            args.push_back("-f");
+        for(const CommandArg &arg : additional_args) {
+            args.push_back(arg.option.c_str());
+            args.push_back(arg.value.c_str());
+        }
+        if(use_browser_useragent) {
+            args.push_back("-H");
+            args.push_back(useragent_str);
+        }
+        args.push_back("--");
+        args.push_back(url.c_str());
+        args.push_back(nullptr);
+        if(debug_download) {
+            for(const char *arg : args) {
+                if(arg)
+                    fprintf(stderr, "'%s' ", arg);
+            }
+            fprintf(stderr, "\n");
+        }
+        ReadProgram read_program;
+        if(exec_program_pipe(args.data(), &read_program) != 0)
+            return DownloadResult::NET_ERR;
+        
+        FILE *file = fdopen(read_program.read_fd, "rb");
+        char read_buffer[8192];
+        rapidjson::FileReadStream is(file, read_buffer, sizeof(read_buffer));
+        rapidjson::ParseResult parse_result = result.ParseStream(is);
+
+        wait_program(read_program.pid);
+        fclose(file);
+        fprintf(stderr, "Download duration for %s: %d ms\n", url.c_str(), timer.getElapsedTime().asMilliseconds());
+        
+        return parse_result.IsError() ? DownloadResult::ERR : DownloadResult::OK;
+    }
 }
\ No newline at end of file