From 221522cf995cbcd39c956f66423a26bbccae8f72 Mon Sep 17 00:00:00 2001 From: dec05eba Date: Fri, 16 Oct 2020 04:49:14 +0200 Subject: Matrix: stream download to rapidjson parser --- src/DownloadUtils.cpp | 45 ++++++++++++++++ src/Program.c | 136 +++++++++++++++++++------------------------------ src/plugins/Matrix.cpp | 16 ++---- 3 files changed, 101 insertions(+), 96 deletions(-) (limited to 'src') diff --git a/src/DownloadUtils.cpp b/src/DownloadUtils.cpp index c44bed5..4f8b5b9 100644 --- a/src/DownloadUtils.cpp +++ b/src/DownloadUtils.cpp @@ -3,6 +3,8 @@ #include "../include/Storage.hpp" #include "../include/base64_url.hpp" #include +#include +#include static const bool debug_download = false; @@ -77,4 +79,47 @@ namespace QuickMedia { return download_result; } } + + // TODO: Add timeout + DownloadResult download_to_json(const std::string &url, rapidjson::Document &result, const std::vector &additional_args, bool use_tor, bool use_browser_useragent, bool fail_on_error) { + sf::Clock timer; + std::vector args; + if(use_tor) + args.push_back("torsocks"); + args.insert(args.end(), { "curl", "-H", "Accept-Language: en-US,en;q=0.5", "-H", "Connection: keep-alive", "--compressed", "-s", "-L" }); + if(fail_on_error) + args.push_back("-f"); + for(const CommandArg &arg : additional_args) { + args.push_back(arg.option.c_str()); + args.push_back(arg.value.c_str()); + } + if(use_browser_useragent) { + args.push_back("-H"); + args.push_back(useragent_str); + } + args.push_back("--"); + args.push_back(url.c_str()); + args.push_back(nullptr); + if(debug_download) { + for(const char *arg : args) { + if(arg) + fprintf(stderr, "'%s' ", arg); + } + fprintf(stderr, "\n"); + } + ReadProgram read_program; + if(exec_program_pipe(args.data(), &read_program) != 0) + return DownloadResult::NET_ERR; + + FILE *file = fdopen(read_program.read_fd, "rb"); + char read_buffer[8192]; + rapidjson::FileReadStream is(file, read_buffer, sizeof(read_buffer)); + rapidjson::ParseResult parse_result = result.ParseStream(is); + + wait_program(read_program.pid); + fclose(file); + fprintf(stderr, "Download duration for %s: %d ms\n", url.c_str(), timer.getElapsedTime().asMilliseconds()); + + return parse_result.IsError() ? DownloadResult::ERR : DownloadResult::OK; + } } \ No newline at end of file diff --git a/src/Program.c b/src/Program.c index 2307798..fc80e5e 100644 --- a/src/Program.c +++ b/src/Program.c @@ -11,7 +11,7 @@ #define READ_END 0 #define WRITE_END 1 -int exec_program(const char **args, ProgramOutputCallback output_callback, void *userdata) { +int exec_program_pipe(const char **args, ReadProgram *read_program) { /* 1 arguments */ if(args[0] == NULL) return -1; @@ -49,58 +49,68 @@ int exec_program(const char **args, ProgramOutputCallback output_callback, void _exit(127); } else { /* parent */ close(fd[WRITE_END]); + read_program->pid = pid; + read_program->read_fd = fd[READ_END]; + return 0; + } +} - int result = 0; - int status; - - char buffer[4097]; +int exec_program(const char **args, ProgramOutputCallback output_callback, void *userdata) { + ReadProgram read_program; + int res = exec_program_pipe(args, &read_program); + if(res != 0) + return res; - for(;;) { - ssize_t bytes_read = read(fd[READ_END], buffer, sizeof(buffer) - 1); - if(bytes_read == 0) { - break; - } else if(bytes_read == -1) { - int err = errno; - fprintf(stderr, "Failed to read from pipe to program %s, error: %s\n", args[0], strerror(err)); - result = -err; - goto cleanup; - } + int result = 0; + int status; - buffer[bytes_read] = '\0'; - if(output_callback && output_callback(buffer, bytes_read, userdata) != 0) - break; - } + char buffer[4097]; - if(waitpid(pid, &status, 0) == -1) { - perror("waitpid failed"); - result = -5; + for(;;) { + ssize_t bytes_read = read(read_program.read_fd, buffer, sizeof(buffer) - 1); + if(bytes_read == 0) { + break; + } else if(bytes_read == -1) { + int err = errno; + fprintf(stderr, "Failed to read from pipe to program %s, error: %s\n", args[0], strerror(err)); + result = -err; goto cleanup; } - if(!WIFEXITED(status)) { - result = -4; - goto cleanup; - } + buffer[bytes_read] = '\0'; + if(output_callback && output_callback(buffer, bytes_read, userdata) != 0) + break; + } - int exit_status = WEXITSTATUS(status); - if(exit_status != 0) { - fprintf(stderr, "Failed to execute program ("); - const char **arg = args; - while(*arg) { - if(arg != args) - fputc(' ', stderr); - fprintf(stderr, "'%s'", *arg); - ++arg; - } - fprintf(stderr, "), exit status %d\n", exit_status); - result = -exit_status; - goto cleanup; - } + if(waitpid(read_program.pid, &status, 0) == -1) { + perror("waitpid failed"); + result = -5; + goto cleanup; + } - cleanup: - close(fd[READ_END]); - return result; + if(!WIFEXITED(status)) { + result = -4; + goto cleanup; + } + + int exit_status = WEXITSTATUS(status); + if(exit_status != 0) { + fprintf(stderr, "Failed to execute program ("); + const char **arg = args; + while(*arg) { + if(arg != args) + fputc(' ', stderr); + fprintf(stderr, "'%s'", *arg); + ++arg; + } + fprintf(stderr, "), exit status %d\n", exit_status); + result = -exit_status; + goto cleanup; } + + cleanup: + close(read_program.read_fd); + return result; } int wait_program(pid_t process_id) { @@ -186,43 +196,3 @@ int exec_program_async(const char **args, pid_t *result_process_id) { } return 0; } - -#if 0 -int program_pipe_write(ProgramPipe *self, const char *data, size_t size) { - ssize_t bytes_written = write(self->write_fd, data, size); - if(bytes_written == -1) { - int err = errno; - perror("Failed to write to pipe to program"); - return -err; - } - return 0; -} - -int program_pipe_read(ProgramPipe *self, ProgramOutputCallback output_callback, void *userdata) { - char buffer[2048]; - - for(;;) { - ssize_t bytes_read = read(self->read_fd, buffer, sizeof(buffer) - 1); - if(bytes_read == 0) { - break; - } else if(bytes_read == -1) { - int err = errno; - perror("Failed to read from pipe to program"); - return -err; - } - - buffer[bytes_read] = '\0'; - if(output_callback && output_callback(buffer, bytes_read, userdata) != 0) - break; - } - - return 0; -} - -void program_pipe_close(ProgramPipe *self) { - close(self->read_fd); - close(self->write_fd); - self->read_fd = -1; - self->write_fd = -1; -} -#endif diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index 3637d41..a0d8568 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -1730,22 +1730,12 @@ namespace QuickMedia { } DownloadResult Matrix::download_json(rapidjson::Document &result, const std::string &url, std::vector additional_args, bool use_browser_useragent, std::string *err_msg) const { - std::string server_response; - if(download_to_string(url, server_response, std::move(additional_args), use_tor, use_browser_useragent, err_msg == nullptr) != DownloadResult::OK) { + if(download_to_json(url, result, std::move(additional_args), use_tor, use_browser_useragent, err_msg == nullptr) != DownloadResult::OK) { + // Cant get error since we parse directory to json. TODO: Make this work somehow? if(err_msg) - *err_msg = server_response; + *err_msg = "Failed to download/parse json"; return DownloadResult::NET_ERR; } - - rapidjson::ParseResult parse_result = result.Parse(server_response.c_str(), server_response.size()); - if(parse_result.IsError()) { - std::string error_code_str = std::to_string(parse_result.Code()); - fprintf(stderr, "download_json error: %s\n", error_code_str.c_str()); - if(err_msg) - *err_msg = "Json parse error: " + std::move(error_code_str); - return DownloadResult::ERR; - } - return DownloadResult::OK; } } \ No newline at end of file -- cgit v1.2.3