diff options
| -rw-r--r-- | TODO | 1 | ||||
| -rw-r--r-- | include/AsyncImageLoader.hpp | 1 | ||||
| -rw-r--r-- | include/FileAnalyzer.hpp | 9 | ||||
| -rw-r--r-- | src/AsyncImageLoader.cpp | 45 | ||||
| -rw-r--r-- | src/FileAnalyzer.cpp | 60 | ||||
| -rw-r--r-- | src/ImageUtils.cpp | 4 | ||||
| -rw-r--r-- | src/QuickMedia.cpp | 2 | ||||
| -rw-r--r-- | src/plugins/FileManager.cpp | 5 | ||||
| -rw-r--r-- | src/plugins/Matrix.cpp | 29 |
9 files changed, 105 insertions, 51 deletions
@@ -146,7 +146,6 @@ Replace sfml font glyph loading completely with FreeType. There is a very old bu Add arguments to pipe plugin to pass input and output fifo for sending commands to QuickMedia and receiving events. Update thumbnails in file-manager if an image is replaced, by including the modify date of the image in the thumbnail cache as well. Create a workaround for dwm terminal swallow patch stealing mpv when moving QuickMedia to another monitor sometimes. Maybe check for structure notify events on mpv and reparent and select input on the mpv window again? -Resize video thumbnail when extracted with ffmpeg to 480x360, clamped from its original size. Add option to decline and mute user in invites. This is to combat invite spam, where muted users cant invite you. Allow hiding videos so they dont show up in recommendations and related videos. Add an option to select video resolution, if we want to use less power and less bandwidth for example. diff --git a/include/AsyncImageLoader.hpp b/include/AsyncImageLoader.hpp index 0366415..69746d2 100644 --- a/include/AsyncImageLoader.hpp +++ b/include/AsyncImageLoader.hpp @@ -25,6 +25,7 @@ namespace QuickMedia { sf::Clock texture_applied_time; }; + // This function is async bool create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, sf::Vector2i resize_target_size); constexpr int NUM_IMAGE_LOAD_THREADS = 4; diff --git a/include/FileAnalyzer.hpp b/include/FileAnalyzer.hpp index be0cc25..51a7aa8 100644 --- a/include/FileAnalyzer.hpp +++ b/include/FileAnalyzer.hpp @@ -15,6 +15,7 @@ namespace QuickMedia { VIDEO_AVI, VIDEO_MP4, VIDEO_WEBM, + VIDEO_FLV, AUDIO_BASIC, AUDIO_AIFF, AUDIO_MPEG, @@ -33,13 +34,17 @@ namespace QuickMedia { bool is_content_type_audio(ContentType content_type); bool is_content_type_image(ContentType content_type); const char* content_type_to_string(ContentType content_type); + bool is_image_ext(const char *ext); + bool is_video_ext(const char *ext); - bool video_get_first_frame(const char *filepath, const char *destination_path); + // Set |width| or |height| to 0 to disable scaling. + // This function is async. + bool video_get_first_frame(const char *filepath, const char *destination_path, int width = 0, int height = 0); class FileAnalyzer { public: FileAnalyzer(); - bool load_file(const char *filepath); + bool load_file(const char *filepath, bool load_file_metadata = true); ContentType get_content_type() const; size_t get_file_size() const; std::optional<Dimensions> get_dimensions() const; diff --git a/src/AsyncImageLoader.cpp b/src/AsyncImageLoader.cpp index b5a61ae..1c676c1 100644 --- a/src/AsyncImageLoader.cpp +++ b/src/AsyncImageLoader.cpp @@ -1,10 +1,13 @@ #include "../include/AsyncImageLoader.hpp" +#include "../include/FileAnalyzer.hpp" #include "../include/DownloadUtils.hpp" #include "../include/Program.hpp" #include "../include/ImageUtils.hpp" #include "../include/Scale.hpp" #include "../include/SfmlFixes.hpp" #include "../external/hash-library/sha256.h" + +#include <sys/stat.h> #include <assert.h> namespace QuickMedia { @@ -29,6 +32,20 @@ namespace QuickMedia { // Create thumbnail and load it. On failure load the original image static void create_thumbnail(const Path &thumbnail_path, const Path &thumbnail_path_resized, ThumbnailData *thumbnail_data, sf::Vector2i resize_target_size) { + FileAnalyzer file_analyzer; + if(!file_analyzer.load_file(thumbnail_path.data.c_str(), false)) { + fprintf(stderr, "Failed to convert %s to a thumbnail, using the original image\n", thumbnail_path.data.c_str()); + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + return; + } + + if(is_content_type_video(file_analyzer.get_content_type())) { + if(video_get_first_frame(thumbnail_path.data.c_str(), thumbnail_path_resized.data.c_str(), resize_target_size.x, resize_target_size.y)) + load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data); + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + return; + } + if(create_thumbnail(thumbnail_path, thumbnail_path_resized, resize_target_size)) { load_image_from_file(*thumbnail_data->image, thumbnail_path_resized.data); } else { @@ -115,11 +132,21 @@ namespace QuickMedia { SHA256 sha256; sha256.add(url.data(), url.size()); Path thumbnail_path = get_cache_dir().join("thumbnails").join(sha256.getHash()); - if(local && get_file_type(url) == FileType::REGULAR) { + if(local) { + struct stat file_stat; + if(stat(url.c_str(), &file_stat) != 0 || !S_ISREG(file_stat.st_mode)) { + thumbnail_data->image = std::make_unique<sf::Image>(); + thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; + return; + } + + thumbnail_path.append("_" + std::to_string(file_stat.st_mtim.tv_sec)); thumbnail_data->loading_state = LoadingState::LOADING; image_load_queue.push({ url, thumbnail_path, true, thumbnail_data, resize_target_size }); return; - } else if(get_file_type(thumbnail_path) == FileType::REGULAR) { + } + + if(get_file_type(thumbnail_path) == FileType::REGULAR) { thumbnail_data->loading_state = LoadingState::LOADING; image_load_queue.push({ url, thumbnail_path, false, thumbnail_data, resize_target_size }); return; @@ -135,7 +162,7 @@ namespace QuickMedia { download_image_thread[free_index].join(); // TODO: Keep the thread running and use conditional variable instead to sleep until a new image should be loaded. Same in ImageViewer. - download_image_thread[free_index] = std::thread([this, free_index, thumbnail_path, url, local, resize_target_size, thumbnail_data]() mutable { + download_image_thread[free_index] = std::thread([this, free_index, thumbnail_path, url, resize_target_size, thumbnail_data]() mutable { thumbnail_data->image = std::make_unique<sf::Image>(); Path thumbnail_path_resized = thumbnail_path; @@ -149,22 +176,16 @@ namespace QuickMedia { return; } - if(!local && get_file_type(thumbnail_path.data) != FileType::REGULAR && download_to_file(url, thumbnail_path.data, {}, true) != DownloadResult::OK) { + if(get_file_type(thumbnail_path.data) == FileType::FILE_NOT_FOUND && download_to_file(url, thumbnail_path.data, {}, true) != DownloadResult::OK) { thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; loading_image[free_index] = false; return; } - Path thumbnail_original_path; - if(local) - thumbnail_original_path = url; - else - thumbnail_original_path = thumbnail_path; - if(resize_target_size.x != 0 && resize_target_size.y != 0) - create_thumbnail(thumbnail_original_path, thumbnail_path_resized, thumbnail_data.get(), resize_target_size); + create_thumbnail(thumbnail_path, thumbnail_path_resized, thumbnail_data.get(), resize_target_size); else - load_image_from_file(*thumbnail_data->image, thumbnail_original_path.data); + load_image_from_file(*thumbnail_data->image, thumbnail_path.data); thumbnail_data->loading_state = LoadingState::FINISHED_LOADING; loading_image[free_index] = false; diff --git a/src/FileAnalyzer.cpp b/src/FileAnalyzer.cpp index b397def..ccad221 100644 --- a/src/FileAnalyzer.cpp +++ b/src/FileAnalyzer.cpp @@ -1,4 +1,5 @@ #include "../include/FileAnalyzer.hpp" +#include "../include/AsyncImageLoader.hpp" #include "../include/Program.hpp" #include <sys/stat.h> #include <stdio.h> @@ -20,14 +21,16 @@ namespace QuickMedia { // What about audio ogg files that are not opus? // TODO: Test all of these - static const std::array<MagicNumber, 23> magic_numbers = { + static const std::array<MagicNumber, 25> magic_numbers = { MagicNumber{ {'R', 'I', 'F', 'F', -1, -1, -1, -1, 'A', 'V', 'I', ' '}, 12, ContentType::VIDEO_AVI }, MagicNumber{ {0x00, 0x00, 0x00, -1, 'f', 't', 'y', 'p', 'i', 's', 'o', 'm'}, 12, ContentType::VIDEO_MP4 }, MagicNumber{ {0x00, 0x00, 0x00, -1, 'f', 't', 'y', 'p', 'm', 'p', '4', '2'}, 12, ContentType::VIDEO_MP4 }, + MagicNumber{ {0x00, 0x00, 0x00, -1, 'f', 't', 'y', 'p', '3', 'g', 'p', '4'}, 12, ContentType::VIDEO_MP4 }, MagicNumber{ {0x00, 0x00, 0x00, -1, 'f', 't', 'y', 'm', 'p', '4', '2'}, 11, ContentType::VIDEO_MP4 }, MagicNumber{ {0x00, 0x00, 0x00, -1, 'f', 't', 'y', '3', 'g', 'p', '5'}, 11, ContentType::VIDEO_MP4 }, MagicNumber{ {0x00, 0x00, 0x00, -1, 'f', 't', 'y', 'p', 'q', 't'}, 10, ContentType::VIDEO_MP4 }, MagicNumber{ {0x1A, 0x45, 0xDF, 0xA3}, 4, ContentType::VIDEO_WEBM }, + MagicNumber{ {'F', 'L', 'V', 0x01}, 4, ContentType::VIDEO_FLV }, MagicNumber{ {'.', 's', 'n', 'd'}, 4, ContentType::AUDIO_BASIC }, MagicNumber{ {'F', 'O', 'R', 'M', -1, -1, -1, -1, 'A', 'I', 'F', 'F'}, 12, ContentType::AUDIO_AIFF }, MagicNumber{ { 'I', 'D', '3' }, 3, ContentType::AUDIO_MPEG }, @@ -48,7 +51,7 @@ namespace QuickMedia { }; bool is_content_type_video(ContentType content_type) { - return content_type >= ContentType::VIDEO_AVI && content_type <= ContentType::VIDEO_WEBM; + return content_type >= ContentType::VIDEO_AVI && content_type <= ContentType::VIDEO_FLV; } bool is_content_type_audio(ContentType content_type) { @@ -65,6 +68,7 @@ namespace QuickMedia { case ContentType::VIDEO_AVI: return "video/avi"; case ContentType::VIDEO_MP4: return "video/mp4"; case ContentType::VIDEO_WEBM: return "video/webm"; + case ContentType::VIDEO_FLV: return "video/x-flv"; case ContentType::AUDIO_BASIC: return "audio/basic"; case ContentType::AUDIO_AIFF: return "audio/aiff"; case ContentType::AUDIO_MPEG: return "audio/mpeg"; @@ -81,20 +85,57 @@ namespace QuickMedia { return "application/octet-stream"; } + bool is_image_ext(const char *ext) { + return strcasecmp(ext, ".jpg") == 0 + || strcasecmp(ext, ".jpeg") == 0 + || strcasecmp(ext, ".png") == 0 + || strcasecmp(ext, ".gif") == 0 + || strcasecmp(ext, ".webp") == 0; + } + + bool is_video_ext(const char *ext) { + return strcasecmp(ext, ".webm") == 0 + || strcasecmp(ext, ".mkv") == 0 + || strcasecmp(ext, ".flv") == 0 + || strcasecmp(ext, ".vob") == 0 + || strcasecmp(ext, ".ogv") == 0 + || strcasecmp(ext, ".avi") == 0 + //|| strcasecmp(ext, ".ts") == 0 + || strcasecmp(ext, ".mov") == 0 + || strcasecmp(ext, ".qt") == 0 + || strcasecmp(ext, ".wmv") == 0 + || strcasecmp(ext, ".mp4") == 0 + || strcasecmp(ext, ".m4v") == 0 + || strcasecmp(ext, ".mpg") == 0 + || strcasecmp(ext, ".mpeg") == 0 + || strcasecmp(ext, ".3gp") == 0; + } + static int accumulate_string(char *data, int size, void *userdata) { std::string *str = (std::string*)userdata; str->append(data, size); return 0; } - bool video_get_first_frame(const char *filepath, const char *destination_path) { - const char *program_args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", filepath, "-vframes", "1", "-f", "singlejpeg", destination_path, nullptr }; + bool video_get_first_frame(const char *filepath, const char *destination_path, int width, int height) { + Path destination_path_tmp = destination_path; + destination_path_tmp.append(".ftmp"); + + const char *program_args[] = { "ffmpeg", "-y", "-v", "quiet", "-i", filepath, "-vframes", "1", "-f", "singlejpeg", destination_path_tmp.data.c_str(), nullptr }; std::string ffmpeg_result; if(exec_program(program_args, nullptr, nullptr) != 0) { fprintf(stderr, "Failed to execute ffmpeg, maybe its not installed?\n"); return false; } - return true; + + if(width > 0 || height > 0) { + if(create_thumbnail(destination_path_tmp, destination_path, sf::Vector2i(width, height))) { + remove(destination_path_tmp.data.c_str()); + return true; + } + } + + return rename(destination_path_tmp.data.c_str(), destination_path) == 0; } // TODO: Remove dependency on ffprobe @@ -154,7 +195,7 @@ namespace QuickMedia { } - bool FileAnalyzer::load_file(const char *filepath) { + bool FileAnalyzer::load_file(const char *filepath, bool load_file_metadata) { if(loaded) { fprintf(stderr, "File already loaded\n"); return false; @@ -182,6 +223,11 @@ namespace QuickMedia { unsigned char magic_number_buffer[MAGIC_NUMBER_BUFFER_SIZE]; size_t num_bytes_read = fread(magic_number_buffer, 1, sizeof(magic_number_buffer), file); + if(feof(file)) { + perror(filepath); + fclose(file); + return false; + } fclose(file); for(const MagicNumber &magic_number : magic_numbers) { @@ -200,7 +246,7 @@ namespace QuickMedia { } } - if(content_type != ContentType::UNKNOWN) { + if(load_file_metadata && content_type != ContentType::UNKNOWN) { if(!ffprobe_extract_metadata(filepath, dimensions, duration_seconds)) { // This is not an error, matrix allows files to be uploaded without metadata fprintf(stderr, "Failed to extract metadata from file: %s, is ffprobe not installed?\n", filepath); diff --git a/src/ImageUtils.cpp b/src/ImageUtils.cpp index 25a6680..3310ea0 100644 --- a/src/ImageUtils.cpp +++ b/src/ImageUtils.cpp @@ -122,8 +122,4 @@ namespace QuickMedia { } return false; } - - bool is_image_ext(const char *ext) { - return strcasecmp(ext, ".jpg") == 0 || strcasecmp(ext, ".jpeg") == 0 || strcasecmp(ext, ".png") == 0 || strcasecmp(ext, ".gif") == 0 || strcasecmp(ext, ".webp") == 0; - } }
\ No newline at end of file diff --git a/src/QuickMedia.cpp b/src/QuickMedia.cpp index c9a206e..fbc2699 100644 --- a/src/QuickMedia.cpp +++ b/src/QuickMedia.cpp @@ -1649,7 +1649,7 @@ namespace QuickMedia { const char *args[] = { "curl", "-sLf", "-r", "0-40", "-H", useragent_str, "--", url, nullptr }; exec_program(args, accumulate_string_limit_head, &result, 42); return (result.size() >= 42) - && (memcmp(&result[4], "ftypisom", 8) == 0 || memcmp(&result[4], "ftypmp42", 8) == 0 || memcmp(&result[4], "ftymp42", 7) == 0 || memcmp(&result[4], "fty3gp5", 7) == 0 || memcmp(&result[4], "ftypqt", 6) == 0) + && (memcmp(&result[4], "ftypisom", 8) == 0 || memcmp(&result[4], "ftypmp42", 8) == 0 || memcmp(&result[4], "ftymp42", 7) == 0 || memcmp(&result[4], "ftyp3gp4", 8) == 0|| memcmp(&result[4], "fty3gp5", 7) == 0 || memcmp(&result[4], "ftypqt", 6) == 0) && (memmem(&result[0], result.size(), "moov", 4) == NULL); } diff --git a/src/plugins/FileManager.cpp b/src/plugins/FileManager.cpp index f65486e..f15deae 100644 --- a/src/plugins/FileManager.cpp +++ b/src/plugins/FileManager.cpp @@ -1,5 +1,5 @@ #include "../../plugins/FileManager.hpp" -#include "../../include/ImageUtils.hpp" +#include "../../include/FileAnalyzer.hpp" #include "../../include/QuickMedia.hpp" namespace QuickMedia { @@ -82,7 +82,8 @@ namespace QuickMedia { for(auto &p : paths) { auto body_item = BodyItem::create(p.path().filename().string()); // TODO: Check file magic number instead of extension? - if(p.is_regular_file() && is_image_ext(get_ext(p.path()))) { + const char *ext = get_ext(p.path()); + if(p.is_regular_file() && (is_image_ext(ext) || is_video_ext(ext))) { body_item->thumbnail_is_local = true; body_item->thumbnail_url = p.path().string(); } diff --git a/src/plugins/Matrix.cpp b/src/plugins/Matrix.cpp index ccae6ba..daae545 100644 --- a/src/plugins/Matrix.cpp +++ b/src/plugins/Matrix.cpp @@ -3205,28 +3205,13 @@ namespace QuickMedia { char tmp_filename[] = "/tmp/quickmedia_video_frame_XXXXXX"; int tmp_file = mkstemp(tmp_filename); if(tmp_file != -1) { - if(video_get_first_frame(filepath.c_str(), tmp_filename)) { - char tmp_filename_thumbnail[] = "/tmp/quickmedia_thumbnail_XXXXXX"; - int tmp_file_thumbnail = mkstemp(tmp_filename_thumbnail); - if(tmp_file_thumbnail != -1) { - std::string thumbnail_path; - if(create_thumbnail(tmp_filename, tmp_filename_thumbnail, thumbnail_max_size)) - thumbnail_path = tmp_filename_thumbnail; - else - thumbnail_path = tmp_filename; - - UploadInfo upload_info_ignored; // Ignore because it wont be set anyways. Thumbnails dont have thumbnails. - PluginResult upload_thumbnail_result = upload_file(room, thumbnail_path, thumbnail_info, upload_info_ignored, err_msg, false); - if(upload_thumbnail_result != PluginResult::OK) { - close(tmp_file_thumbnail); - remove(tmp_filename_thumbnail); - return upload_thumbnail_result; - } - - close(tmp_file_thumbnail); - remove(tmp_filename_thumbnail); - } else { - fprintf(stderr, "Failed to create temporary file for video thumbnail, ignoring thumbnail...\n"); + if(video_get_first_frame(filepath.c_str(), tmp_filename, thumbnail_max_size.x, thumbnail_max_size.y)) { + UploadInfo upload_info_ignored; // Ignore because it wont be set anyways. Thumbnails dont have thumbnails. + PluginResult upload_thumbnail_result = upload_file(room, tmp_filename, thumbnail_info, upload_info_ignored, err_msg, false); + if(upload_thumbnail_result != PluginResult::OK) { + close(tmp_file); + remove(tmp_filename); + return upload_thumbnail_result; } } else { fprintf(stderr, "Failed to get first frame of video, ignoring thumbnail...\n"); |