From 48e757baffbf75bc8a1e4171ad94c27d7356cafa Mon Sep 17 00:00:00 2001 From: dec05eba Date: Fri, 7 May 2021 07:38:59 +0200 Subject: Migrate mangadex to new api, remove .in_progress files in tracked dir if they are old --- mangadex-upgrade.py | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100755 mangadex-upgrade.py (limited to 'mangadex-upgrade.py') diff --git a/mangadex-upgrade.py b/mangadex-upgrade.py new file mode 100755 index 0000000..36533fa --- /dev/null +++ b/mangadex-upgrade.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +import os +import json +import re +import requests + +class Chapter: + id = "" + title = "" + time = "" + +class Manga: + id = "" + updated = "" + chapters = None + directory = "" + + def __init__(self): + self.chapters = [] + +# Returns None if the manga is not tracked using mangadex or if it has already been migrated +def manga_data_file_read(filepath): + with open(filepath, "rb") as file: + data = json.load(file) + if data["plugin"] != "mangadex.py": + return None + + manga_link = data["link"] + title_search = re.search(r"/title/([0-9]+)", manga_link) + if not title_search or len(title_search.groups()) == 0: + print("Mangadex manga already migrated: %s" % manga_link) + return None + + manga_data = Manga() + manga_data.id = int(title_search.group(1)) + manga_data.updated = data["updated"] + + downloaded = data.get("downloaded") + if type(downloaded) is list: + for downloaded_item in downloaded: + chapter_url = downloaded_item["url"] + chapter_title_search = re.search(r"chapter/([0-9]+)", chapter_url) + if not chapter_title_search: + print("Failed to extract title from manga %s, chapter %s" % (manga_link, chapter_url)) + exit(1) + + chapter_data = Chapter() + chapter_data.id = int(chapter_title_search.group(1)) + chapter_data.title = downloaded_item["title"] + chapter_data.time = downloaded_item["time"] + manga_data.chapters.append(chapter_data) + return manga_data + +# Returns a list with tuple where the first element is the legacy id and the second element is the new id +def legacy_id_to_new_id(ids, type): + mapping = [] + id_start = 0 + while id_start < len(ids): + id_end = min(id_start + 1000, len(ids)) + if id_end - id_start == 0: + break + response = requests.post("https://api.mangadex.org/legacy/mapping", json={"type": type, "ids": ids[id_start:id_end]}) + response.raise_for_status() + response_json = response.json() + + for response_item in response_json: + if response_item["result"] != "ok": + print("legacy mapping returned an error") + exit(1) + + attributes = response_item["data"]["attributes"] + mapping.append((attributes["legacyId"], attributes["newId"])) + + id_start = id_end + + if len(mapping) != len(ids): + print("Failed to get the legacy to new id mapping for all ids. Got %d mappings, expected %d" % (len(mapping), len(ids))) + exit(1) + + return mapping + +def get_manga_by_id(manga_list, manga_id): + for manga_data in manga_list: + if manga_data.id == manga_id: + return manga_data + +def get_chapter_by_id_in_legacy_mapping(legacy_chapter_ids_to_new_ids, id): + for chapter_id_mapping in legacy_chapter_ids_to_new_ids: + if chapter_id_mapping[0] == id: + return chapter_id_mapping + +def file_overwrite_atomic(filepath, content): + tmp_filepath = filepath + ".tmp" + with open(tmp_filepath, "wb") as file: + file.write(content.encode()) + file.flush() + os.fsync(file.fileno()) + os.rename(tmp_filepath, filepath) + +if __name__ == "__main__": + migrate_finished_filepath = os.path.expanduser("~/.config/automedia/mangadex-upgraded") + if os.path.isfile(migrate_finished_filepath): + print("Mangadex tracked manga have already migrated, nothing to do") + exit(0) + + tracked_dir = os.path.expanduser("~/.config/automedia/html/tracked") + manga_list = [] + for manga_name in os.listdir(tracked_dir): + manga_tracked_dir = os.path.join(tracked_dir, manga_name) + manga_data_file = os.path.join(manga_tracked_dir, "data") + manga_data = manga_data_file_read(manga_data_file) + if manga_data: + manga_data.directory = os.path.join(tracked_dir, manga_name) + manga_list.append(manga_data) + + manga_ids = [] + chapter_ids = [] + cc = {} + for manga_data in manga_list: + manga_ids.append(manga_data.id) + for chapter_data in manga_data.chapters: + chapter_ids.append(chapter_data.id) + + legancy_manga_ids_to_new_ids = legacy_id_to_new_id(manga_ids, "manga") + legacy_chapter_ids_to_new_ids = legacy_id_to_new_id(chapter_ids, "chapter") + + for manga_id in legancy_manga_ids_to_new_ids: + manga_data = get_manga_by_id(manga_list, manga_id[0]) + if not manga_data: + print("Failed to get manga by id: %s" % manga_id[0]) + exit(1) + + new_manga_link = str(manga_id[1]) + new_manga_data = {} + new_manga_data["plugin"] = "mangadex.py" + new_manga_data["link"] = new_manga_link + new_manga_data["updated"] = manga_data.updated + downloaded = [] + for chapter_data in manga_data.chapters: + chapter_id_mapping = get_chapter_by_id_in_legacy_mapping(legacy_chapter_ids_to_new_ids, chapter_data.id) + if not chapter_id_mapping: + print("Failed to get new id from manga %d, chapter %d" % (manga_data.id, chapter_data.id)) + exit(1) + + downloaded_item = {} + downloaded_item["title"] = chapter_data.title + downloaded_item["time"] = chapter_data.time + downloaded_item["url"] = chapter_id_mapping[1] + downloaded.append(downloaded_item) + new_manga_data["downloaded"] = downloaded + + file_overwrite_atomic(os.path.join(manga_data.directory, "link"), new_manga_link) + new_manga_data_str = json.dumps(new_manga_data, indent=4) + file_overwrite_atomic(os.path.join(manga_data.directory, "data"), new_manga_data_str) + + file_overwrite_atomic(migrate_finished_filepath, "1") + print("Successfully migrated %d manga with a total of %d chapters" % (len(manga_ids), len(chapter_ids))) + \ No newline at end of file -- cgit v1.2.3