diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | .idea/.gitignore | 8 | ||||
| -rw-r--r-- | .idea/inspectionProfiles/profiles_settings.xml | 6 | ||||
| -rw-r--r-- | .idea/misc.xml | 7 | ||||
| -rw-r--r-- | __pycache__/config.cpython-312.pyc | bin | 0 -> 946 bytes | |||
| -rw-r--r-- | __pycache__/imgutils.cpython-312.pyc | bin | 0 -> 10849 bytes | |||
| -rw-r--r-- | __pycache__/render.cpython-311.pyc | bin | 0 -> 6177 bytes | |||
| -rw-r--r-- | __pycache__/render.cpython-312.pyc | bin | 0 -> 11732 bytes | |||
| -rw-r--r-- | assets/chapter.html.jinja | 45 | ||||
| -rwxr-xr-x | assets/favicon.ico | bin | 0 -> 67646 bytes | |||
| -rw-r--r-- | assets/home.html.jinja | 32 | ||||
| -rwxr-xr-x | assets/home.mp4 | bin | 0 -> 13354 bytes | |||
| -rw-r--r-- | assets/index.html.jinja | 65 | ||||
| -rwxr-xr-x | assets/lexend.woff2 | bin | 0 -> 71592 bytes | |||
| -rw-r--r-- | assets/manifest.json.jinja | 16 | ||||
| -rw-r--r-- | assets/series.html.jinja | 39 | ||||
| -rw-r--r-- | assets/style.css.jinja | 178 | ||||
| -rw-r--r-- | bake.py | 237 | ||||
| -rw-r--r-- | config.py | 54 | ||||
| -rw-r--r-- | imgutils.py | 230 | ||||
| -rw-r--r-- | render.py | 209 | ||||
| -rw-r--r-- | requirements.txt | 6 |
22 files changed, 1133 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d2a9b3 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bake
\ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ +<component name="InspectionProjectProfileManager"> + <settings> + <option name="USE_PROJECT_PROFILE" value="false" /> + <version value="1.0" /> + </settings> +</component>
\ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..0a88d55 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="Black"> + <option name="sdkName" value="manga-bake" /> + </component> + <component name="ProjectRootManager" version="2" project-jdk-name="manga-bake" project-jdk-type="Python SDK" /> +</project>
\ No newline at end of file diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc Binary files differnew file mode 100644 index 0000000..d51d7c8 --- /dev/null +++ b/__pycache__/config.cpython-312.pyc diff --git a/__pycache__/imgutils.cpython-312.pyc b/__pycache__/imgutils.cpython-312.pyc Binary files differnew file mode 100644 index 0000000..b945405 --- /dev/null +++ b/__pycache__/imgutils.cpython-312.pyc diff --git a/__pycache__/render.cpython-311.pyc b/__pycache__/render.cpython-311.pyc Binary files differnew file mode 100644 index 0000000..526dff5 --- /dev/null +++ b/__pycache__/render.cpython-311.pyc diff --git a/__pycache__/render.cpython-312.pyc b/__pycache__/render.cpython-312.pyc Binary files differnew file mode 100644 index 0000000..6aa555a --- /dev/null +++ b/__pycache__/render.cpython-312.pyc diff --git a/assets/chapter.html.jinja b/assets/chapter.html.jinja new file mode 100644 index 0000000..972ac75 --- /dev/null +++ b/assets/chapter.html.jinja @@ -0,0 +1,45 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>{{ site }} - {{ series_name }} - {{ chapter_name }}</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <link rel="preload" href="{{ base }}lexend.woff2" as="font" type="font/woff2" crossorigin> + <link rel="stylesheet" href="{{ base }}{{ style }}"> + <link rel="icon" href="{{ base }}favicon.ico" sizes="any"> +</head> +<body> + + <a class="back" href="{{ base }}{{ series_loc }}/">Back</a> + + {% if prev_loc or next_loc %} + <div class="nav"> + {% if prev_loc %} + <a class="nav-link" href="{{ base }}{{ prev_loc }}/">Previous</a> + {% endif %} + + {% if next_loc %} + <a class="nav-link right" href="{{ base }}{{ next_loc }}/">Next</a> + {% endif %} + </div> + {% endif %} + + <div class="container"> + {% for im in images %} + <img src="{{ base }}{{ loc }}/{{ im }}" {{ 'loading="lazy"' if loop.index != 1 }} alt="page {{ loop.index }}"> + {% endfor %} + </div> + + {% if prev_loc or next_loc %} + <div class="nav" style="margin-bottom: 60px"> + {% if prev_loc %} + <a class="nav-link" href="{{ base }}{{ prev_loc }}/">Previous</a> + {% endif %} + + {% if next_loc %} + <a class="nav-link right" href="{{ base }}{{ next_loc }}/">Next</a> + {% endif %} + </div> + {% endif %} +</body> +</html>
\ No newline at end of file diff --git a/assets/favicon.ico b/assets/favicon.ico Binary files differnew file mode 100755 index 0000000..30f46aa --- /dev/null +++ b/assets/favicon.ico diff --git a/assets/home.html.jinja b/assets/home.html.jinja new file mode 100644 index 0000000..fbd69c3 --- /dev/null +++ b/assets/home.html.jinja @@ -0,0 +1,32 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>{{ site }}</title> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <link rel="preload" href="{{ base }}lexend.woff2" as="font" type="font/woff2" crossorigin> + <link rel="stylesheet" href="{{ base }}{{ style }}"> + <link rel="icon" href="{{ base }}favicon.ico" sizes="any"> + <link rel="manifest" href="{{ base }}manifest.json"> +</head> +<body> +<div style="padding: 20px"> + + <h1>{{ home_title }}</h1> + + <a class="home-link" href="{{ base }}t/long-strip/">Tags</a> + <br> + <a class="home-link" href="{{ base }}i/a/">Titles</a> + + <br> + <div style="text-align: center"> + <video src="{{ base }}home.mp4" width="128" height="128" loop="" playsinline="" preload="none" autoplay muted></video> + </div> + + <span style="position: absolute; bottom: 0; left: 0; padding: 15px"> + Made with <span style="color: pink">&lt;3</span> by <span style="color: pink">Shy</span> ♥ + </span> +</div> +</body> +</html> diff --git a/assets/home.mp4 b/assets/home.mp4 Binary files differnew file mode 100755 index 0000000..db28071 --- /dev/null +++ b/assets/home.mp4 diff --git a/assets/index.html.jinja b/assets/index.html.jinja new file mode 100644 index 0000000..5370ad7 --- /dev/null +++ b/assets/index.html.jinja @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>{{ site }} - {{ name | upper }}</title> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <link rel="preload" href="{{ base }}lexend.woff2" as="font" type="font/woff2" crossorigin> + <link rel="stylesheet" href="{{ base }}{{ style }}"> + <link rel="icon" href="{{ base }}favicon.ico" sizes="any"> +</head> +<body> +<div style="padding: 20px"> + + <a class="back" href="{{ base }}">Back</a> + + <h1>Index - {{ name | upper }}</h1> + + <div class="index {{ "index-tags" if tags }}"> + {% for page, page_raw, page_loc, not_empty in index_pages %} + {% if page_raw == loc %} + <span><b><u>{{ page }}</u></b></span> + {% elif not_empty %} + <a href="{{ base }}{{ page_loc }}/">{{ page }}</a> + {% else %} + <span>{{ page }}</span> + {% endif %} + {% endfor %} + </div> + + <br> + + <div class="index index-rating"> + {% if rating == None %} + <span>Any</span> + {% else %} + <a href="{{ base }}{{ prefix }}/{{ loc }}">Any</a> + {% endif %} + + {% for r in content_ratings %} + {% if rating == r %} + <span>{{ r }}</span> + {% else %} + <a href="{{ base }}{{ prefix }}-{{ r | lower }}/{{ loc }}/">{{ r }}</a> + {% endif %} + {% endfor %} + </div> +</div> + +<div class="gallery"> + {% for tn in thumbnails %} + <a href="{{ base }}{{ tn["loc"] }}/"> + <figure> + <img class="t {{ tn["offset"] }}" src="{{ base }}{{ tn["img"] }}" {{ 'loading="lazy"' if loop.index > 32 }} alt=""> + <figcaption>{{ tn["name"] }}</figcaption> + </figure> + </a> + {% endfor %} + + {% if not thumbnails %} + <p style="font-style: italic">(Empty)</p> + {% endif %} +</div> +</body> +</html>
\ No newline at end of file diff --git a/assets/lexend.woff2 b/assets/lexend.woff2 Binary files differnew file mode 100755 index 0000000..89fe074 --- /dev/null +++ b/assets/lexend.woff2 diff --git a/assets/manifest.json.jinja b/assets/manifest.json.jinja new file mode 100644 index 0000000..49fb73e --- /dev/null +++ b/assets/manifest.json.jinja @@ -0,0 +1,16 @@ +{ + "name": "Yuri Directory", + "short_name": "Yuri", + "theme_color": "#6495ed", + "background_color": "#000000", + "display": "minimal-ui", + "orientation": "portrait", + "scope": "{{ base }}", + "start_url": "{{ base }}", + "icons": [ + { + "src": "https://yuri.directory{{ base }}favicon.ico", + "sizes": "any" + } + ] +}
\ No newline at end of file diff --git a/assets/series.html.jinja b/assets/series.html.jinja new file mode 100644 index 0000000..81722ed --- /dev/null +++ b/assets/series.html.jinja @@ -0,0 +1,39 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>{{ site }} - {{ series_name }}</title> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <link rel="preload" href="{{ base }}lexend.woff2" as="font" type="font/woff2" crossorigin> + <link rel="stylesheet" href="{{ base }}{{ style }}"> + <link rel="icon" href="{{ base }}favicon.ico" sizes="any"> +</head> +<body> + + <div class="sheet"> + + <a class="back" href="javascript:(new URL(document.referrer)).pathname.match(/^\/[it]\/.*/) ? history.back() : window.location.href = '{{ base }}'">Back</a> + + <h1>{{ series_name }}</h1> + + <img class="cover" src="{{ base }}{{ loc }}/cover.webp" alt="cover"> + + <p><b>Content rating:</b> {{ rating }}</p> + + <h3>Tags</h3> + <ul class="li-link"> + {% for tag in tags %} + <li><a href="{{ base }}t/{{ tag | slug }}/">{{ tag }}</a></li> + {% endfor %} + </ul> + + <h3>Chapters</h3> + <div class="nav nav-vertical"> + {% for chapter in chapters %} + <a class="nav-link" href="{{ base }}{{ chapter["location"] }}/">{{ chapter["name"] }}</a> + {% endfor %} + </div> + </div> + +</body> +</html>
\ No newline at end of file diff --git a/assets/style.css.jinja b/assets/style.css.jinja new file mode 100644 index 0000000..0265333 --- /dev/null +++ b/assets/style.css.jinja @@ -0,0 +1,178 @@ +@font-face { + font-family: lexend; + src: url("{{ base }}lexend.woff2"); +} + +.container { + display: flex; + flex-direction: column; + margin-left: auto; + margin-right: auto; + overflow-x: hidden; +} + +img { + margin: 0; + width: 100%; +} + +.cover { + width: 100%; +} + +body { + background-color: black; + color: white; + font-family: lexend, arial, serif; +} + +a { + color: white; +} + +.index { + font-size: 24px; + display: flex; + flex-wrap: wrap; + justify-content: space-between; + gap: 20px; +} + +.index-tags { + font-size: 18px; + gap: 3px; +} + +.index-rating { + gap: 5px; + flex-wrap: nowrap; + max-width: 400px; + margin-top: 15px; +} + +.index span { + color: rgb(255, 255, 255, 0.5); +} + +.nav { + display: flex; + justify-content: space-between; + gap: 10px; + margin-bottom: 20px; + margin-top: 20px; +} + +.nav-vertical { + flex-direction: column; +} + +.nav-link { + width: 100%; + border: 1px solid white; + line-height: 2; + padding: 0 5px; +} + +.nav-vertical .nav-link:visited { + border: 1px solid pink; +} + +.home-link { + border: 1px solid white; + line-height: 2; + font-size: 26px; + padding: 5px; + margin-bottom: 20px; +} + +.right { + text-align: right; +} + +.gallery { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: space-around; + gap: 3px; +} + +a:has(> figure) { + margin: 0; + font-size: 12px; + font-weight: bold; + text-align: center; +} + +figure { + margin: 0; +} + +@media screen and (min-width: 1000px) { + + .cover { + width: 360px; + } + + .container { + max-width: 720px; + } + + .gallery { + margin-left: 5%; + margin-right: 5%; + justify-content: left; + gap: 16px; + } + + .index { + font-size: 18px; + margin-left: 5%; + margin-right: 5%; + } + a:has(> figure):hover { + text-decoration: underline; + } +} + +a:has(> figure):visited { + color: pink; +} + +a { + text-decoration: none; +} + +a:not(:has(> figure)):hover { + text-decoration: underline; +} + +.sheet { + margin-left: 10%; + margin-right: 10%; +} + +.li-link { + font-size: 24px; + line-height: 1.5; +} + +.back { + font-size: 20px; +} + +figure { + width: {{ thumbnail_width }}px; +} + +.t { + width: {{ thumbnail_width}}px; + height: {{thumbnail_height}}px; + object-fit: none; +} + +{% for class_name, offset in offsets %} +.{{ class_name }} { + object-position: {{ -offset }}px 0 +} +{% endfor %}
\ No newline at end of file @@ -0,0 +1,237 @@ +import os +import json +import shutil +import string +from glob import glob +from slugify import slugify +from tqdm import tqdm +import statistics + +from config import INPUT_DIRECTORY, OUTPUT_DIRECTORY, DOWNLOAD_FILES, CONTENT_RATINGS +from render import render_chapter_pages, render_series_pages, render_index_pages, render_home, render_style + +BAD_TAGS = [ + "girls-love" +] + +SERIES_LOC = set() + +def chapter_loc(loc, i): + return f"{loc}/{int(i)+1:x}" + +def _loc_with_num(loc, num): + if num == 0: + return loc + return f"{loc}.{num}" + +def series_loc(series): + + loc = slugify(series["sort_name"]) + original_loc = loc + + i = 0 + loc = _loc_with_num(original_loc, i) + while loc in SERIES_LOC: + loc = _loc_with_num(original_loc, i) + i += 1 + + SERIES_LOC.add(loc) + + return loc + + +def parse_metadata(metadata, series_dir): + result = {} + + result["id"] = metadata["id"] + + result["name"] = metadata["title"] + result["rating"] = metadata["rating"] + + assert result["rating"] in CONTENT_RATINGS + + sort_name = metadata["title"].lower() + if sort_name.startswith("the "): + sort_name = sort_name[4:] + result["sort_name"] = sort_name + loc = series_loc(result) + result["location"] = loc + result["description"] = metadata["description"] + result["cover"] = glob(os.path.join(series_dir, "cover.*"))[0] + tags = [slugify(tag) for tag in metadata["tags"]] + + for tag in BAD_TAGS: + if tag in tags: + tags.remove(tag) + + tags.append("all") + + result["tags"] = tags + + result["score"] = get_score(metadata["id"]) + + chapters = [] + for i, chapter_data in enumerate(metadata["chapters"]): + + chapter = chapter_data["name"] + chapter_dir = chapter.replace("Volume.", "Vol.").replace("Chapter.", "Ch.") + + chapter = { + "index": i, + "name": chapter, + "location": chapter_loc(loc, i), + "images": list( + im for im in sorted(glob(os.path.join(series_dir, chapter_dir, "*"))) + if im.endswith((".webp", ".png", ".jpeg", ".jpg")) + ) + } + + if len(chapter["images"]) == 0: + continue + + chapters.append(chapter) + + # Add prev/next links + for i, chapter in enumerate(chapters): + if i > 0: + chapter["prev"] = chapter_loc(loc, i-1) + + if i < len(chapters) - 1: + chapter["next"] = chapter_loc(loc, i+1) + + result["chapters"] = chapters + + + return result + +def create_index(directory): + index = [] + + i = 0 + + for dirname in tqdm(os.listdir(directory)): + series_dir = str(os.path.join(directory, dirname)) + + try: + with open(os.path.join(series_dir, "manga_info.json")) as f: + metadata = json.load(f)[0] + except FileNotFoundError: + continue + except PermissionError: + continue + + if not glob(os.path.join(series_dir, "cover.*")): + continue + + series_data = parse_metadata(metadata, series_dir) + + first_letter = series_data["sort_name"][0].lower() + series_data["index_letter"] = first_letter if first_letter in string.ascii_lowercase else "#" + + index.append(series_data) + + i += 1 + # if i > 100: + # break + + return index + +def load_scores(): + + score_files = [] + + for filename in DOWNLOAD_FILES: + with open(filename) as f: + num_lines = len([_ for _ in f]) + + scores = {} + + with open(filename) as f: + for i, line in enumerate(f): + + manga_id = line.strip().split("/")[-1] + score = num_lines - i + + scores[manga_id] = score + score_files.append(scores) + + return score_files + +def get_score(manga_id): + manga_scores = [] + + for score_file in scores: + if manga_id in score_file: + manga_scores.append(score_file[manga_id]) + + return statistics.mean(manga_scores) + +def _filter_index(original_index, r, keep_empty): + if r is None: + return original_index + + new_index = {} + + for key, s in original_index.items(): + new_list = list(filter(lambda x: x["rating"] == r, s)) + if new_list or keep_empty: + new_index[key] = new_list + + return new_index + + +if __name__ == "__main__": + + scores = load_scores() + + print("create index") + index = create_index(INPUT_DIRECTORY) + + print("Prepare output folder") + shutil.rmtree(OUTPUT_DIRECTORY, ignore_errors=True) + os.makedirs(OUTPUT_DIRECTORY) + + # Letter index + letter_index = {} + for letter in string.ascii_lowercase + "#": + letter_index[letter] = [] + + for series in index: + if series["index_letter"] == letter: + letter_index[letter].append(series) + + # Tag index + all_tags = set() + for series in index: + all_tags.update(series["tags"]) + all_tags = list(sorted(all_tags)) + + tag_index = {} + for tag in all_tags: + tag_index[tag] = [] + + for series in index: + if tag in series["tags"]: + tag_index[tag].append(series) + + render_home(OUTPUT_DIRECTORY) + render_style(OUTPUT_DIRECTORY) + print("series") + render_series_pages(OUTPUT_DIRECTORY, index) + print("chapters") + render_chapter_pages(OUTPUT_DIRECTORY, index) + + + print("letter index") + for i, rating in enumerate([None] + CONTENT_RATINGS): + filtered_tag_index = _filter_index(letter_index, rating, keep_empty=True) + render_index_pages(OUTPUT_DIRECTORY, f"i", filtered_tag_index, rating) + + print("tag index") + for i, rating in enumerate([None] + CONTENT_RATINGS): + filtered_letter_index = _filter_index(tag_index, rating, keep_empty=False) + render_index_pages(OUTPUT_DIRECTORY, f"t", filtered_letter_index, rating) + + for asset in ["lexend.woff2", "home.mp4", "favicon.ico"]: + shutil.copy(os.path.join("assets", asset), OUTPUT_DIRECTORY) + diff --git a/config.py b/config.py new file mode 100644 index 0000000..e442079 --- /dev/null +++ b/config.py @@ -0,0 +1,54 @@ +import os +import time + +STAGING = False +YAOI = True + +DOWNLOAD_FILES = [ + "/zfs-main/projects/yuri_directory_download/yaoi_follow.txt", + "/zfs-main/projects/yuri_directory_download/yaoi_rating.txt", +] if YAOI else [ + "/zfs-main/projects/yuri_directory_download/mangadex_follow.txt", + "/zfs-main/projects/yuri_directory_download/mangadex_rating.txt", +] + +INPUT_DIRECTORY = "/zfs-main/projects/yuri_directory_download/yaoi_downloads/" if YAOI else "/zfs-main/projects/yuri_directory_download/downloads/" +OUTPUT_DIRECTORY = "/zfs-main/www/yuri.directory/yaoi/" if YAOI else "/zfs-main/www/yuri.directory/" +if STAGING: + OUTPUT_DIRECTORY = os.path.join(OUTPUT_DIRECTORY, "staging") + + +CONTENT_RATINGS = [ + "Safe", "Suggestive", "Erotica" +] + +WILDCARD_LETTER = "#" + +PROCS = 30 +IMAGE_QUALITY = 65 +IMAGE_MAX_WIDTH = 1080 +IMAGE_MAX_HEIGHT = 2000 +IMAGE_STRIP_HEIGHT = 4096 + +THUMBNAIL_CHUNK_SIZE = 16 +THUMBNAIL_QUALITY = 70 +THUMBNAIL_WIDTH = 180 +THUMBNAIL_HEIGHT = 270 + +IMAGE_CACHE = "/zfs-main/projects/yuri_directory_bake/cache/" + +base = "/yaoi/" if YAOI else "/" +if STAGING: + base += "staging/" + + +GLOBALS = { + "site": "yuri.directory/yaoi" if YAOI else "yuri.directory", + "base": base, + "thumbnail_width": THUMBNAIL_WIDTH, + "thumbnail_height": THUMBNAIL_HEIGHT, + "offsets": [(f"o{i:x}", i * THUMBNAIL_WIDTH) for i in range(0, THUMBNAIL_CHUNK_SIZE)], + "style": f"style.{int(time.time()):x}.css", + "content_ratings": CONTENT_RATINGS, + "home_title": "Yaoi directory" if YAOI else "Yuri directory" +} diff --git a/imgutils.py b/imgutils.py new file mode 100644 index 0000000..955cf8f --- /dev/null +++ b/imgutils.py @@ -0,0 +1,230 @@ +import os +import shutil +import subprocess +from io import BytesIO +from tempfile import NamedTemporaryFile + +from PIL import Image, ImageOps + +from config import IMAGE_STRIP_HEIGHT, IMAGE_QUALITY, IMAGE_CACHE, IMAGE_MAX_WIDTH, THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT, \ + THUMBNAIL_QUALITY + +Image.MAX_IMAGE_PIXELS = 1000000000 +from glob import glob + +def _copy(src, dst): + try: + os.link(src, dst) + except: + shutil.copy(src, dst) + + +def rechunk(im): + + height = im.height + width = im.width + last_chunk = (height // IMAGE_STRIP_HEIGHT) - 1 + + for i, cursor in enumerate(range(0, height, IMAGE_STRIP_HEIGHT)): + + if i == last_chunk: + chunk_height = IMAGE_STRIP_HEIGHT + (height % IMAGE_STRIP_HEIGHT) + else: + chunk_height = min(IMAGE_STRIP_HEIGHT, height) + + yield im.crop((0, cursor, width, cursor + chunk_height)) + + if i == last_chunk: + break + +def webp_convert(input_file, output_file, quality=IMAGE_QUALITY): + + subprocess.run([ + "ffmpeg", "-y", "-loglevel", "error", "-hide_banner", "-nostats", "-i", input_file, + "-c:v", "libwebp", "-preset", "drawing", "-quality", str(quality), "-compression_level", "6", + output_file + ], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) + +def copy_from_cache(cache_folder, output_folder): + + for filename in glob(os.path.join(cache_folder, "*.webp")): + _copy(filename, os.path.join(output_folder, os.path.basename(filename))) + +def is_cached(cache_folder): + if os.path.exists(os.path.join(cache_folder, "ok")): + return True + + return False + +def convert_cover(cache_key, filename, output_folder): + cache_folder = os.path.join(IMAGE_CACHE, cache_key) + + os.makedirs(output_folder, exist_ok=True) + output_filename = os.path.join(output_folder, "cover.webp") + if os.path.exists(output_filename): + os.remove(output_filename) + + if is_cached(cache_folder): + copy_from_cache(cache_folder, output_folder) + return + + shutil.rmtree(cache_folder, ignore_errors=True) + os.makedirs(cache_folder) + + im = open_image(filename) + + if im.width > IMAGE_MAX_WIDTH: + aspect_ratio = im.height / im.width + new_height = int(IMAGE_MAX_WIDTH * aspect_ratio) + + im = im.resize((IMAGE_MAX_WIDTH, new_height), resample=Image.Resampling.LANCZOS) + + if im.mode != "RGB": + im = im.convert("RGB") + + try: + with NamedTemporaryFile(delete=False, suffix=".png") as f: + im.save(f) + + webp_convert(f.name, os.path.join(cache_folder, f"cover.webp"), quality=90) + + finally: + os.remove(f.name) + + with open(os.path.join(cache_folder, "ok"), "w"): + pass + + copy_from_cache(cache_folder, output_folder) + +def convert_rgb(im): + if im.mode == "RGBA": + background = Image.new('RGBA', im.size, color=(0, 0, 0)) + alpha_composite = Image.alpha_composite(background, im) + im = alpha_composite.convert("RGB") + + if im.mode != "RGB": + im = im.convert("RGB") + + return im + +def convert_rgba(im): + if im.mode != "RGBA": + im = im.convert("RGBA") + + return im + + +def convert_chapter(cache_key, images, output_folder): + + # TODO: for non-long strips, stack horizontally + # Limit height to + + cache_folder = os.path.join(IMAGE_CACHE, cache_key) + + shutil.rmtree(output_folder, ignore_errors=True) + os.makedirs(output_folder) + + if is_cached(cache_folder): + copy_from_cache(cache_folder, output_folder) + return + + shutil.rmtree(cache_folder, ignore_errors=True) + os.makedirs(cache_folder) + + resized = [] + + # Resize images + for filename in images: + im = open_image(filename) + + im = convert_rgb(im) + + if im.width > IMAGE_MAX_WIDTH: + + aspect_ratio = im.height / im.width + new_height = int(IMAGE_MAX_WIDTH * aspect_ratio) + + if new_height == 0: + continue + + im = im.resize((IMAGE_MAX_WIDTH, new_height), resample=Image.Resampling.LANCZOS) + + resized.append(im) + + # Prepare canvases + canvases = [ + [resized[0]] + ] + current_canvas = canvases[0] + last_image_width = resized[0].width + for i in range(1, len(resized)): + + if last_image_width != resized[i].width: + canvases.append([]) + current_canvas = canvases[-1] + + current_canvas.append(resized[i]) + last_image_width = resized[i].width + + # Stack + stacked_images = [] + for canvas in canvases: + width = canvas[0].width + height = sum(im.height for im in canvas) + + new_image = Image.new("RGB", (width, height), color=(0, 0, 0, 0)) + + cursor = 0 + for img in canvas: + new_image.paste(img, (0, cursor)) + cursor += img.height + + stacked_images.append(new_image) + + # Rechunk + chunks = [] + for im in stacked_images: + for im_chunk in rechunk(im): + chunks.append(im_chunk) + + # Convert + for i, chunk in enumerate(chunks): + + try: + with NamedTemporaryFile(delete=False, suffix=".png") as f: + chunk.save(f) + + webp_convert(f.name, os.path.join(cache_folder, f"{i:03d}.webp")) + + finally: + os.remove(f.name) + + with open(os.path.join(cache_folder, "ok"), "w"): + pass + + copy_from_cache(cache_folder, output_folder) + +def open_image(filename): + + with open(filename, "rb") as f: + data = f.read() + + io = BytesIO(data) + io.name = filename + return Image.open(io) + +def create_thumbnail_sheet(images, output_filename): + images = [open_image(image["cover"]) for image in images] + + width = len(images) * THUMBNAIL_WIDTH + height = THUMBNAIL_HEIGHT + + new_image = Image.new("RGBA", (width, height), color=(0, 0, 0, 0)) + + for i, im in enumerate(images): + im = convert_rgba(im) + + im = ImageOps.pad(im, (THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT), color=(0, 0, 0, 0), method=Image.Resampling.LANCZOS) + new_image.paste(im, (i * THUMBNAIL_WIDTH, 0)) + + new_image.save(output_filename, quality=THUMBNAIL_QUALITY) diff --git a/render.py b/render.py new file mode 100644 index 0000000..d2ecf2e --- /dev/null +++ b/render.py @@ -0,0 +1,209 @@ +import json +import os +import time +from glob import glob +from itertools import batched + +import jinja2 +from PIL import ImageFile + +from config import WILDCARD_LETTER, THUMBNAIL_CHUNK_SIZE, PROCS, GLOBALS +from imgutils import create_thumbnail_sheet, convert_cover, \ + convert_chapter + +ImageFile.LOAD_TRUNCATED_IMAGES = True +from minify_html import minify +from slugify import slugify +from tqdm import tqdm +from rcssmin import cssmin +from multiprocessing.pool import ThreadPool + +env = jinja2.Environment( + undefined=jinja2.StrictUndefined, +) + +env.filters["slug"] = slugify +env.filters["minify_css"] = cssmin + + +def render_index_pages(path, prefix, index_items, content_rating): + + with open("assets/index.html.jinja") as f: + template = env.from_string(f.read()) + + index_parent_dir = f"{prefix}{('-' + content_rating.lower()) if content_rating else ''}" + + def _work(task): + item, series = task + + index_loc = item.replace(WILDCARD_LETTER, "_") + index_dir = str(os.path.join(path, index_parent_dir, index_loc)) + os.makedirs(index_dir) + + if prefix == "t": + series = sorted(series, key=lambda s: s["score"], reverse=True) + else: + series = sorted(series, key=lambda s: s["sort_name"]) + + thumbnails = [ + { + "cover": glob(os.path.join(path, s["location"], "cover.*"))[0], + "name": s["name"], + "loc": s["location"] + } + for s in series + ] + + for i, chunk in enumerate(batched(thumbnails, THUMBNAIL_CHUNK_SIZE)): + sheet_name = f"{int(time.time()):x}{i:x}" + create_thumbnail_sheet(chunk, os.path.join(index_dir, sheet_name + ".webp")) + + sheet_img = os.path.join(index_parent_dir, index_loc, sheet_name + ".webp") + + for j, tn in enumerate(chunk): + offset_class = f"o{j:x}" + + tn["offset"] = offset_class + tn["img"] = sheet_img + + is_long_list = len(index_items) > 27 + + html = minify(template.render( + name=item, + thumbnails=thumbnails, + parent_loc=index_parent_dir, + loc=index_loc, + prefix=prefix, + index_pages=[ + ( + item.replace("-", " ") if is_long_list else item.upper(), + item, + os.path.join(index_parent_dir, item.replace(WILDCARD_LETTER, "_")), + len(series) > 0 + ) + for item, series in index_items.items() + ], + tags=is_long_list, + rating=content_rating, + **GLOBALS + )) + + with open(os.path.join(index_dir, "index.html"), "w") as f: + f.write(html) + + + with ThreadPool(processes=PROCS) as pool: + tasks = list(index_items.items()) + + for _ in tqdm(pool.imap_unordered(_work, tasks), total=len(tasks)): + pass + + +def render_series_pages(path, index): + with open("assets/series.html.jinja") as f: + template = env.from_string(f.read()) + + def _work(series): + + series_dir = str(os.path.join(path, series["location"])) + os.makedirs(series_dir) + + convert_cover(series["id"] + "_c", series["cover"], series_dir) + + html = minify(template.render( + series_name = series["name"], + loc = series["location"], + chapters = series["chapters"], + tags = list(filter(lambda t: t != "all", series["tags"])), + rating = series["rating"], + **GLOBALS + )) + + with open(os.path.join(series_dir, "index.html"), "w") as f: + #TODO: Debug + # f.write(f"<!-- rm -r data/{slugify(series['name'])} && echo '{series['_url']}' >> history.txt && echo '{series['_url']}' >> bl.txt -->") + f.write(html) + + with ThreadPool(processes=PROCS) as pool: + for _ in tqdm(pool.imap_unordered(_work, index), total=len(index)): + pass + + +def render_chapter_pages(path, index): + + with open("assets/chapter.html.jinja") as f: + template = env.from_string(f.read()) + + + def _work(task): + series, chapter = task + + chapter_dir = str(os.path.join(path, *chapter["location"].split("/"))) + + convert_chapter( + cache_key=f"{series['id']}.{chapter['index']:03d}", + images=chapter["images"], + output_folder=chapter_dir + ) + + images = [ + os.path.basename(filename) + for filename in + sorted(glob(os.path.join(chapter_dir, "*.webp"))) + ] + + html = minify(template.render( + series_name = series["name"], + series_loc = series["location"], + chapter_name = chapter["name"], + loc = chapter["location"], + prev_loc = chapter.get("prev"), + next_loc = chapter.get("next"), + images = images, + **GLOBALS + )) + + with open(os.path.join(chapter_dir, "index.html"), "w") as f: + f.write(html) + + tasks = [] + for series in index: + for chapter in series["chapters"]: + tasks.append((series, chapter)) + + with ThreadPool(processes=PROCS) as pool: + for _ in tqdm(pool.imap_unordered(_work, tasks), total=len(tasks)): + pass + + +def render_home(path): + with open("assets/home.html.jinja") as f: + template = env.from_string(f.read()) + + html = minify(template.render( + **GLOBALS + )) + + with open(os.path.join(path, "index.html"), "w") as f: + f.write(html) + + with open("assets/manifest.json.jinja") as f: + template = env.from_string(f.read()) + + data = json.loads(template.render( + **GLOBALS + )) + + with open(os.path.join(path, "manifest.json"), "w") as f: + json.dump(data, f, sort_keys=True) + +def render_style(path): + with open("assets/style.css.jinja") as f: + template = env.from_string(f.read()) + + css = cssmin(template.render( + **GLOBALS + )) + + with open(os.path.join(path, GLOBALS["style"]), "w") as f: + f.write(css) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ba2a629 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +Jinja2 +pillow +minify-html +python-slugify +tqdm +rcssmin
\ No newline at end of file |