Add disk cache

.gitignore

@@ -1,3 +1,5 @@
+cache
+
 node_modules
 /frontend/dist
 

Dockerfile

@@ -11,7 +11,8 @@ FROM python:3-alpine as requirements
 
 WORKDIR /usr/src/app/
 
-RUN pip install --no-cache-dir pipenv
+RUN apk add --no-cache git \
+    && pip install --no-cache-dir pipenv
 
 COPY Pipfile .
 COPY Pipfile.lock .
@@ -26,12 +27,15 @@ COPY --from=requirements /usr/src/app/requirements.txt .
 
 RUN apk add --no-cache --virtual .deps \
         build-base \
+        git \
+        libffi-dev \
         libjpeg-turbo-dev \
         libwebp-dev \
         zlib-dev \
     && pip install -r requirements.txt \
     && apk del .deps \
     && apk add --no-cache \
+        libffi \
         libjpeg-turbo \
         libwebp
 
@@ -39,6 +43,8 @@ COPY --from=frontend /usr/src/app/frontend/dist/ frontend/dist
 
 COPY [^frontend]* ./
 
+VOLUME ["/usr/src/app/cache/"]
+
 ENTRYPOINT ["gunicorn", "mangareader:app", "--bind", "0.0.0.0:8000", "--chdir", "/library"]
 
 EXPOSE 8000

Pipfile

@@ -11,6 +11,7 @@ pillow = "*"
 flask-cors = "*"
 gunicorn = "*"
 flask-caching = "*"
+webp = {editable = true,git = "https://github.com/sbruder/pywebp"}
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "92afeeaf4d754a31e04e294f6974bd9616eb0f388ce3d977ae235f3d49fa2eae"
+            "sha256": "5a623242219f0ec36e67df9c0caaadf71b39cfd21f5ed070f2766f9df8d3fd4d"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -16,6 +16,39 @@
         ]
     },
     "default": {
+        "cffi": {
+            "hashes": [
+                "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
+                "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
+                "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
+                "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
+                "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
+                "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
+                "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
+                "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
+                "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
+                "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
+                "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
+                "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
+                "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
+                "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
+                "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
+                "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
+                "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
+                "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
+                "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
+                "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
+                "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
+                "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
+                "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
+                "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
+                "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
+                "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
+                "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
+                "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
+            ],
+            "version": "==1.12.3"
+        },
         "click": {
             "hashes": [
                 "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
@@ -102,6 +135,27 @@
             ],
             "version": "==1.1.1"
         },
+        "numpy": {
+            "hashes": [
+                "sha256:03e311b0a4c9f5755da7d52161280c6a78406c7be5c5cc7facfbcebb641efb7e",
+                "sha256:0cdd229a53d2720d21175012ab0599665f8c9588b3b8ffa6095dd7b90f0691dd",
+                "sha256:312bb18e95218bedc3563f26fcc9c1c6bfaaf9d453d15942c0839acdd7e4c473",
+                "sha256:464b1c48baf49e8505b1bb754c47a013d2c305c5b14269b5c85ea0625b6a988a",
+                "sha256:5adfde7bd3ee4864536e230bcab1c673f866736698724d5d28c11a4d63672658",
+                "sha256:7724e9e31ee72389d522b88c0d4201f24edc34277999701ccd4a5392e7d8af61",
+                "sha256:8d36f7c53ae741e23f54793ffefb2912340b800476eb0a831c6eb602e204c5c4",
+                "sha256:910d2272403c2ea8a52d9159827dc9f7c27fb4b263749dca884e2e4a8af3b302",
+                "sha256:951fefe2fb73f84c620bec4e001e80a80ddaa1b84dce244ded7f1e0cbe0ed34a",
+                "sha256:9588c6b4157f493edeb9378788dcd02cb9e6a6aeaa518b511a1c79d06cbd8094",
+                "sha256:9ce8300950f2f1d29d0e49c28ebfff0d2f1e2a7444830fbb0b913c7c08f31511",
+                "sha256:be39cca66cc6806652da97103605c7b65ee4442c638f04ff064a7efd9a81d50a",
+                "sha256:c3ab2d835b95ccb59d11dfcd56eb0480daea57cdf95d686d22eff35584bc4554",
+                "sha256:eb0fc4a492cb896346c9e2c7a22eae3e766d407df3eb20f4ce027f23f76e4c54",
+                "sha256:ec0c56eae6cee6299f41e780a0280318a93db519bbb2906103c43f3e2be1206c",
+                "sha256:f4e4612de60a4f1c4d06c8c2857cdcb2b8b5289189a12053f37d3f41f06c60d0"
+            ],
+            "version": "==1.17.0"
+        },
         "pillow": {
             "hashes": [
                 "sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de",
@@ -134,6 +188,12 @@
             "index": "pypi",
             "version": "==6.1.0"
         },
+        "pycparser": {
+            "hashes": [
+                "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
+            ],
+            "version": "==2.19"
+        },
         "six": {
             "hashes": [
                 "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@@ -141,6 +201,11 @@
             ],
             "version": "==1.12.0"
         },
+        "webp": {
+            "editable": true,
+            "git": "https://github.com/sbruder/pywebp",
+            "ref": "39e39c0e1071985b7944651cc13ec11834751ce3"
+        },
         "werkzeug": {
             "hashes": [
                 "sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4",

backend.py

@@ -1,14 +1,20 @@
 from PIL import Image
+from disk_cache import DiskCache
 from io import BytesIO
 from mimetypes import types_map as mimetypes
 from zipfile import ZipFile
 from zlib import crc32
 import os.path
 import sqlite3
+import webp
 import werkzeug.exceptions as exceptions
 
 mimetypes['.webp'] = 'image/webp'
 
+if os.environ.get('DISK_CACHE', '1') == '0':
+    disk_cache = DiskCache(enabled=False)
+else:
+    disk_cache = DiskCache()
 thumbnail_cache = {}
 
 
@@ -46,9 +52,25 @@ def generate_thumbnail(filepath):
 
 
 class CalibreDB:
-    def __init__(self, path='metadata.db'):
+    def __init__(self, path='metadata.db', enable_webp=True, webp_quality=80, webp_size=2048):
         self.database_path = f'file:{path}?mode=ro'
 
+        self.webp = enable_webp
+        if self.webp is True:
+            if webp_quality == 101:
+                lossless = True
+                webp_quality = 100
+            else:
+                lossless = False
+
+            self.webp_config = webp.WebPConfig.new(
+                preset=webp.WebPPreset.DRAWING,
+                quality=webp_quality,
+                lossless=lossless
+            )
+
+            self.webp_size = webp_size
+
     def create_cursor(self):
         conn = sqlite3.connect(self.database_path, uri=True)
         conn.row_factory = dict_factory
@@ -209,13 +231,30 @@ class CalibreDB:
                 return None
 
             page_filename = zip_info.filename
+            mimetype = mimetypes[os.path.splitext(page_filename)[1]]
 
-            page_data = BytesIO()
+            if self.webp is True and mimetype != 'image/webp':
-            page_data.write(volume.read(page_filename))
+                mimetype = 'image/webp'
-            page_data.seek(0)
+
+                try:
+                    page_data = disk_cache.get(f'{volume_id}-{page_number}')
+                except FileNotFoundError:
+                    with volume.open(page_filename) as orig_page_data:
+                        image = Image.open(orig_page_data)
+                        image.thumbnail((self.webp_size, self.webp_size))
+                        image = image.convert('RGB')
+                        image = webp.WebPPicture.from_pil(image)
+                        page_data = BytesIO(image.encode(self.webp_config).buffer())
+
+                        disk_cache.set(f'{volume_id}-{page_number}', page_data)
+                        page_data.seek(0)
+            else:
+                page_data = BytesIO()
+                page_data.write(volume.read(page_filename))
+                page_data.seek(0)
 
             return {
                 'data': page_data,
-                'mimetype': mimetypes[os.path.splitext(page_filename)[1]],
+                'mimetype': mimetype,
                 'etag': str(zip_info.CRC)
             }

disk_cache.py

@@ -0,0 +1,32 @@
+from io import BytesIO
+import os
+
+class DiskCache:
+    def __init__(self, path=None, enabled=True):
+        if path is None:
+            self.path = os.path.join(os.path.dirname(__file__), 'cache')
+        else:
+            self.path = path
+
+        self.enabled = enabled
+        if self.enabled is False:
+            return
+
+        os.makedirs(self.path, exist_ok=True)
+
+    def set(self, identifier, buf):
+        if self.enabled is False:
+            return
+
+        with open(self._build_path(identifier), 'wb') as f:
+            f.write(buf.read())
+
+    def get(self, identifier):
+        if self.enabled is False:
+            raise FileNotFoundError
+
+        with open(self._build_path(identifier), 'rb') as f:
+            return BytesIO(f.read())
+
+    def _build_path(self, identifier):
+        return os.path.join(self.path, identifier)