Added cache permancance on disk

.gitignore

@@ -19,3 +19,6 @@ pnpm-debug.log*
 
 # macOS-specific files
 .DS_Store
+
+# caching
+.cache

TODO.md

@@ -26,7 +26,6 @@
 - [Feat] Improve page load speed by using
   - streaming https://docs.astro.build/en/recipes/streaming-improve-page-performance/
   - https://developer.mozilla.org/en-US/docs/Web/API/Speculation_Rules_API
-- [Feat] Persistant cache system
 - [Feat] History replace instead of push when browsing scans and gallery
 - [Feat] Use subgrid to align the generic previews
 - [Bugs] [Timeline] Error if collectible not published?

src/cache/dataCache.ts

@@ -1,12 +1,18 @@
 import type { PayloadSDK } from "src/shared/payload/payload-sdk";
 import { getLogger } from "src/utils/logger";
+import { writeFile, mkdir, readFile } from "fs/promises";
+import { existsSync } from "fs";
+
+const ON_DISK_ROOT = `.cache/dataCache`;
+const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`;
+const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`;
 
 export class DataCache {
   private readonly logger = getLogger("[DataCache]");
   private initialized = false;
 
-  private readonly responseCache = new Map<string, any>();
+  private responseCache = new Map<string, any>();
-  private readonly idsCacheMap = new Map<string, Set<string>>();
+  private invalidationMap = new Map<string, Set<string>>();
 
   constructor(
     private readonly payload: PayloadSDK,
@@ -24,13 +30,37 @@ export class DataCache {
   }
 
   private async precache() {
-    const { data } = await this.payload.getAllSdkUrls();
+    if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) {
-    for (const url of data.urls) {
+      this.logger.log("Loading cache from disk...");
-      try {
+      // Handle RESPONSE_CACHE_FILE
-        await this.payload.request(url);
+      {
-      } catch {
+        const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE);
-        this.logger.warn("Precaching failed for url", url);
+        const data = JSON.parse(buffer.toString());
+        this.responseCache = new Map(data);
       }
+
+      // Handle INVALIDATION_MAP_FILE
+      {
+        const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE);
+        const data = JSON.parse(buffer.toString()) as [string, string[]][];
+        const deserialize = data.map<[string, Set<string>]>(([key, value]) => [
+          key,
+          new Set(value),
+        ]);
+        this.invalidationMap = new Map(deserialize);
+      }
+    } else {
+      const { data } = await this.payload.getAllSdkUrls();
+
+      for (const url of data.urls) {
+        try {
+          await this.payload.request(url);
+        } catch {
+          this.logger.warn("Precaching failed for url", url);
+        }
+      }
+
+      await this.save();
     }
     this.logger.log("Precaching completed!", this.responseCache.size, "responses cached");
   }
@@ -50,25 +80,28 @@ export class DataCache {
     const uniqueIds = [...new Set(ids)];
 
     uniqueIds.forEach((id) => {
-      const current = this.idsCacheMap.get(id);
+      const current = this.invalidationMap.get(id);
       if (current) {
         current.add(url);
       } else {
-        this.idsCacheMap.set(id, new Set([url]));
+        this.invalidationMap.set(id, new Set([url]));
       }
     });
 
     this.responseCache.set(url, response);
     this.logger.log("Cached response for", url);
+    if (this.initialized) {
+      this.save();
+    }
   }
 
   async invalidate(ids: string[], urls: string[]) {
     const urlsToInvalidate = new Set<string>(urls);
 
     ids.forEach((id) => {
-      const urlsForThisId = this.idsCacheMap.get(id);
+      const urlsForThisId = this.invalidationMap.get(id);
       if (!urlsForThisId) return;
-      this.idsCacheMap.delete(id);
+      this.invalidationMap.delete(id);
       [...urlsForThisId].forEach((url) => urlsToInvalidate.add(url));
     });
 
@@ -84,5 +117,28 @@ export class DataCache {
 
     this.onInvalidate([...urlsToInvalidate]);
     this.logger.log("There are currently", this.responseCache.size, "responses in cache.");
+    if (this.initialized) {
+      this.save();
+    }
+  }
+
+  private async save() {
+    if (!existsSync(ON_DISK_ROOT)) {
+      await mkdir(ON_DISK_ROOT, { recursive: true });
+    }
+
+    const serializedResponseCache = JSON.stringify([...this.responseCache]);
+    await writeFile(ON_DISK_RESPONSE_CACHE_FILE, serializedResponseCache, {
+      encoding: "utf-8",
+    });
+    this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE);
+
+    const serializedIdsCache = JSON.stringify(
+      [...this.invalidationMap].map(([key, value]) => [key, [...value]])
+    );
+    await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, {
+      encoding: "utf-8",
+    });
+    this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE);
   }
 }

src/cache/pageCache.ts

@@ -1,12 +1,23 @@
 import type { PayloadSDK } from "src/shared/payload/payload-sdk";
 import { getLogger } from "src/utils/logger";
+import { writeFile, mkdir, readFile } from "fs/promises";
+import { existsSync } from "fs";
+import {
+  deserializeResponse,
+  serializeResponse,
+  type SerializableResponse,
+} from "src/utils/responses";
+
+const ON_DISK_ROOT = `.cache/pageCache`;
+const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`;
+const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`;
 
 export class PageCache {
   private readonly logger = getLogger("[PageCache]");
   private initialized = false;
 
-  private readonly responseCache = new Map<string, Response>();
+  private responseCache = new Map<string, Response>();
-  private readonly invalidationMap = new Map<string, Set<string>>();
+  private invalidationMap = new Map<string, Set<string>>();
 
   constructor(private readonly payload: PayloadSDK) {}
 
@@ -21,26 +32,53 @@ export class PageCache {
   }
 
   private async precacheAll() {
-    const { data: languages } = await this.payload.getLanguages();
+    if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) {
-    const locales = languages.map(({ id }) => id);
+      this.logger.log("Loading cache from disk...");
+      // Handle RESPONSE_CACHE_FILE
+      {
+        const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE);
+        const data = JSON.parse(buffer.toString()) as [string, SerializableResponse][];
+        const deserializedData = data.map<[string, Response]>(([key, value]) => [
+          key,
+          deserializeResponse(value),
+        ]);
+        this.responseCache = new Map(deserializedData);
+      }
 
-    await this.precache("/", locales);
+      // Handle INVALIDATION_MAP_FILE
-    await this.precache("/settings", locales);
+      {
-    await this.precache("/timeline", locales);
+        const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE);
+        const data = JSON.parse(buffer.toString()) as [string, string[]][];
+        const deserialize = data.map<[string, Set<string>]>(([key, value]) => [
+          key,
+          new Set(value),
+        ]);
+        this.invalidationMap = new Map(deserialize);
+      }
+    } else {
+      const { data: languages } = await this.payload.getLanguages();
+      const locales = languages.map(({ id }) => id);
 
-    const { data: folders } = await this.payload.getFolderSlugs();
+      await this.precache("/", locales);
-    for (const slug of folders) {
+      await this.precache("/settings", locales);
-      await this.precache(`/folders/${slug}`, locales);
+      await this.precache("/timeline", locales);
-    }
 
-    const { data: pages } = await this.payload.getPageSlugs();
+      const { data: folders } = await this.payload.getFolderSlugs();
-    for (const slug of pages) {
+      for (const slug of folders) {
-      await this.precache(`/pages/${slug}`, locales);
+        await this.precache(`/folders/${slug}`, locales);
-    }
+      }
 
-    const { data: collectibles } = await this.payload.getCollectibleSlugs();
+      const { data: pages } = await this.payload.getPageSlugs();
-    for (const slug of collectibles) {
+      for (const slug of pages) {
-      await this.precache(`/collectibles/${slug}`, locales);
+        await this.precache(`/pages/${slug}`, locales);
+      }
+
+      const { data: collectibles } = await this.payload.getCollectibleSlugs();
+      for (const slug of collectibles) {
+        await this.precache(`/collectibles/${slug}`, locales);
+      }
+
+      await this.save();
     }
 
     this.logger.log("Precaching completed!", this.responseCache.size, "responses cached");
@@ -80,6 +118,9 @@ export class PageCache {
 
     this.responseCache.set(url, response.clone());
     this.logger.log("Cached response for", url);
+    if (this.initialized) {
+      this.save();
+    }
   }
 
   async invalidate(sdkUrls: string[]) {
@@ -103,5 +144,31 @@ export class PageCache {
     }
 
     this.logger.log("There are currently", this.responseCache.size, "responses in cache.");
+    if (this.initialized) {
+      this.save();
+    }
+  }
+
+  private async save() {
+    if (!existsSync(ON_DISK_ROOT)) {
+      await mkdir(ON_DISK_ROOT, { recursive: true });
+    }
+
+    const serializedResponses = await Promise.all(
+      [...this.responseCache].map(async ([key, value]) => [key, await serializeResponse(value)])
+    );
+    const serializedResponseCache = JSON.stringify(serializedResponses);
+    await writeFile(ON_DISK_RESPONSE_CACHE_FILE, serializedResponseCache, {
+      encoding: "utf-8",
+    });
+    this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE);
+
+    const serializedIdsCache = JSON.stringify(
+      [...this.invalidationMap].map(([key, value]) => [key, [...value]])
+    );
+    await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, {
+      encoding: "utf-8",
+    });
+    this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE);
   }
 }

src/components/AppLayout/components/Html.astro

@@ -1,5 +1,5 @@
 ---
-import UAParser from "ua-parser-js";
+import { UAParser } from "ua-parser-js";
 import { getI18n } from "src/i18n/i18n";
 import type {
   EndpointAudio,

src/i18n/wordings-keys.ts

@@ -69,6 +69,8 @@ export type WordingKey =
   | "collectibles.bookFormat.binding.hardcover"
   | "collectibles.bookFormat.binding.readingDirection.leftToRight"
   | "collectibles.bookFormat.binding.readingDirection.rightToLeft"
+  | "collectibles.gallery"
+  | "collectibles.scans"
   | "collectibles.imageCount"
   | "header.topbar.settings.tooltip"
   | "collectibles.contents"
@@ -87,29 +89,29 @@ export type WordingKey =
   | "pages.tableOfContent.break"
   | "global.languageOverride.availableLanguages"
   | "timeline.title"
-  | "timeline.description"
+  | "timeline.eras.drakengard3"
-  | "timeline.eras.cataclysm"
   | "timeline.eras.drakengard"
   | "timeline.eras.drakengard2"
-  | "timeline.eras.drakengard3"
   | "timeline.eras.nier"
   | "timeline.eras.nierAutomata"
-  | "timeline.jumpTo"
+  | "timeline.eras.cataclysm"
-  | "timeline.notes.content"
+  | "timeline.description"
   | "timeline.notes.title"
+  | "timeline.notes.content"
   | "timeline.priorCataclysmNote.title"
   | "timeline.priorCataclysmNote.content"
+  | "timeline.jumpTo"
   | "timeline.year.during"
-  | "timeline.eventFooter.sources"
   | "timeline.eventFooter.languages"
+  | "timeline.eventFooter.sources"
   | "timeline.eventFooter.note"
+  | "global.sources.typeLabel.url"
+  | "global.sources.typeLabel.page"
   | "global.sources.typeLabel.collectible"
-  | "global.sources.typeLabel.collectible.range.custom"
+  | "global.sources.typeLabel.folder"
   | "global.sources.typeLabel.collectible.range.page"
   | "global.sources.typeLabel.collectible.range.timestamp"
-  | "global.sources.typeLabel.folder"
+  | "global.sources.typeLabel.collectible.range.custom"
-  | "global.sources.typeLabel.page"
-  | "global.sources.typeLabel.url"
   | "global.openMediaPage"
   | "global.downloadButton"
   | "global.previewTypes.video"
@@ -119,8 +121,6 @@ export type WordingKey =
   | "global.previewTypes.collectible"
   | "global.previewTypes.unknown"
   | "collectibles.scans.title"
-  | "collectibles.gallery.title"
-  | "collectibles.gallery.subtitle"
   | "collectibles.scans.subtitle"
   | "collectibles.scans.shortIndex.flapFront"
   | "collectibles.scans.shortIndex.front"
@@ -134,9 +134,11 @@ export type WordingKey =
   | "collectibles.scans.obi"
   | "collectibles.scans.obiInside"
   | "collectibles.scans.pages"
+  | "collectibles.gallery.title"
+  | "collectibles.gallery.subtitle"
+  | "global.sources.typeLabel.scans"
   | "collectibles.scans.dustjacket.description"
   | "collectibles.scans.obi.description"
-  | "global.sources.typeLabel.scans"
   | "global.sources.typeLabel.gallery"
   | "global.media.attributes.filename"
   | "global.media.attributes.duration"
@@ -150,4 +152,5 @@ export type WordingKey =
   | "collectibles.nature.digital"
   | "global.previewTypes.zip"
   | "global.previewTypes.pdf"
+  | "files.thumbnail.noPreview"
   | "collectibles.files";

src/middleware/commonHeaders.ts

@@ -9,6 +9,8 @@ export const addCommonHeadersMiddleware = defineMiddleware(async ({ url }, next)
     response.headers.set("Content-Language", currentLocale);
   }
 
+  // TODO: Remove when in production
+  response.headers.set("X-Robots-Tag", "none");
   response.headers.set("Vary", "Cookie");
   response.headers.set("Cache-Control", "max-age=3600, stale-while-revalidate=3600");
 

src/middleware/pageCaching.ts

@@ -1,8 +1,6 @@
 import { defineMiddleware } from "astro:middleware";
 import { pageCache } from "src/utils/payload";
 
-const blacklist = ["/en/api/hooks/collection-operation", "/en/api/on-startup"];
-
 export const pageCachingMiddleware = defineMiddleware(async ({ url, request, locals }, next) => {
   const pathname = url.pathname;
   const cachedPage = pageCache.get(pathname);
@@ -27,7 +25,7 @@ export const pageCachingMiddleware = defineMiddleware(async ({ url, request, loc
   if (response.ok) {
     response.headers.set("Last-Modified", new Date().toUTCString());
 
-    if (!blacklist.includes(pathname)) {
+    if (!pathname.includes("/api/")) {
       pageCache.set(pathname, response, [...locals.sdkCalls]);
     }
   }

src/pages/[locale]/api/on-startup.ts

@@ -2,8 +2,8 @@ import type { APIRoute } from "astro";
 import { contextCache, dataCache, pageCache } from "src/utils/payload";
 
 export const GET: APIRoute = async () => {
-  await contextCache.init();
   await dataCache.init();
+  await contextCache.init();
   await pageCache.init();
   return new Response(null, { status: 200, statusText: "Ok" });
 };

src/shared/openExchange/rates.json

@@ -1,7 +1,7 @@
 {
   "disclaimer": "Usage subject to terms: https://openexchangerates.org/terms",
   "license": "https://openexchangerates.org/license",
-  "timestamp": 1719604800,
+  "timestamp": 1719694805,
   "base": "USD",
   "rates": {
     "AED": 3.673,
@@ -10,81 +10,81 @@
     "AMD": 388.16,
     "ANG": 1.802366,
     "AOA": 853.629,
-    "ARS": 911.483,
+    "ARS": 910.493696,
-    "AUD": 1.49888,
+    "AUD": 1.496558,
     "AWG": 1.8025,
     "AZN": 1.7,
     "BAM": 1.828243,
     "BBD": 2,
     "BDT": 117.567308,
-    "BGN": 1.824066,
+    "BGN": 1.824663,
-    "BHD": 0.376918,
+    "BHD": 0.37651,
     "BIF": 2877.015756,
     "BMD": 1,
     "BND": 1.355347,
     "BOB": 6.913636,
-    "BRL": 5.5903,
+    "BRL": 5.5936,
     "BSD": 1,
-    "BTC": 0.00001666854,
+    "BTC": 0.000016406223,
     "BTN": 83.51892,
     "BWP": 13.588017,
     "BYN": 3.274442,
     "BZD": 2.016789,
-    "CAD": 1.368179,
+    "CAD": 1.36945,
     "CDF": 2843.605253,
-    "CHF": 0.898334,
+    "CHF": 0.898864,
-    "CLF": 0.034326,
+    "CLF": 0.034322,
-    "CLP": 947.15,
+    "CLP": 947.05,
-    "CNH": 7.3002,
+    "CNH": 7.29859,
-    "CNY": 7.2677,
+    "CNY": 7.2673,
-    "COP": 4182.257769,
+    "COP": 4178.502002,
     "CRC": 523.103268,
     "CUC": 1,
     "CUP": 25.75,
     "CVE": 103.074514,
-    "CZK": 23.392,
+    "CZK": 23.367501,
     "DJF": 177.827972,
-    "DKK": 6.962786,
+    "DKK": 6.9619,
     "DOP": 59.096817,
-    "DZD": 134.738656,
+    "DZD": 134.720884,
-    "EGP": 48.0286,
+    "EGP": 47.977051,
     "ERN": 15,
     "ETB": 57.758394,
-    "EUR": 0.933607,
+    "EUR": 0.932821,
-    "FJD": 2.2387,
+    "FJD": 2.2382,
-    "FKP": 0.791061,
+    "FKP": 0.790576,
-    "GBP": 0.791061,
+    "GBP": 0.790576,
     "GEL": 2.8,
-    "GGP": 0.791061,
+    "GGP": 0.790576,
     "GHS": 15.25842,
-    "GIP": 0.791061,
+    "GIP": 0.790576,
     "GMD": 67.775,
     "GNF": 8612.182198,
     "GTQ": 7.771251,
     "GYD": 209.334678,
-    "HKD": 7.809383,
+    "HKD": 7.80915,
     "HNL": 24.765136,
-    "HRK": 7.034895,
+    "HRK": 7.03298,
     "HTG": 132.555762,
-    "HUF": 368.794829,
+    "HUF": 368.78,
-    "IDR": 16351.422732,
+    "IDR": 16350.45,
-    "ILS": 3.76585,
+    "ILS": 3.76595,
-    "IMP": 0.791061,
+    "IMP": 0.790576,
-    "INR": 83.388488,
+    "INR": 83.36765,
     "IQD": 1310.765417,
     "IRR": 42100,
-    "ISK": 138.83,
+    "ISK": 138.78,
-    "JEP": 0.791061,
+    "JEP": 0.790576,
     "JMD": 156.080264,
     "JOD": 0.7087,
-    "JPY": 160.8655,
+    "JPY": 160.90493072,
     "KES": 129.25,
     "KGS": 86.4454,
     "KHR": 4110.671159,
     "KMF": 459.849919,
     "KPW": 900,
-    "KRW": 1379.946543,
+    "KRW": 1381.28,
-    "KWD": 0.306773,
+    "KWD": 0.30676,
     "KYD": 0.833423,
     "KZT": 466.81538,
     "LAK": 22075,
@@ -96,7 +96,7 @@
     "MAD": 9.940081,
     "MDL": 17.830168,
     "MGA": 4477.581302,
-    "MKD": 57.45758,
+    "MKD": 57.395402,
     "MMK": 2481.91,
     "MNT": 3450,
     "MOP": 8.044281,
@@ -104,35 +104,35 @@
     "MUR": 47.2,
     "MVR": 15.405,
     "MWK": 1734.567667,
-    "MXN": 18.292389,
+    "MXN": 18.3385,
     "MYR": 4.7175,
     "MZN": 63.899991,
     "NAD": 18.36094,
-    "NGN": 1515.9,
+    "NGN": 1406,
     "NIO": 36.81119,
-    "NOK": 10.675842,
+    "NOK": 10.681139,
     "NPR": 133.327095,
-    "NZD": 1.641672,
+    "NZD": 1.642037,
-    "OMR": 0.384947,
+    "OMR": 0.384965,
     "PAB": 1,
     "PEN": 3.83492,
     "PGK": 3.851055,
-    "PHP": 58.433004,
+    "PHP": 58.409994,
     "PKR": 278.529263,
-    "PLN": 4.025131,
+    "PLN": 4.024893,
     "PYG": 7540.873261,
     "QAR": 3.647595,
-    "RON": 4.6472,
+    "RON": 4.644,
-    "RSD": 109.245,
+    "RSD": 109.291,
-    "RUB": 85.744825,
+    "RUB": 85.656885,
     "RWF": 1306.491928,
-    "SAR": 3.751727,
+    "SAR": 3.751821,
     "SBD": 8.43942,
-    "SCR": 13.867107,
+    "SCR": 13.852325,
     "SDG": 601,
-    "SEK": 10.596578,
+    "SEK": 10.601,
-    "SGD": 1.35596,
+    "SGD": 1.3564,
-    "SHP": 0.791061,
+    "SHP": 0.790576,
     "SLL": 20969.5,
     "SOS": 571.49784,
     "SRD": 30.8385,
@@ -142,35 +142,35 @@
     "SVC": 8.755235,
     "SYP": 2512.53,
     "SZL": 18.183346,
-    "THB": 36.719,
+    "THB": 36.696982,
     "TJS": 10.656085,
     "TMT": 3.51,
     "TND": 3.1465,
     "TOP": 2.363716,
-    "TRY": 32.656998,
+    "TRY": 32.7383,
     "TTD": 6.798721,
-    "TWD": 32.5085,
+    "TWD": 32.5195,
     "TZS": 2635,
     "UAH": 40.51974,
     "UGX": 3712.013854,
     "USD": 1,
-    "UYU": 39.446434,
+    "UYU": 39.438456,
     "UZS": 12586.719022,
-    "VES": 36.390223,
+    "VES": 36.402496,
     "VND": 25455.011984,
     "VUV": 118.722,
     "WST": 2.8,
-    "XAF": 612.406173,
+    "XAF": 611.890574,
-    "XAG": 0.03435128,
+    "XAG": 0.03431827,
-    "XAU": 0.00043009,
+    "XAU": 0.00042979,
     "XCD": 2.70255,
     "XDR": 0.759476,
-    "XOF": 612.406173,
+    "XOF": 611.890574,
-    "XPD": 0.00104121,
+    "XPD": 0.00103766,
-    "XPF": 111.408973,
+    "XPF": 111.315175,
-    "XPT": 0.00100687,
+    "XPT": 0.00100724,
     "YER": 250.399984,
-    "ZAR": 18.190651,
+    "ZAR": 18.201,
     "ZMW": 25.739177,
     "ZWL": 322
   }

src/utils/responses.ts

@@ -8,3 +8,31 @@ export const fetchOr404 = async <T>(promise: () => Promise<T>): Promise<T | Resp
     });
   }
 };
+
+export type SerializableResponse = {
+  headers: [string, string][];
+  status: number;
+  statusText: string;
+  body?: string;
+};
+
+new Response();
+
+export const serializeResponse = async (response: Response): Promise<SerializableResponse> => {
+  const clonedResponse = response.clone();
+  return {
+    body: await clonedResponse.text(),
+    status: clonedResponse.status,
+    statusText: clonedResponse.statusText,
+    headers: [...clonedResponse.headers],
+  };
+};
+
+export const deserializeResponse = ({
+  body,
+  headers,
+  status,
+  statusText,
+}: SerializableResponse): Response => {
+  return new Response(body, { headers, status, statusText });
+};