diff --git a/.gitignore b/.gitignore index 6d4c0aa..c403ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ pnpm-debug.log* # macOS-specific files .DS_Store + +# caching +.cache \ No newline at end of file diff --git a/TODO.md b/TODO.md index 4af85a0..e4fe17a 100644 --- a/TODO.md +++ b/TODO.md @@ -26,7 +26,6 @@ - [Feat] Improve page load speed by using - streaming https://docs.astro.build/en/recipes/streaming-improve-page-performance/ - https://developer.mozilla.org/en-US/docs/Web/API/Speculation_Rules_API -- [Feat] Persistant cache system - [Feat] History replace instead of push when browsing scans and gallery - [Feat] Use subgrid to align the generic previews - [Bugs] [Timeline] Error if collectible not published? diff --git a/src/cache/dataCache.ts b/src/cache/dataCache.ts index 354c7b6..a7e4f82 100644 --- a/src/cache/dataCache.ts +++ b/src/cache/dataCache.ts @@ -1,12 +1,18 @@ import type { PayloadSDK } from "src/shared/payload/payload-sdk"; import { getLogger } from "src/utils/logger"; +import { writeFile, mkdir, readFile } from "fs/promises"; +import { existsSync } from "fs"; + +const ON_DISK_ROOT = `.cache/dataCache`; +const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`; +const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`; export class DataCache { private readonly logger = getLogger("[DataCache]"); private initialized = false; - private readonly responseCache = new Map(); - private readonly idsCacheMap = new Map>(); + private responseCache = new Map(); + private invalidationMap = new Map>(); constructor( private readonly payload: PayloadSDK, @@ -24,13 +30,37 @@ export class DataCache { } private async precache() { - const { data } = await this.payload.getAllSdkUrls(); - for (const url of data.urls) { - try { - await this.payload.request(url); - } catch { - this.logger.warn("Precaching failed for url", url); + if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) { + this.logger.log("Loading cache from disk..."); + // Handle RESPONSE_CACHE_FILE + { + const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE); + const data = JSON.parse(buffer.toString()); + this.responseCache = new Map(data); } + + // Handle INVALIDATION_MAP_FILE + { + const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE); + const data = JSON.parse(buffer.toString()) as [string, string[]][]; + const deserialize = data.map<[string, Set]>(([key, value]) => [ + key, + new Set(value), + ]); + this.invalidationMap = new Map(deserialize); + } + } else { + const { data } = await this.payload.getAllSdkUrls(); + + for (const url of data.urls) { + try { + await this.payload.request(url); + } catch { + this.logger.warn("Precaching failed for url", url); + } + } + + await this.save(); } this.logger.log("Precaching completed!", this.responseCache.size, "responses cached"); } @@ -50,25 +80,28 @@ export class DataCache { const uniqueIds = [...new Set(ids)]; uniqueIds.forEach((id) => { - const current = this.idsCacheMap.get(id); + const current = this.invalidationMap.get(id); if (current) { current.add(url); } else { - this.idsCacheMap.set(id, new Set([url])); + this.invalidationMap.set(id, new Set([url])); } }); this.responseCache.set(url, response); this.logger.log("Cached response for", url); + if (this.initialized) { + this.save(); + } } async invalidate(ids: string[], urls: string[]) { const urlsToInvalidate = new Set(urls); ids.forEach((id) => { - const urlsForThisId = this.idsCacheMap.get(id); + const urlsForThisId = this.invalidationMap.get(id); if (!urlsForThisId) return; - this.idsCacheMap.delete(id); + this.invalidationMap.delete(id); [...urlsForThisId].forEach((url) => urlsToInvalidate.add(url)); }); @@ -84,5 +117,28 @@ export class DataCache { this.onInvalidate([...urlsToInvalidate]); this.logger.log("There are currently", this.responseCache.size, "responses in cache."); + if (this.initialized) { + this.save(); + } + } + + private async save() { + if (!existsSync(ON_DISK_ROOT)) { + await mkdir(ON_DISK_ROOT, { recursive: true }); + } + + const serializedResponseCache = JSON.stringify([...this.responseCache]); + await writeFile(ON_DISK_RESPONSE_CACHE_FILE, serializedResponseCache, { + encoding: "utf-8", + }); + this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE); + + const serializedIdsCache = JSON.stringify( + [...this.invalidationMap].map(([key, value]) => [key, [...value]]) + ); + await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, { + encoding: "utf-8", + }); + this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE); } } diff --git a/src/cache/pageCache.ts b/src/cache/pageCache.ts index 3ee9da1..1ce70eb 100644 --- a/src/cache/pageCache.ts +++ b/src/cache/pageCache.ts @@ -1,12 +1,23 @@ import type { PayloadSDK } from "src/shared/payload/payload-sdk"; import { getLogger } from "src/utils/logger"; +import { writeFile, mkdir, readFile } from "fs/promises"; +import { existsSync } from "fs"; +import { + deserializeResponse, + serializeResponse, + type SerializableResponse, +} from "src/utils/responses"; + +const ON_DISK_ROOT = `.cache/pageCache`; +const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`; +const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`; export class PageCache { private readonly logger = getLogger("[PageCache]"); private initialized = false; - private readonly responseCache = new Map(); - private readonly invalidationMap = new Map>(); + private responseCache = new Map(); + private invalidationMap = new Map>(); constructor(private readonly payload: PayloadSDK) {} @@ -21,26 +32,53 @@ export class PageCache { } private async precacheAll() { - const { data: languages } = await this.payload.getLanguages(); - const locales = languages.map(({ id }) => id); + if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) { + this.logger.log("Loading cache from disk..."); + // Handle RESPONSE_CACHE_FILE + { + const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE); + const data = JSON.parse(buffer.toString()) as [string, SerializableResponse][]; + const deserializedData = data.map<[string, Response]>(([key, value]) => [ + key, + deserializeResponse(value), + ]); + this.responseCache = new Map(deserializedData); + } - await this.precache("/", locales); - await this.precache("/settings", locales); - await this.precache("/timeline", locales); + // Handle INVALIDATION_MAP_FILE + { + const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE); + const data = JSON.parse(buffer.toString()) as [string, string[]][]; + const deserialize = data.map<[string, Set]>(([key, value]) => [ + key, + new Set(value), + ]); + this.invalidationMap = new Map(deserialize); + } + } else { + const { data: languages } = await this.payload.getLanguages(); + const locales = languages.map(({ id }) => id); - const { data: folders } = await this.payload.getFolderSlugs(); - for (const slug of folders) { - await this.precache(`/folders/${slug}`, locales); - } + await this.precache("/", locales); + await this.precache("/settings", locales); + await this.precache("/timeline", locales); - const { data: pages } = await this.payload.getPageSlugs(); - for (const slug of pages) { - await this.precache(`/pages/${slug}`, locales); - } + const { data: folders } = await this.payload.getFolderSlugs(); + for (const slug of folders) { + await this.precache(`/folders/${slug}`, locales); + } - const { data: collectibles } = await this.payload.getCollectibleSlugs(); - for (const slug of collectibles) { - await this.precache(`/collectibles/${slug}`, locales); + const { data: pages } = await this.payload.getPageSlugs(); + for (const slug of pages) { + await this.precache(`/pages/${slug}`, locales); + } + + const { data: collectibles } = await this.payload.getCollectibleSlugs(); + for (const slug of collectibles) { + await this.precache(`/collectibles/${slug}`, locales); + } + + await this.save(); } this.logger.log("Precaching completed!", this.responseCache.size, "responses cached"); @@ -80,6 +118,9 @@ export class PageCache { this.responseCache.set(url, response.clone()); this.logger.log("Cached response for", url); + if (this.initialized) { + this.save(); + } } async invalidate(sdkUrls: string[]) { @@ -103,5 +144,31 @@ export class PageCache { } this.logger.log("There are currently", this.responseCache.size, "responses in cache."); + if (this.initialized) { + this.save(); + } + } + + private async save() { + if (!existsSync(ON_DISK_ROOT)) { + await mkdir(ON_DISK_ROOT, { recursive: true }); + } + + const serializedResponses = await Promise.all( + [...this.responseCache].map(async ([key, value]) => [key, await serializeResponse(value)]) + ); + const serializedResponseCache = JSON.stringify(serializedResponses); + await writeFile(ON_DISK_RESPONSE_CACHE_FILE, serializedResponseCache, { + encoding: "utf-8", + }); + this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE); + + const serializedIdsCache = JSON.stringify( + [...this.invalidationMap].map(([key, value]) => [key, [...value]]) + ); + await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, { + encoding: "utf-8", + }); + this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE); } } diff --git a/src/components/AppLayout/components/Html.astro b/src/components/AppLayout/components/Html.astro index 0a4d93c..f00a676 100644 --- a/src/components/AppLayout/components/Html.astro +++ b/src/components/AppLayout/components/Html.astro @@ -1,5 +1,5 @@ --- -import UAParser from "ua-parser-js"; +import { UAParser } from "ua-parser-js"; import { getI18n } from "src/i18n/i18n"; import type { EndpointAudio, diff --git a/src/i18n/wordings-keys.ts b/src/i18n/wordings-keys.ts index 24e82d1..babc282 100644 --- a/src/i18n/wordings-keys.ts +++ b/src/i18n/wordings-keys.ts @@ -69,6 +69,8 @@ export type WordingKey = | "collectibles.bookFormat.binding.hardcover" | "collectibles.bookFormat.binding.readingDirection.leftToRight" | "collectibles.bookFormat.binding.readingDirection.rightToLeft" + | "collectibles.gallery" + | "collectibles.scans" | "collectibles.imageCount" | "header.topbar.settings.tooltip" | "collectibles.contents" @@ -87,29 +89,29 @@ export type WordingKey = | "pages.tableOfContent.break" | "global.languageOverride.availableLanguages" | "timeline.title" - | "timeline.description" - | "timeline.eras.cataclysm" + | "timeline.eras.drakengard3" | "timeline.eras.drakengard" | "timeline.eras.drakengard2" - | "timeline.eras.drakengard3" | "timeline.eras.nier" | "timeline.eras.nierAutomata" - | "timeline.jumpTo" - | "timeline.notes.content" + | "timeline.eras.cataclysm" + | "timeline.description" | "timeline.notes.title" + | "timeline.notes.content" | "timeline.priorCataclysmNote.title" | "timeline.priorCataclysmNote.content" + | "timeline.jumpTo" | "timeline.year.during" - | "timeline.eventFooter.sources" | "timeline.eventFooter.languages" + | "timeline.eventFooter.sources" | "timeline.eventFooter.note" + | "global.sources.typeLabel.url" + | "global.sources.typeLabel.page" | "global.sources.typeLabel.collectible" - | "global.sources.typeLabel.collectible.range.custom" + | "global.sources.typeLabel.folder" | "global.sources.typeLabel.collectible.range.page" | "global.sources.typeLabel.collectible.range.timestamp" - | "global.sources.typeLabel.folder" - | "global.sources.typeLabel.page" - | "global.sources.typeLabel.url" + | "global.sources.typeLabel.collectible.range.custom" | "global.openMediaPage" | "global.downloadButton" | "global.previewTypes.video" @@ -119,8 +121,6 @@ export type WordingKey = | "global.previewTypes.collectible" | "global.previewTypes.unknown" | "collectibles.scans.title" - | "collectibles.gallery.title" - | "collectibles.gallery.subtitle" | "collectibles.scans.subtitle" | "collectibles.scans.shortIndex.flapFront" | "collectibles.scans.shortIndex.front" @@ -134,9 +134,11 @@ export type WordingKey = | "collectibles.scans.obi" | "collectibles.scans.obiInside" | "collectibles.scans.pages" + | "collectibles.gallery.title" + | "collectibles.gallery.subtitle" + | "global.sources.typeLabel.scans" | "collectibles.scans.dustjacket.description" | "collectibles.scans.obi.description" - | "global.sources.typeLabel.scans" | "global.sources.typeLabel.gallery" | "global.media.attributes.filename" | "global.media.attributes.duration" @@ -150,4 +152,5 @@ export type WordingKey = | "collectibles.nature.digital" | "global.previewTypes.zip" | "global.previewTypes.pdf" + | "files.thumbnail.noPreview" | "collectibles.files"; diff --git a/src/middleware/commonHeaders.ts b/src/middleware/commonHeaders.ts index ec00e83..eaf70a8 100644 --- a/src/middleware/commonHeaders.ts +++ b/src/middleware/commonHeaders.ts @@ -9,6 +9,8 @@ export const addCommonHeadersMiddleware = defineMiddleware(async ({ url }, next) response.headers.set("Content-Language", currentLocale); } + // TODO: Remove when in production + response.headers.set("X-Robots-Tag", "none"); response.headers.set("Vary", "Cookie"); response.headers.set("Cache-Control", "max-age=3600, stale-while-revalidate=3600"); diff --git a/src/middleware/pageCaching.ts b/src/middleware/pageCaching.ts index 78f7e66..f9fc589 100644 --- a/src/middleware/pageCaching.ts +++ b/src/middleware/pageCaching.ts @@ -1,8 +1,6 @@ import { defineMiddleware } from "astro:middleware"; import { pageCache } from "src/utils/payload"; -const blacklist = ["/en/api/hooks/collection-operation", "/en/api/on-startup"]; - export const pageCachingMiddleware = defineMiddleware(async ({ url, request, locals }, next) => { const pathname = url.pathname; const cachedPage = pageCache.get(pathname); @@ -27,7 +25,7 @@ export const pageCachingMiddleware = defineMiddleware(async ({ url, request, loc if (response.ok) { response.headers.set("Last-Modified", new Date().toUTCString()); - if (!blacklist.includes(pathname)) { + if (!pathname.includes("/api/")) { pageCache.set(pathname, response, [...locals.sdkCalls]); } } diff --git a/src/pages/[locale]/api/on-startup.ts b/src/pages/[locale]/api/on-startup.ts index f73c756..ead2ad9 100644 --- a/src/pages/[locale]/api/on-startup.ts +++ b/src/pages/[locale]/api/on-startup.ts @@ -2,8 +2,8 @@ import type { APIRoute } from "astro"; import { contextCache, dataCache, pageCache } from "src/utils/payload"; export const GET: APIRoute = async () => { - await contextCache.init(); await dataCache.init(); + await contextCache.init(); await pageCache.init(); return new Response(null, { status: 200, statusText: "Ok" }); }; diff --git a/src/shared/openExchange/rates.json b/src/shared/openExchange/rates.json index 8a6199d..064b5d5 100644 --- a/src/shared/openExchange/rates.json +++ b/src/shared/openExchange/rates.json @@ -1,7 +1,7 @@ { "disclaimer": "Usage subject to terms: https://openexchangerates.org/terms", "license": "https://openexchangerates.org/license", - "timestamp": 1719604800, + "timestamp": 1719694805, "base": "USD", "rates": { "AED": 3.673, @@ -10,81 +10,81 @@ "AMD": 388.16, "ANG": 1.802366, "AOA": 853.629, - "ARS": 911.483, - "AUD": 1.49888, + "ARS": 910.493696, + "AUD": 1.496558, "AWG": 1.8025, "AZN": 1.7, "BAM": 1.828243, "BBD": 2, "BDT": 117.567308, - "BGN": 1.824066, - "BHD": 0.376918, + "BGN": 1.824663, + "BHD": 0.37651, "BIF": 2877.015756, "BMD": 1, "BND": 1.355347, "BOB": 6.913636, - "BRL": 5.5903, + "BRL": 5.5936, "BSD": 1, - "BTC": 0.00001666854, + "BTC": 0.000016406223, "BTN": 83.51892, "BWP": 13.588017, "BYN": 3.274442, "BZD": 2.016789, - "CAD": 1.368179, + "CAD": 1.36945, "CDF": 2843.605253, - "CHF": 0.898334, - "CLF": 0.034326, - "CLP": 947.15, - "CNH": 7.3002, - "CNY": 7.2677, - "COP": 4182.257769, + "CHF": 0.898864, + "CLF": 0.034322, + "CLP": 947.05, + "CNH": 7.29859, + "CNY": 7.2673, + "COP": 4178.502002, "CRC": 523.103268, "CUC": 1, "CUP": 25.75, "CVE": 103.074514, - "CZK": 23.392, + "CZK": 23.367501, "DJF": 177.827972, - "DKK": 6.962786, + "DKK": 6.9619, "DOP": 59.096817, - "DZD": 134.738656, - "EGP": 48.0286, + "DZD": 134.720884, + "EGP": 47.977051, "ERN": 15, "ETB": 57.758394, - "EUR": 0.933607, - "FJD": 2.2387, - "FKP": 0.791061, - "GBP": 0.791061, + "EUR": 0.932821, + "FJD": 2.2382, + "FKP": 0.790576, + "GBP": 0.790576, "GEL": 2.8, - "GGP": 0.791061, + "GGP": 0.790576, "GHS": 15.25842, - "GIP": 0.791061, + "GIP": 0.790576, "GMD": 67.775, "GNF": 8612.182198, "GTQ": 7.771251, "GYD": 209.334678, - "HKD": 7.809383, + "HKD": 7.80915, "HNL": 24.765136, - "HRK": 7.034895, + "HRK": 7.03298, "HTG": 132.555762, - "HUF": 368.794829, - "IDR": 16351.422732, - "ILS": 3.76585, - "IMP": 0.791061, - "INR": 83.388488, + "HUF": 368.78, + "IDR": 16350.45, + "ILS": 3.76595, + "IMP": 0.790576, + "INR": 83.36765, "IQD": 1310.765417, "IRR": 42100, - "ISK": 138.83, - "JEP": 0.791061, + "ISK": 138.78, + "JEP": 0.790576, "JMD": 156.080264, "JOD": 0.7087, - "JPY": 160.8655, + "JPY": 160.90493072, "KES": 129.25, "KGS": 86.4454, "KHR": 4110.671159, "KMF": 459.849919, "KPW": 900, - "KRW": 1379.946543, - "KWD": 0.306773, + "KRW": 1381.28, + "KWD": 0.30676, "KYD": 0.833423, "KZT": 466.81538, "LAK": 22075, @@ -96,7 +96,7 @@ "MAD": 9.940081, "MDL": 17.830168, "MGA": 4477.581302, - "MKD": 57.45758, + "MKD": 57.395402, "MMK": 2481.91, "MNT": 3450, "MOP": 8.044281, @@ -104,35 +104,35 @@ "MUR": 47.2, "MVR": 15.405, "MWK": 1734.567667, - "MXN": 18.292389, + "MXN": 18.3385, "MYR": 4.7175, "MZN": 63.899991, "NAD": 18.36094, - "NGN": 1515.9, + "NGN": 1406, "NIO": 36.81119, - "NOK": 10.675842, + "NOK": 10.681139, "NPR": 133.327095, - "NZD": 1.641672, - "OMR": 0.384947, + "NZD": 1.642037, + "OMR": 0.384965, "PAB": 1, "PEN": 3.83492, "PGK": 3.851055, - "PHP": 58.433004, + "PHP": 58.409994, "PKR": 278.529263, - "PLN": 4.025131, + "PLN": 4.024893, "PYG": 7540.873261, "QAR": 3.647595, - "RON": 4.6472, - "RSD": 109.245, - "RUB": 85.744825, + "RON": 4.644, + "RSD": 109.291, + "RUB": 85.656885, "RWF": 1306.491928, - "SAR": 3.751727, + "SAR": 3.751821, "SBD": 8.43942, - "SCR": 13.867107, + "SCR": 13.852325, "SDG": 601, - "SEK": 10.596578, - "SGD": 1.35596, - "SHP": 0.791061, + "SEK": 10.601, + "SGD": 1.3564, + "SHP": 0.790576, "SLL": 20969.5, "SOS": 571.49784, "SRD": 30.8385, @@ -142,35 +142,35 @@ "SVC": 8.755235, "SYP": 2512.53, "SZL": 18.183346, - "THB": 36.719, + "THB": 36.696982, "TJS": 10.656085, "TMT": 3.51, "TND": 3.1465, "TOP": 2.363716, - "TRY": 32.656998, + "TRY": 32.7383, "TTD": 6.798721, - "TWD": 32.5085, + "TWD": 32.5195, "TZS": 2635, "UAH": 40.51974, "UGX": 3712.013854, "USD": 1, - "UYU": 39.446434, + "UYU": 39.438456, "UZS": 12586.719022, - "VES": 36.390223, + "VES": 36.402496, "VND": 25455.011984, "VUV": 118.722, "WST": 2.8, - "XAF": 612.406173, - "XAG": 0.03435128, - "XAU": 0.00043009, + "XAF": 611.890574, + "XAG": 0.03431827, + "XAU": 0.00042979, "XCD": 2.70255, "XDR": 0.759476, - "XOF": 612.406173, - "XPD": 0.00104121, - "XPF": 111.408973, - "XPT": 0.00100687, + "XOF": 611.890574, + "XPD": 0.00103766, + "XPF": 111.315175, + "XPT": 0.00100724, "YER": 250.399984, - "ZAR": 18.190651, + "ZAR": 18.201, "ZMW": 25.739177, "ZWL": 322 } diff --git a/src/utils/responses.ts b/src/utils/responses.ts index 6ae3e65..644b077 100644 --- a/src/utils/responses.ts +++ b/src/utils/responses.ts @@ -8,3 +8,31 @@ export const fetchOr404 = async (promise: () => Promise): Promise => { + const clonedResponse = response.clone(); + return { + body: await clonedResponse.text(), + status: clonedResponse.status, + statusText: clonedResponse.statusText, + headers: [...clonedResponse.headers], + }; +}; + +export const deserializeResponse = ({ + body, + headers, + status, + statusText, +}: SerializableResponse): Response => { + return new Response(body, { headers, status, statusText }); +};