Added cache permancance on disk

This commit is contained in:
DrMint 2024-06-30 01:38:49 +02:00
parent c9b6d11c9b
commit a9e4e91e8d
11 changed files with 270 additions and 114 deletions

3
.gitignore vendored
View File

@ -19,3 +19,6 @@ pnpm-debug.log*
# macOS-specific files
.DS_Store
# caching
.cache

View File

@ -26,7 +26,6 @@
- [Feat] Improve page load speed by using
- streaming https://docs.astro.build/en/recipes/streaming-improve-page-performance/
- https://developer.mozilla.org/en-US/docs/Web/API/Speculation_Rules_API
- [Feat] Persistant cache system
- [Feat] History replace instead of push when browsing scans and gallery
- [Feat] Use subgrid to align the generic previews
- [Bugs] [Timeline] Error if collectible not published?

View File

@ -1,12 +1,18 @@
import type { PayloadSDK } from "src/shared/payload/payload-sdk";
import { getLogger } from "src/utils/logger";
import { writeFile, mkdir, readFile } from "fs/promises";
import { existsSync } from "fs";
const ON_DISK_ROOT = `.cache/dataCache`;
const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`;
const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`;
export class DataCache {
private readonly logger = getLogger("[DataCache]");
private initialized = false;
private readonly responseCache = new Map<string, any>();
private readonly idsCacheMap = new Map<string, Set<string>>();
private responseCache = new Map<string, any>();
private invalidationMap = new Map<string, Set<string>>();
constructor(
private readonly payload: PayloadSDK,
@ -24,13 +30,37 @@ export class DataCache {
}
private async precache() {
const { data } = await this.payload.getAllSdkUrls();
for (const url of data.urls) {
try {
await this.payload.request(url);
} catch {
this.logger.warn("Precaching failed for url", url);
if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) {
this.logger.log("Loading cache from disk...");
// Handle RESPONSE_CACHE_FILE
{
const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE);
const data = JSON.parse(buffer.toString());
this.responseCache = new Map(data);
}
// Handle INVALIDATION_MAP_FILE
{
const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE);
const data = JSON.parse(buffer.toString()) as [string, string[]][];
const deserialize = data.map<[string, Set<string>]>(([key, value]) => [
key,
new Set(value),
]);
this.invalidationMap = new Map(deserialize);
}
} else {
const { data } = await this.payload.getAllSdkUrls();
for (const url of data.urls) {
try {
await this.payload.request(url);
} catch {
this.logger.warn("Precaching failed for url", url);
}
}
await this.save();
}
this.logger.log("Precaching completed!", this.responseCache.size, "responses cached");
}
@ -50,25 +80,28 @@ export class DataCache {
const uniqueIds = [...new Set(ids)];
uniqueIds.forEach((id) => {
const current = this.idsCacheMap.get(id);
const current = this.invalidationMap.get(id);
if (current) {
current.add(url);
} else {
this.idsCacheMap.set(id, new Set([url]));
this.invalidationMap.set(id, new Set([url]));
}
});
this.responseCache.set(url, response);
this.logger.log("Cached response for", url);
if (this.initialized) {
this.save();
}
}
async invalidate(ids: string[], urls: string[]) {
const urlsToInvalidate = new Set<string>(urls);
ids.forEach((id) => {
const urlsForThisId = this.idsCacheMap.get(id);
const urlsForThisId = this.invalidationMap.get(id);
if (!urlsForThisId) return;
this.idsCacheMap.delete(id);
this.invalidationMap.delete(id);
[...urlsForThisId].forEach((url) => urlsToInvalidate.add(url));
});
@ -84,5 +117,28 @@ export class DataCache {
this.onInvalidate([...urlsToInvalidate]);
this.logger.log("There are currently", this.responseCache.size, "responses in cache.");
if (this.initialized) {
this.save();
}
}
private async save() {
if (!existsSync(ON_DISK_ROOT)) {
await mkdir(ON_DISK_ROOT, { recursive: true });
}
const serializedResponseCache = JSON.stringify([...this.responseCache]);
await writeFile(ON_DISK_RESPONSE_CACHE_FILE, serializedResponseCache, {
encoding: "utf-8",
});
this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE);
const serializedIdsCache = JSON.stringify(
[...this.invalidationMap].map(([key, value]) => [key, [...value]])
);
await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, {
encoding: "utf-8",
});
this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE);
}
}

103
src/cache/pageCache.ts vendored
View File

@ -1,12 +1,23 @@
import type { PayloadSDK } from "src/shared/payload/payload-sdk";
import { getLogger } from "src/utils/logger";
import { writeFile, mkdir, readFile } from "fs/promises";
import { existsSync } from "fs";
import {
deserializeResponse,
serializeResponse,
type SerializableResponse,
} from "src/utils/responses";
const ON_DISK_ROOT = `.cache/pageCache`;
const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`;
const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`;
export class PageCache {
private readonly logger = getLogger("[PageCache]");
private initialized = false;
private readonly responseCache = new Map<string, Response>();
private readonly invalidationMap = new Map<string, Set<string>>();
private responseCache = new Map<string, Response>();
private invalidationMap = new Map<string, Set<string>>();
constructor(private readonly payload: PayloadSDK) {}
@ -21,26 +32,53 @@ export class PageCache {
}
private async precacheAll() {
const { data: languages } = await this.payload.getLanguages();
const locales = languages.map(({ id }) => id);
if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) {
this.logger.log("Loading cache from disk...");
// Handle RESPONSE_CACHE_FILE
{
const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE);
const data = JSON.parse(buffer.toString()) as [string, SerializableResponse][];
const deserializedData = data.map<[string, Response]>(([key, value]) => [
key,
deserializeResponse(value),
]);
this.responseCache = new Map(deserializedData);
}
await this.precache("/", locales);
await this.precache("/settings", locales);
await this.precache("/timeline", locales);
// Handle INVALIDATION_MAP_FILE
{
const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE);
const data = JSON.parse(buffer.toString()) as [string, string[]][];
const deserialize = data.map<[string, Set<string>]>(([key, value]) => [
key,
new Set(value),
]);
this.invalidationMap = new Map(deserialize);
}
} else {
const { data: languages } = await this.payload.getLanguages();
const locales = languages.map(({ id }) => id);
const { data: folders } = await this.payload.getFolderSlugs();
for (const slug of folders) {
await this.precache(`/folders/${slug}`, locales);
}
await this.precache("/", locales);
await this.precache("/settings", locales);
await this.precache("/timeline", locales);
const { data: pages } = await this.payload.getPageSlugs();
for (const slug of pages) {
await this.precache(`/pages/${slug}`, locales);
}
const { data: folders } = await this.payload.getFolderSlugs();
for (const slug of folders) {
await this.precache(`/folders/${slug}`, locales);
}
const { data: collectibles } = await this.payload.getCollectibleSlugs();
for (const slug of collectibles) {
await this.precache(`/collectibles/${slug}`, locales);
const { data: pages } = await this.payload.getPageSlugs();
for (const slug of pages) {
await this.precache(`/pages/${slug}`, locales);
}
const { data: collectibles } = await this.payload.getCollectibleSlugs();
for (const slug of collectibles) {
await this.precache(`/collectibles/${slug}`, locales);
}
await this.save();
}
this.logger.log("Precaching completed!", this.responseCache.size, "responses cached");
@ -80,6 +118,9 @@ export class PageCache {
this.responseCache.set(url, response.clone());
this.logger.log("Cached response for", url);
if (this.initialized) {
this.save();
}
}
async invalidate(sdkUrls: string[]) {
@ -103,5 +144,31 @@ export class PageCache {
}
this.logger.log("There are currently", this.responseCache.size, "responses in cache.");
if (this.initialized) {
this.save();
}
}
private async save() {
if (!existsSync(ON_DISK_ROOT)) {
await mkdir(ON_DISK_ROOT, { recursive: true });
}
const serializedResponses = await Promise.all(
[...this.responseCache].map(async ([key, value]) => [key, await serializeResponse(value)])
);
const serializedResponseCache = JSON.stringify(serializedResponses);
await writeFile(ON_DISK_RESPONSE_CACHE_FILE, serializedResponseCache, {
encoding: "utf-8",
});
this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE);
const serializedIdsCache = JSON.stringify(
[...this.invalidationMap].map(([key, value]) => [key, [...value]])
);
await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, {
encoding: "utf-8",
});
this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE);
}
}

View File

@ -1,5 +1,5 @@
---
import UAParser from "ua-parser-js";
import { UAParser } from "ua-parser-js";
import { getI18n } from "src/i18n/i18n";
import type {
EndpointAudio,

View File

@ -69,6 +69,8 @@ export type WordingKey =
| "collectibles.bookFormat.binding.hardcover"
| "collectibles.bookFormat.binding.readingDirection.leftToRight"
| "collectibles.bookFormat.binding.readingDirection.rightToLeft"
| "collectibles.gallery"
| "collectibles.scans"
| "collectibles.imageCount"
| "header.topbar.settings.tooltip"
| "collectibles.contents"
@ -87,29 +89,29 @@ export type WordingKey =
| "pages.tableOfContent.break"
| "global.languageOverride.availableLanguages"
| "timeline.title"
| "timeline.description"
| "timeline.eras.cataclysm"
| "timeline.eras.drakengard3"
| "timeline.eras.drakengard"
| "timeline.eras.drakengard2"
| "timeline.eras.drakengard3"
| "timeline.eras.nier"
| "timeline.eras.nierAutomata"
| "timeline.jumpTo"
| "timeline.notes.content"
| "timeline.eras.cataclysm"
| "timeline.description"
| "timeline.notes.title"
| "timeline.notes.content"
| "timeline.priorCataclysmNote.title"
| "timeline.priorCataclysmNote.content"
| "timeline.jumpTo"
| "timeline.year.during"
| "timeline.eventFooter.sources"
| "timeline.eventFooter.languages"
| "timeline.eventFooter.sources"
| "timeline.eventFooter.note"
| "global.sources.typeLabel.url"
| "global.sources.typeLabel.page"
| "global.sources.typeLabel.collectible"
| "global.sources.typeLabel.collectible.range.custom"
| "global.sources.typeLabel.folder"
| "global.sources.typeLabel.collectible.range.page"
| "global.sources.typeLabel.collectible.range.timestamp"
| "global.sources.typeLabel.folder"
| "global.sources.typeLabel.page"
| "global.sources.typeLabel.url"
| "global.sources.typeLabel.collectible.range.custom"
| "global.openMediaPage"
| "global.downloadButton"
| "global.previewTypes.video"
@ -119,8 +121,6 @@ export type WordingKey =
| "global.previewTypes.collectible"
| "global.previewTypes.unknown"
| "collectibles.scans.title"
| "collectibles.gallery.title"
| "collectibles.gallery.subtitle"
| "collectibles.scans.subtitle"
| "collectibles.scans.shortIndex.flapFront"
| "collectibles.scans.shortIndex.front"
@ -134,9 +134,11 @@ export type WordingKey =
| "collectibles.scans.obi"
| "collectibles.scans.obiInside"
| "collectibles.scans.pages"
| "collectibles.gallery.title"
| "collectibles.gallery.subtitle"
| "global.sources.typeLabel.scans"
| "collectibles.scans.dustjacket.description"
| "collectibles.scans.obi.description"
| "global.sources.typeLabel.scans"
| "global.sources.typeLabel.gallery"
| "global.media.attributes.filename"
| "global.media.attributes.duration"
@ -150,4 +152,5 @@ export type WordingKey =
| "collectibles.nature.digital"
| "global.previewTypes.zip"
| "global.previewTypes.pdf"
| "files.thumbnail.noPreview"
| "collectibles.files";

View File

@ -9,6 +9,8 @@ export const addCommonHeadersMiddleware = defineMiddleware(async ({ url }, next)
response.headers.set("Content-Language", currentLocale);
}
// TODO: Remove when in production
response.headers.set("X-Robots-Tag", "none");
response.headers.set("Vary", "Cookie");
response.headers.set("Cache-Control", "max-age=3600, stale-while-revalidate=3600");

View File

@ -1,8 +1,6 @@
import { defineMiddleware } from "astro:middleware";
import { pageCache } from "src/utils/payload";
const blacklist = ["/en/api/hooks/collection-operation", "/en/api/on-startup"];
export const pageCachingMiddleware = defineMiddleware(async ({ url, request, locals }, next) => {
const pathname = url.pathname;
const cachedPage = pageCache.get(pathname);
@ -27,7 +25,7 @@ export const pageCachingMiddleware = defineMiddleware(async ({ url, request, loc
if (response.ok) {
response.headers.set("Last-Modified", new Date().toUTCString());
if (!blacklist.includes(pathname)) {
if (!pathname.includes("/api/")) {
pageCache.set(pathname, response, [...locals.sdkCalls]);
}
}

View File

@ -2,8 +2,8 @@ import type { APIRoute } from "astro";
import { contextCache, dataCache, pageCache } from "src/utils/payload";
export const GET: APIRoute = async () => {
await contextCache.init();
await dataCache.init();
await contextCache.init();
await pageCache.init();
return new Response(null, { status: 200, statusText: "Ok" });
};

View File

@ -1,7 +1,7 @@
{
"disclaimer": "Usage subject to terms: https://openexchangerates.org/terms",
"license": "https://openexchangerates.org/license",
"timestamp": 1719604800,
"timestamp": 1719694805,
"base": "USD",
"rates": {
"AED": 3.673,
@ -10,81 +10,81 @@
"AMD": 388.16,
"ANG": 1.802366,
"AOA": 853.629,
"ARS": 911.483,
"AUD": 1.49888,
"ARS": 910.493696,
"AUD": 1.496558,
"AWG": 1.8025,
"AZN": 1.7,
"BAM": 1.828243,
"BBD": 2,
"BDT": 117.567308,
"BGN": 1.824066,
"BHD": 0.376918,
"BGN": 1.824663,
"BHD": 0.37651,
"BIF": 2877.015756,
"BMD": 1,
"BND": 1.355347,
"BOB": 6.913636,
"BRL": 5.5903,
"BRL": 5.5936,
"BSD": 1,
"BTC": 0.00001666854,
"BTC": 0.000016406223,
"BTN": 83.51892,
"BWP": 13.588017,
"BYN": 3.274442,
"BZD": 2.016789,
"CAD": 1.368179,
"CAD": 1.36945,
"CDF": 2843.605253,
"CHF": 0.898334,
"CLF": 0.034326,
"CLP": 947.15,
"CNH": 7.3002,
"CNY": 7.2677,
"COP": 4182.257769,
"CHF": 0.898864,
"CLF": 0.034322,
"CLP": 947.05,
"CNH": 7.29859,
"CNY": 7.2673,
"COP": 4178.502002,
"CRC": 523.103268,
"CUC": 1,
"CUP": 25.75,
"CVE": 103.074514,
"CZK": 23.392,
"CZK": 23.367501,
"DJF": 177.827972,
"DKK": 6.962786,
"DKK": 6.9619,
"DOP": 59.096817,
"DZD": 134.738656,
"EGP": 48.0286,
"DZD": 134.720884,
"EGP": 47.977051,
"ERN": 15,
"ETB": 57.758394,
"EUR": 0.933607,
"FJD": 2.2387,
"FKP": 0.791061,
"GBP": 0.791061,
"EUR": 0.932821,
"FJD": 2.2382,
"FKP": 0.790576,
"GBP": 0.790576,
"GEL": 2.8,
"GGP": 0.791061,
"GGP": 0.790576,
"GHS": 15.25842,
"GIP": 0.791061,
"GIP": 0.790576,
"GMD": 67.775,
"GNF": 8612.182198,
"GTQ": 7.771251,
"GYD": 209.334678,
"HKD": 7.809383,
"HKD": 7.80915,
"HNL": 24.765136,
"HRK": 7.034895,
"HRK": 7.03298,
"HTG": 132.555762,
"HUF": 368.794829,
"IDR": 16351.422732,
"ILS": 3.76585,
"IMP": 0.791061,
"INR": 83.388488,
"HUF": 368.78,
"IDR": 16350.45,
"ILS": 3.76595,
"IMP": 0.790576,
"INR": 83.36765,
"IQD": 1310.765417,
"IRR": 42100,
"ISK": 138.83,
"JEP": 0.791061,
"ISK": 138.78,
"JEP": 0.790576,
"JMD": 156.080264,
"JOD": 0.7087,
"JPY": 160.8655,
"JPY": 160.90493072,
"KES": 129.25,
"KGS": 86.4454,
"KHR": 4110.671159,
"KMF": 459.849919,
"KPW": 900,
"KRW": 1379.946543,
"KWD": 0.306773,
"KRW": 1381.28,
"KWD": 0.30676,
"KYD": 0.833423,
"KZT": 466.81538,
"LAK": 22075,
@ -96,7 +96,7 @@
"MAD": 9.940081,
"MDL": 17.830168,
"MGA": 4477.581302,
"MKD": 57.45758,
"MKD": 57.395402,
"MMK": 2481.91,
"MNT": 3450,
"MOP": 8.044281,
@ -104,35 +104,35 @@
"MUR": 47.2,
"MVR": 15.405,
"MWK": 1734.567667,
"MXN": 18.292389,
"MXN": 18.3385,
"MYR": 4.7175,
"MZN": 63.899991,
"NAD": 18.36094,
"NGN": 1515.9,
"NGN": 1406,
"NIO": 36.81119,
"NOK": 10.675842,
"NOK": 10.681139,
"NPR": 133.327095,
"NZD": 1.641672,
"OMR": 0.384947,
"NZD": 1.642037,
"OMR": 0.384965,
"PAB": 1,
"PEN": 3.83492,
"PGK": 3.851055,
"PHP": 58.433004,
"PHP": 58.409994,
"PKR": 278.529263,
"PLN": 4.025131,
"PLN": 4.024893,
"PYG": 7540.873261,
"QAR": 3.647595,
"RON": 4.6472,
"RSD": 109.245,
"RUB": 85.744825,
"RON": 4.644,
"RSD": 109.291,
"RUB": 85.656885,
"RWF": 1306.491928,
"SAR": 3.751727,
"SAR": 3.751821,
"SBD": 8.43942,
"SCR": 13.867107,
"SCR": 13.852325,
"SDG": 601,
"SEK": 10.596578,
"SGD": 1.35596,
"SHP": 0.791061,
"SEK": 10.601,
"SGD": 1.3564,
"SHP": 0.790576,
"SLL": 20969.5,
"SOS": 571.49784,
"SRD": 30.8385,
@ -142,35 +142,35 @@
"SVC": 8.755235,
"SYP": 2512.53,
"SZL": 18.183346,
"THB": 36.719,
"THB": 36.696982,
"TJS": 10.656085,
"TMT": 3.51,
"TND": 3.1465,
"TOP": 2.363716,
"TRY": 32.656998,
"TRY": 32.7383,
"TTD": 6.798721,
"TWD": 32.5085,
"TWD": 32.5195,
"TZS": 2635,
"UAH": 40.51974,
"UGX": 3712.013854,
"USD": 1,
"UYU": 39.446434,
"UYU": 39.438456,
"UZS": 12586.719022,
"VES": 36.390223,
"VES": 36.402496,
"VND": 25455.011984,
"VUV": 118.722,
"WST": 2.8,
"XAF": 612.406173,
"XAG": 0.03435128,
"XAU": 0.00043009,
"XAF": 611.890574,
"XAG": 0.03431827,
"XAU": 0.00042979,
"XCD": 2.70255,
"XDR": 0.759476,
"XOF": 612.406173,
"XPD": 0.00104121,
"XPF": 111.408973,
"XPT": 0.00100687,
"XOF": 611.890574,
"XPD": 0.00103766,
"XPF": 111.315175,
"XPT": 0.00100724,
"YER": 250.399984,
"ZAR": 18.190651,
"ZAR": 18.201,
"ZMW": 25.739177,
"ZWL": 322
}

View File

@ -8,3 +8,31 @@ export const fetchOr404 = async <T>(promise: () => Promise<T>): Promise<T | Resp
});
}
};
export type SerializableResponse = {
headers: [string, string][];
status: number;
statusText: string;
body?: string;
};
new Response();
export const serializeResponse = async (response: Response): Promise<SerializableResponse> => {
const clonedResponse = response.clone();
return {
body: await clonedResponse.text(),
status: clonedResponse.status,
statusText: clonedResponse.statusText,
headers: [...clonedResponse.headers],
};
};
export const deserializeResponse = ({
body,
headers,
status,
statusText,
}: SerializableResponse): Response => {
return new Response(body, { headers, status, statusText });
};