From 7dd91f58475e70d7bb0a0c4b3a91ed921dcbfe1a Mon Sep 17 00:00:00 2001 From: DrMint <29893320+DrMint@users.noreply.github.com> Date: Fri, 26 Jul 2024 09:22:33 +0200 Subject: [PATCH] Use backlinks and rework the caching system --- src/cache/contextCache.ts | 31 +++- src/cache/dataCache.ts | 48 ++---- src/cache/pageCache.ts | 157 ++++++++++-------- src/components/AppLayout/AppLayout.astro | 13 +- .../AppLayout/components/Topbar/Topbar.astro | 10 +- .../Topbar/components/ParentPagesButton.astro | 18 +- .../Topbar/components/ReturnToButton.astro | 10 +- src/components/Previews/FolderPreview.astro | 9 +- .../{SourceRow.astro => RelationRow.astro} | 10 +- src/env.d.ts | 1 - src/i18n/i18n.ts | 83 ++++----- src/middleware/pageCaching.ts | 2 +- src/middleware/setAstroLocals.ts | 1 - .../api/hooks/collection-operation.ts | 40 +---- src/pages/[locale]/api/on-startup.ts | 5 +- src/pages/[locale]/api/pages/partial.astro | 1 - src/pages/[locale]/api/timeline/partial.astro | 1 - src/pages/[locale]/audios/[id].astro | 3 +- .../collectibles/[slug]/gallery/[index].astro | 5 +- .../collectibles/[slug]/gallery/index.astro | 5 +- .../[locale]/collectibles/[slug]/index.astro | 5 +- .../collectibles/[slug]/scans/[index].astro | 5 +- .../collectibles/[slug]/scans/index.astro | 5 +- src/pages/[locale]/files/[id].astro | 3 +- src/pages/[locale]/folders/[slug].astro | 5 +- src/pages/[locale]/images/[id].astro | 3 +- src/pages/[locale]/pages/[slug].astro | 5 +- src/pages/[locale]/recorders/[id].astro | 1 - .../_components/TimelineSourcesButton.astro | 8 +- src/pages/[locale]/timeline/index.astro | 1 - src/pages/[locale]/videos/[id].astro | 3 +- src/services.ts | 21 ++- src/shared | 2 +- 33 files changed, 240 insertions(+), 280 deletions(-) rename src/components/{SourceRow.astro => RelationRow.astro} (84%) diff --git a/src/cache/contextCache.ts b/src/cache/contextCache.ts index 241fc7f..46ecdfa 100644 --- a/src/cache/contextCache.ts +++ b/src/cache/contextCache.ts @@ -3,7 +3,8 @@ import type { EndpointWebsiteConfig, EndpointWording, } from "src/shared/payload/endpoint-types"; -import type { PayloadSDK } from "src/shared/payload/sdk"; +import { SDKEndpointNames, type PayloadSDK } from "src/shared/payload/sdk"; +import type { EndpointChange } from "src/shared/payload/webhooks"; import { getLogger } from "src/utils/logger"; export class ContextCache { @@ -35,24 +36,42 @@ export class ContextCache { await this.refreshWordings(); } - async refreshWordings() { + async invalidate(changes: EndpointChange[]) { + for (const change of changes) { + switch (change.type) { + case SDKEndpointNames.getWordings: + return await this.refreshWordings(); + + case SDKEndpointNames.getLanguages: + return await this.refreshLocales(); + + case SDKEndpointNames.getCurrencies: + return await this.refreshCurrencies(); + + case SDKEndpointNames.getWebsiteConfig: + return await this.refreshWebsiteConfig(); + } + } + } + + private async refreshWordings() { this.wordings = (await this.payload.getWordings()).data; this.logger.log("Wordings refreshed"); } - async refreshCurrencies() { + private async refreshCurrencies() { this.currencies = (await this.payload.getCurrencies()).data.map(({ id }) => id); this.logger.log("Currencies refreshed"); } - async refreshLocales() { + private async refreshLocales() { this.languages = (await this.payload.getLanguages()).data; this.locales = this.languages.filter(({ selectable }) => selectable).map(({ id }) => id); this.logger.log("Locales refreshed"); } - async refreshWebsiteConfig() { - this.config = (await this.payload.getConfig()).data; + private async refreshWebsiteConfig() { + this.config = (await this.payload.getWebsiteConfig()).data; this.logger.log("WebsiteConfig refreshed"); } } diff --git a/src/cache/dataCache.ts b/src/cache/dataCache.ts index ddc11fc..6ac56c3 100644 --- a/src/cache/dataCache.ts +++ b/src/cache/dataCache.ts @@ -2,6 +2,7 @@ import { getLogger } from "src/utils/logger"; import { writeFile, mkdir, readFile } from "fs/promises"; import { existsSync } from "fs"; import type { PayloadSDK } from "src/shared/payload/sdk"; +import type { EndpointChange } from "src/shared/payload/webhooks"; const ON_DISK_ROOT = `.cache/dataCache`; const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`; @@ -11,14 +12,12 @@ export class DataCache { private initialized = false; private readonly responseCache = new Map(); - private readonly invalidationMap = new Map>(); private scheduleSaveTimeout: NodeJS.Timeout | undefined; constructor( private readonly payload: PayloadSDK, - private readonly uncachedPayload: PayloadSDK, - private readonly onInvalidate: (urls: string[]) => Promise + private readonly uncachedPayload: PayloadSDK ) {} async init() { @@ -32,8 +31,8 @@ export class DataCache { } private async precache() { - // Get all keys from CMS - const allSDKUrls = (await this.uncachedPayload.getAllSdkUrls()).data.urls; + // Get all documents from CMS + const allDocs = (await this.uncachedPayload.getAll()).data; // Load cache from disk if available if (existsSync(ON_DISK_RESPONSE_CACHE_FILE)) { @@ -42,20 +41,20 @@ export class DataCache { const data = JSON.parse(buffer.toString()) as [string, any][]; for (const [key, value] of data) { // Do not include cache where the key is no longer in the CMS - if (!allSDKUrls.includes(key)) continue; + if (!allDocs.find(({ url }) => url === key)) continue; this.set(key, value); } } const cacheSizeBeforePrecaching = this.responseCache.size; - for (const url of allSDKUrls) { + for (const doc of allDocs) { // Do not precache response if already included in the loaded cache from disk - if (this.responseCache.has(url)) continue; + if (this.responseCache.has(doc.url)) continue; try { - await this.payload.request(url); + await this.payload.request(doc.url); } catch { - this.logger.warn("Precaching failed for url", url); + this.logger.warn("Precaching failed for url", doc.url); } } @@ -77,20 +76,6 @@ export class DataCache { set(url: string, response: any) { if (import.meta.env.DATA_CACHING !== "true") return; - const stringData = JSON.stringify(response); - const regex = /[a-f0-9]{24}/g; - const ids = [...stringData.matchAll(regex)].map((match) => match[0]); - const uniqueIds = [...new Set(ids)]; - - uniqueIds.forEach((id) => { - const current = this.invalidationMap.get(id); - if (current) { - current.add(url); - } else { - this.invalidationMap.set(id, new Set([url])); - } - }); - this.responseCache.set(url, response); this.logger.log("Cached response for", url); if (this.initialized) { @@ -98,18 +83,11 @@ export class DataCache { } } - async invalidate(ids: string[], urls: string[]) { + async invalidate(changes: EndpointChange[]) { if (import.meta.env.DATA_CACHING !== "true") return; - const urlsToInvalidate = new Set(urls); - ids.forEach((id) => { - const urlsForThisId = this.invalidationMap.get(id); - if (!urlsForThisId) return; - this.invalidationMap.delete(id); - [...urlsForThisId].forEach((url) => urlsToInvalidate.add(url)); - }); - - for (const url of urlsToInvalidate) { + const urls = changes.map(({ url }) => url); + for (const url of urls) { this.responseCache.delete(url); this.logger.log("Invalidated cache for", url); try { @@ -118,8 +96,6 @@ export class DataCache { this.logger.log("Revalidation fails for", url); } } - - this.onInvalidate([...urlsToInvalidate]); this.logger.log("There are currently", this.responseCache.size, "responses in cache."); if (this.initialized) { this.scheduleSave(); diff --git a/src/cache/pageCache.ts b/src/cache/pageCache.ts index 3031ef9..b47fdd1 100644 --- a/src/cache/pageCache.ts +++ b/src/cache/pageCache.ts @@ -6,22 +6,25 @@ import { serializeResponse, type SerializableResponse, } from "src/utils/responses"; -import type { PayloadSDK } from "src/shared/payload/sdk"; +import { SDKEndpointNames, type PayloadSDK } from "src/shared/payload/sdk"; +import type { EndpointChange } from "src/shared/payload/webhooks"; +import type { ContextCache } from "src/cache/contextCache"; const ON_DISK_ROOT = `.cache/pageCache`; const ON_DISK_RESPONSE_CACHE_FILE = `${ON_DISK_ROOT}/responseCache.json`; -const ON_DISK_INVALIDATION_MAP_FILE = `${ON_DISK_ROOT}/invalidationMap.json`; export class PageCache { private readonly logger = getLogger("[PageCache]"); private initialized = false; private responseCache = new Map(); - private invalidationMap = new Map>(); private scheduleSaveTimeout: NodeJS.Timeout | undefined; - constructor(private readonly uncachedPayload: PayloadSDK) {} + constructor( + private readonly uncachedPayload: PayloadSDK, + private readonly contextCache: ContextCache + ) {} async init() { if (this.initialized) return; @@ -35,59 +38,32 @@ export class PageCache { private async precache() { if (import.meta.env.DATA_CACHING !== "true") return; - const { data: languages } = await this.uncachedPayload.getLanguages(); - const locales = languages.filter(({ selectable }) => selectable).map(({ id }) => id); // Get all pages urls from CMS - const allIds = (await this.uncachedPayload.getAllIds()).data; - - const allPagesUrls = [ - "/", - ...allIds.audios.ids.map((id) => `/audios/${id}`), - ...allIds.collectibles.slugs.map((slug) => `/collectibles/${slug}`), - ...allIds.files.ids.map((id) => `/files/${id}`), - ...allIds.folders.slugs.map((slug) => `/folders/${slug}`), - ...allIds.images.ids.map((id) => `/images/${id}`), - ...allIds.pages.slugs.map((slug) => `/pages/${slug}`), - ...allIds.recorders.ids.map((id) => `/recorders/${id}`), - "/settings", - "/timeline", - ...allIds.videos.ids.map((id) => `/videos/${id}`), - ].flatMap((url) => locales.map((id) => `/${id}${url}`)); + const allDocs = (await this.uncachedPayload.getAll()).data; + const allPageUrls = allDocs.flatMap((doc) => this.getUrlFromEndpointChange(doc)); + // TODO: Add static pages likes "/" and "/settings" // Load cache from disk if available - if (existsSync(ON_DISK_RESPONSE_CACHE_FILE) && existsSync(ON_DISK_INVALIDATION_MAP_FILE)) { + if (existsSync(ON_DISK_RESPONSE_CACHE_FILE)) { this.logger.log("Loading cache from disk..."); - // Handle RESPONSE_CACHE_FILE - { - const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE); - const data = JSON.parse(buffer.toString()) as [string, SerializableResponse][]; - let deserializedData = data.map<[string, Response]>(([key, value]) => [ - key, - deserializeResponse(value), - ]); - // Do not include cache where the key is no longer in the CMS - deserializedData = deserializedData.filter(([key]) => allPagesUrls.includes(key)); + const buffer = await readFile(ON_DISK_RESPONSE_CACHE_FILE); + const data = JSON.parse(buffer.toString()) as [string, SerializableResponse][]; + let deserializedData = data.map<[string, Response]>(([key, value]) => [ + key, + deserializeResponse(value), + ]); - this.responseCache = new Map(deserializedData); - } + // Do not include cache where the key is no longer in the CMS + deserializedData = deserializedData.filter(([key]) => allPageUrls.includes(key)); - // Handle INVALIDATION_MAP_FILE - { - const buffer = await readFile(ON_DISK_INVALIDATION_MAP_FILE); - const data = JSON.parse(buffer.toString()) as [string, string[]][]; - const deserialize = data.map<[string, Set]>(([key, value]) => [ - key, - new Set(value), - ]); - this.invalidationMap = new Map(deserialize); - } + this.responseCache = new Map(deserializedData); } const cacheSizeBeforePrecaching = this.responseCache.size; - for (const url of allPagesUrls) { + for (const url of allPageUrls) { // Do not precache response if already included in the loaded cache from disk if (this.responseCache.has(url)) continue; try { @@ -114,17 +90,8 @@ export class PageCache { return; } - set(url: string, response: Response, sdkCalls: string[]) { + set(url: string, response: Response) { if (import.meta.env.PAGE_CACHING !== "true") return; - sdkCalls.forEach((id) => { - const current = this.invalidationMap.get(id); - if (current) { - current.add(url); - } else { - this.invalidationMap.set(id, new Set([url])); - } - }); - this.responseCache.set(url, response.clone()); this.logger.log("Cached response for", url); if (this.initialized) { @@ -132,16 +99,70 @@ export class PageCache { } } - async invalidate(sdkUrls: string[]) { - if (import.meta.env.PAGE_CACHING !== "true") return; - const pagesToInvalidate = new Set(); + private getUrlFromEndpointChange(change: EndpointChange): string[] { + const getUnlocalizedUrl = (): string[] => { + switch (change.type) { + case SDKEndpointNames.getFolder: + return [`/folders/${change.slug}`]; - sdkUrls.forEach((url) => { - const pagesForThisSDKUrl = this.invalidationMap.get(url); - if (!pagesForThisSDKUrl) return; - this.invalidationMap.delete(url); - [...pagesForThisSDKUrl].forEach((page) => pagesToInvalidate.add(page)); - }); + case SDKEndpointNames.getCollectible: + return [`/collectibles/${change.slug}`]; + + case SDKEndpointNames.getCollectibleGallery: + return [`/collectibles/${change.slug}/gallery`]; + + case SDKEndpointNames.getCollectibleGalleryImage: + return [`/collectibles/${change.slug}/gallery/${change.index}`]; + + case SDKEndpointNames.getCollectibleScans: + return [`/collectibles/${change.slug}/scans`]; + + case SDKEndpointNames.getCollectibleScanPage: + return [`/collectibles/${change.slug}/scans/${change.index}`]; + + case SDKEndpointNames.getPage: + return [`/pages/${change.slug}`]; + + case SDKEndpointNames.getAudioByID: + return [`/audios/${change.id}`]; + + case SDKEndpointNames.getImageByID: + return [`/images/${change.id}`]; + + case SDKEndpointNames.getVideoByID: + return [`/videos/${change.id}`]; + + case SDKEndpointNames.getFileByID: + return [`/files/${change.id}`]; + + case SDKEndpointNames.getRecorderByID: + return [`/recorders/${change.id}`]; + + case SDKEndpointNames.getChronologyEvents: + case SDKEndpointNames.getChronologyEventByID: + return [`/timeline`]; + + case SDKEndpointNames.getWebsiteConfig: + case SDKEndpointNames.getLanguages: + case SDKEndpointNames.getCurrencies: + case SDKEndpointNames.getWordings: + return [...this.responseCache.keys()]; + + default: + return []; + } + }; + + return getUnlocalizedUrl().flatMap((url) => + this.contextCache.locales.map((id) => `/${id}${url}`) + ); + } + + async invalidate(changes: EndpointChange[]) { + if (import.meta.env.PAGE_CACHING !== "true") return; + const pagesToInvalidate = new Set( + changes.flatMap((change) => this.getUrlFromEndpointChange(change)) + ); for (const url of pagesToInvalidate) { this.responseCache.delete(url); @@ -181,13 +202,5 @@ export class PageCache { encoding: "utf-8", }); this.logger.log("Saved", ON_DISK_RESPONSE_CACHE_FILE); - - const serializedIdsCache = JSON.stringify( - [...this.invalidationMap].map(([key, value]) => [key, [...value]]) - ); - await writeFile(ON_DISK_INVALIDATION_MAP_FILE, serializedIdsCache, { - encoding: "utf-8", - }); - this.logger.log("Saved", ON_DISK_INVALIDATION_MAP_FILE); } } diff --git a/src/components/AppLayout/AppLayout.astro b/src/components/AppLayout/AppLayout.astro index 462202b..3b2da8f 100644 --- a/src/components/AppLayout/AppLayout.astro +++ b/src/components/AppLayout/AppLayout.astro @@ -4,12 +4,11 @@ import Topbar from "./components/Topbar/Topbar.astro"; import Footer from "./components/Footer.astro"; import AppLayoutBackgroundImg from "./components/AppLayoutBackgroundImg.astro"; import type { ComponentProps } from "astro/types"; -import type { EndpointSource } from "src/shared/payload/endpoint-types"; -import { getSDKEndpoint } from "src/shared/payload/sdk"; +import type { EndpointRelation } from "src/shared/payload/endpoint-types"; interface Props { openGraph?: ComponentProps["openGraph"]; - parentPages?: EndpointSource[]; + backlinks?: EndpointRelation[]; backgroundImage?: ComponentProps["img"] | undefined; hideFooterLinks?: boolean; hideHomeButton?: boolean; @@ -17,13 +16,9 @@ interface Props { class?: string | undefined; } -Astro.locals.sdkCalls.add(getSDKEndpoint.getCurrenciesEndpoint()); -Astro.locals.sdkCalls.add(getSDKEndpoint.getLanguagesEndpoint()); -Astro.locals.sdkCalls.add(getSDKEndpoint.getWordingsEndpoint()); - const { openGraph, - parentPages, + backlinks, backgroundImage, hideFooterLinks = false, hideHomeButton = false, @@ -38,7 +33,7 @@ const { {backgroundImage && }
diff --git a/src/components/AppLayout/components/Topbar/Topbar.astro b/src/components/AppLayout/components/Topbar/Topbar.astro index 6bf3c5f..27aeebb 100644 --- a/src/components/AppLayout/components/Topbar/Topbar.astro +++ b/src/components/AppLayout/components/Topbar/Topbar.astro @@ -6,15 +6,15 @@ import LanguageSelector from "./components/LanguageSelector.astro"; import CurrencySelector from "./components/CurrencySelector.astro"; import ParentPagesButton from "./components/ParentPagesButton.astro"; import { getI18n } from "src/i18n/i18n"; -import type { EndpointSource } from "src/shared/payload/endpoint-types"; +import type { EndpointRelation } from "src/shared/payload/endpoint-types"; interface Props { - parentPages?: EndpointSource[] | undefined; + backlinks?: EndpointRelation[] | undefined; hideHomeButton?: boolean; hideSearchButton?: boolean; } -const { parentPages = [], hideHomeButton = false, hideSearchButton = false } = Astro.props; +const { backlinks = [], hideHomeButton = false, hideSearchButton = false } = Astro.props; const { t, getLocalizedUrl } = await getI18n(Astro.locals.currentLocale); --- @@ -23,14 +23,14 @@ const { t, getLocalizedUrl } = await getI18n(Astro.locals.currentLocale);