From 9105b0403257b1434097620fdd82e962829c6a29 Mon Sep 17 00:00:00 2001 From: DrMint <29893320+DrMint@users.noreply.github.com> Date: Sat, 15 Jun 2024 23:05:51 +0200 Subject: [PATCH] Added precaching of all data --- MIGRATION.md | 6 +- README.md | 3 +- TODO.md | 3 +- astro.config.ts | 12 +++- package.json | 2 +- public/robots.txt | 4 +- .../api/hooks/collection-operation.ts | 16 +++-- src/pages/[locale]/api/on-startup.ts | 7 +++ src/shared/analytics/analytics.ts | 1 + src/shared/payload/payload-sdk.ts | 50 +++++++++------- src/utils/payload.ts | 59 ++++++++++++++++--- 11 files changed, 121 insertions(+), 42 deletions(-) create mode 100644 src/pages/[locale]/api/on-startup.ts diff --git a/MIGRATION.md b/MIGRATION.md index 01a450a..3676837 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -83,6 +83,7 @@ ## DOD3 - Missing collectibles + - https://accords-library.com/library/utahime-five - https://accords-library.com/library/dod3-official-score-book - Soundtrack @@ -90,6 +91,7 @@ - https://accords-library.com/library/dod3-original-soundtrack - Missing content + - https://v3.accords-library.com/en/collectibles/dod-chips-music - https://v3.accords-library.com/en/collectibles/dod-history-films - https://accords-library.com/library/dod3-collector-edition @@ -102,6 +104,7 @@ - https://accords-library.com/contents/dod-history - Missing parent page + - Novellas - https://accords-library.com/contents/folder/dod3-prelude-novellas - https://accords-library.com/contents/folder/dod3-dlc-short-stories @@ -109,8 +112,9 @@ - https://accords-library.com/contents/folder/dod3-story-side - Missing scans + - https://v3.accords-library.com/en/collectibles/novel-prelude - Wrong rotation on pages for https://v3.accords-library.com/en/collectibles/visual-artbook/scans - https://v3.accords-library.com/en/collectibles/dod10-dlc-code -- Check release data for https://v3.accords-library.com/en/collectibles/dod10th-anniversary-box and its subitems \ No newline at end of file +- Check release data for https://v3.accords-library.com/en/collectibles/dod10th-anniversary-box and its subitems diff --git a/README.md b/README.md index d56b95b..2bff185 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,11 @@ Accord's Library v3.0 (shorten to AL3.0) follows the Metamodernist Web model des - Lazy loaded - Space reservation to reduce Cumulative Layout Shift - Use of efficient formats (mostly WebP) and meaningful quality settings - - Server side rendered (both good and bad for speed) + - Server side rendered - Reduced data transfer - Reduced client-side complexity - Would require edge computing to reduce latency + - Data caching to speed up response time - Astro built-in View transitions and client-side navigation - Some data caching between the web server and CMS (to be improved) diff --git a/TODO.md b/TODO.md index a9ee7e6..0a35519 100644 --- a/TODO.md +++ b/TODO.md @@ -36,8 +36,7 @@ ## Long term - [Feat] Speed up loading with https://developer.mozilla.org/en-US/docs/Web/API/Speculation_Rules_API -- [Feat] More data caching between the CMS and Astro - - Etag? Last-Modified? Cache-control +- [Feat] Etag? Last-Modified? Cache-control? - [Feat] Explore posibilities for View Transitions - [Feat] Revemp theme system using light-dark https://caniuse.com/mdn-css_types_color_light-dark - [Feat] Add reduce motion to element that zoom when hovering diff --git a/astro.config.ts b/astro.config.ts index 198629c..8a7c4bf 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -15,6 +15,16 @@ export default defineConfig({ mode: "standalone", }), integrations: [ + // We can't just call a function on startup because of some Vite's BS... + // So instead I'm exposing some endpoint that only does something the first time it's called. + { + name: "on-server-start", + hooks: { + "astro:server:start": async () => { + await fetch(`http://${ASTRO_HOST}:${ASTRO_PORT}/en/api/on-startup`); + }, + }, + }, astroMetaTags(), icon({ include: { @@ -23,7 +33,7 @@ export default defineConfig({ }), ], prefetch: false, - // devToolbar: { enabled: false }, + devToolbar: { enabled: false }, server: { port: parseInt(ASTRO_PORT ?? "4321"), host: ASTRO_HOST ?? true, diff --git a/package.json b/package.json index a02ba03..2aa8949 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "v3.accords-library.com", - "version": "3.0.0-beta.3", + "version": "3.0.0-beta.4", "scripts": { "dev": "astro dev", "start": "astro dev", diff --git a/public/robots.txt b/public/robots.txt index 77470cb..8711d63 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,2 +1,4 @@ User-agent: * -Disallow: / \ No newline at end of file +Disallow: / +Disallow: /*/api/ +Disallow: /*/dev/ diff --git a/src/pages/[locale]/api/hooks/collection-operation.ts b/src/pages/[locale]/api/hooks/collection-operation.ts index 5b3287d..51ed17d 100644 --- a/src/pages/[locale]/api/hooks/collection-operation.ts +++ b/src/pages/[locale]/api/hooks/collection-operation.ts @@ -1,5 +1,9 @@ import type { APIRoute } from "astro"; -import { Collections, type WebHookMessage } from "src/shared/payload/payload-sdk"; +import { + Collections, + WebHookOperationType, + type WebHookMessage, +} from "src/shared/payload/payload-sdk"; import { invalidateDataCache, refreshCurrencies, @@ -15,14 +19,14 @@ export const POST: APIRoute = async ({ request }) => { return new Response(null, { status: 403, statusText: "Forbidden" }); } - const message = (await request.json()) as WebHookMessage; - console.log("[Webhook] Received message from CMS:", message); + const { collection, operation, id } = (await request.json()) as WebHookMessage; + console.log("[Webhook] Received message from CMS:", { collection, Collections, id }); - if (message.id) { - await invalidateDataCache(message.id); + if (id && operation !== WebHookOperationType.create) { + await invalidateDataCache(id); } - switch (message.collection) { + switch (collection) { case Collections.Wordings: await refreshWordings(); break; diff --git a/src/pages/[locale]/api/on-startup.ts b/src/pages/[locale]/api/on-startup.ts new file mode 100644 index 0000000..a120fc5 --- /dev/null +++ b/src/pages/[locale]/api/on-startup.ts @@ -0,0 +1,7 @@ +import type { APIRoute } from "astro"; +import { initPayload } from "src/utils/payload"; + +export const GET: APIRoute = async () => { + await initPayload(); + return new Response(null, { status: 200, statusText: "Ok" }); +}; diff --git a/src/shared/analytics/analytics.ts b/src/shared/analytics/analytics.ts index 47d85d8..bff9356 100644 --- a/src/shared/analytics/analytics.ts +++ b/src/shared/analytics/analytics.ts @@ -59,6 +59,7 @@ type AnalyticsBody = Record & { }; const track = async (body: AnalyticsBody) => { + if (!import.meta.env.ANALYTICS_URL) return; try { await fetch(import.meta.env.ANALYTICS_URL, { method: "POST", diff --git a/src/shared/payload/payload-sdk.ts b/src/shared/payload/payload-sdk.ts index 09651ec..4e57d25 100644 --- a/src/shared/payload/payload-sdk.ts +++ b/src/shared/payload/payload-sdk.ts @@ -2016,6 +2016,16 @@ export type PayloadImage = PayloadMedia & { height: number; }; +export type EndpointAllPaths = { + collectibles: string[]; + pages: string[]; + folders: string[]; + videos: string[]; + audios: string[]; + images: string[]; + recorders: string[]; +}; + // SDK type GetPayloadSDKParams = { @@ -2042,7 +2052,7 @@ export const getPayloadSDK = ({ responseCache, }: GetPayloadSDKParams) => { const refreshToken = async () => { - const loginUrl = payloadApiUrl(Collections.Recorders, "login"); + const loginUrl = `${apiURL}/${Collections.Recorders}/login`; const loginResult = await fetch(loginUrl, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -2062,9 +2072,6 @@ export const getPayloadSDK = ({ return token; }; - const payloadApiUrl = (collection: Collections, endpoint?: string, isGlobal?: boolean): string => - `${apiURL}/${isGlobal === undefined ? "" : "globals/"}${collection}${endpoint === undefined ? "" : `/${endpoint}`}`; - const request = async (url: string): Promise => { const cachedResponse = responseCache?.get(url); if (cachedResponse) { @@ -2089,45 +2096,46 @@ export const getPayloadSDK = ({ return { getConfig: async (): Promise => - await request(payloadApiUrl(Collections.WebsiteConfig, `config`, true)), + await request(`${apiURL}/globals/${Collections.WebsiteConfig}/config`), getFolder: async (slug: string): Promise => - await request(payloadApiUrl(Collections.Folders, `slug/${slug}`)), + await request(`${apiURL}/${Collections.Folders}/slug/${slug}`), getLanguages: async (): Promise => - await request(payloadApiUrl(Collections.Languages, `all`)), + await request(`${apiURL}/${Collections.Languages}/all`), getCurrencies: async (): Promise => - await request(payloadApiUrl(Collections.Currencies, `all`)), + await request(`${apiURL}/${Collections.Currencies}/all`), getWordings: async (): Promise => - await request(payloadApiUrl(Collections.Wordings, `all`)), + await request(`${apiURL}/${Collections.Wordings}/all`), getPage: async (slug: string): Promise => - await request(payloadApiUrl(Collections.Pages, `slug/${slug}`)), + await request(`${apiURL}/${Collections.Pages}/slug/${slug}`), getCollectible: async (slug: string): Promise => - await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}`)), + await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}`), getCollectibleScans: async (slug: string): Promise => - await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/scans`)), + await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/scans`), getCollectibleScanPage: async ( slug: string, index: string ): Promise => - await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/scans/${index}`)), + await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/scans/${index}`), getCollectibleGallery: async (slug: string): Promise => - await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/gallery`)), + await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/gallery`), getCollectibleGalleryImage: async ( slug: string, index: string ): Promise => - await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/gallery/${index}`)), + await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/gallery/${index}`), getChronologyEvents: async (): Promise => - await request(payloadApiUrl(Collections.ChronologyEvents, `all`)), + await request(`${apiURL}/${Collections.ChronologyEvents}/all`), getChronologyEventByID: async (id: string): Promise => - await request(payloadApiUrl(Collections.ChronologyEvents, `id/${id}`)), + await request(`${apiURL}/${Collections.ChronologyEvents}/id/${id}`), getImageByID: async (id: string): Promise => - await request(payloadApiUrl(Collections.Images, `id/${id}`)), + await request(`${apiURL}/${Collections.Images}/id/${id}`), getAudioByID: async (id: string): Promise => - await request(payloadApiUrl(Collections.Audios, `id/${id}`)), + await request(`${apiURL}/${Collections.Audios}/id/${id}`), getVideoByID: async (id: string): Promise => - await request(payloadApiUrl(Collections.Videos, `id/${id}`)), + await request(`${apiURL}/${Collections.Videos}/id/${id}`), getRecorderByID: async (id: string): Promise => - await request(payloadApiUrl(Collections.Recorders, `id/${id}`)), + await request(`${apiURL}/${Collections.Recorders}/id/${id}`), + getAllPaths: async (): Promise => await request(`${apiURL}/all-paths`), request: async (url: string): Promise => await request(url), }; }; diff --git a/src/utils/payload.ts b/src/utils/payload.ts index 8b8f06e..21bdc89 100644 --- a/src/utils/payload.ts +++ b/src/utils/payload.ts @@ -22,8 +22,6 @@ export const payload = getPayloadSDK({ console.log("[PayloadSDK] No token to be retrieved or the token expired"); return undefined; } - const diffInMinutes = Math.floor((expiration - Date.now()) / 1000 / 60); - console.log("[PayloadSDK] Retrieved token from cache. TTL is", diffInMinutes, "minutes."); return token; }, set: (newToken, newExpiration) => { @@ -36,13 +34,10 @@ export const payload = getPayloadSDK({ responseCache: { get: (url) => { const cachedResponse = responseCache.get(url); - if (!cachedResponse) { - console.log("[ResponseCaching] No cached response found for", url); - return undefined; + if (cachedResponse) { + console.log("[ResponseCaching] Retrieved cache response for", url); + return cachedResponse; } - console.log("[ResponseCaching] Retrieved cache response for", url); - - return cachedResponse; }, set: (url, response) => { const stringData = JSON.stringify(response); @@ -114,3 +109,51 @@ export const refreshLocales = async () => { export const refreshWebsiteConfig = async () => { cache.config = await payload.getConfig(); }; + +let payloadInitialized = false; +export const initPayload = async () => { + if (!payloadInitialized) { + const result = await payload.getAllPaths(); + + for (const slug of result.pages) { + await payload.getPage(slug); + } + + for (const slug of result.folders) { + await payload.getFolder(slug); + } + + for (const slug of result.collectibles) { + const collectible = await payload.getCollectible(slug); + if (collectible.scans) { + await payload.getCollectibleScans(slug); + } + if (collectible.gallery) { + await payload.getCollectibleGallery(slug); + } + } + + for (const id of result.recorders) { + await payload.getRecorderByID(id); + } + + for (const id of result.recorders) { + await payload.getRecorderByID(id); + } + + for (const id of result.audios) { + await payload.getAudioByID(id); + } + + for (const id of result.videos) { + await payload.getVideoByID(id); + } + + for (const id of result.images) { + await payload.getImageByID(id); + } + + payloadInitialized = true; + console.log("[ResponseCaching] Precaching completed!", responseCache.size, "responses cached"); + } +};