Added precaching of all data

MIGRATION.md

@@ -83,6 +83,7 @@
 ## DOD3
 
 - Missing collectibles
+
   - https://accords-library.com/library/utahime-five
   - https://accords-library.com/library/dod3-official-score-book
   - Soundtrack
@@ -90,6 +91,7 @@
     - https://accords-library.com/library/dod3-original-soundtrack
 
 - Missing content
+
   - https://v3.accords-library.com/en/collectibles/dod-chips-music
   - https://v3.accords-library.com/en/collectibles/dod-history-films
   - https://accords-library.com/library/dod3-collector-edition
@@ -102,6 +104,7 @@
   - https://accords-library.com/contents/dod-history
 
 - Missing parent page
+
   - Novellas
     - https://accords-library.com/contents/folder/dod3-prelude-novellas
     - https://accords-library.com/contents/folder/dod3-dlc-short-stories
@@ -109,6 +112,7 @@
     - https://accords-library.com/contents/folder/dod3-story-side
 
 - Missing scans
+
   - https://v3.accords-library.com/en/collectibles/novel-prelude
   - Wrong rotation on pages for https://v3.accords-library.com/en/collectibles/visual-artbook/scans
   - https://v3.accords-library.com/en/collectibles/dod10-dlc-code

README.md

@@ -68,10 +68,11 @@ Accord's Library v3.0 (shorten to AL3.0) follows the Metamodernist Web model des
     - Lazy loaded
     - Space reservation to reduce Cumulative Layout Shift
     - Use of efficient formats (mostly WebP) and meaningful quality settings
-  - Server side rendered (both good and bad for speed)
+  - Server side rendered
     - Reduced data transfer
     - Reduced client-side complexity
     - Would require edge computing to reduce latency
+    - Data caching to speed up response time
   - Astro built-in View transitions and client-side navigation
   - Some data caching between the web server and CMS (to be improved)
 

TODO.md

@@ -36,8 +36,7 @@
 ## Long term
 
 - [Feat] Speed up loading with https://developer.mozilla.org/en-US/docs/Web/API/Speculation_Rules_API
-- [Feat] More data caching between the CMS and Astro
+- [Feat] Etag? Last-Modified? Cache-control?
-  - Etag? Last-Modified? Cache-control
 - [Feat] Explore posibilities for View Transitions
 - [Feat] Revemp theme system using light-dark https://caniuse.com/mdn-css_types_color_light-dark
 - [Feat] Add reduce motion to element that zoom when hovering

astro.config.ts

@@ -15,6 +15,16 @@ export default defineConfig({
     mode: "standalone",
   }),
   integrations: [
+    // We can't just call a function on startup because of some Vite's BS...
+    // So instead I'm exposing some endpoint that only does something the first time it's called.
+    {
+      name: "on-server-start",
+      hooks: {
+        "astro:server:start": async () => {
+          await fetch(`http://${ASTRO_HOST}:${ASTRO_PORT}/en/api/on-startup`);
+        },
+      },
+    },
     astroMetaTags(),
     icon({
       include: {
@@ -23,7 +33,7 @@ export default defineConfig({
     }),
   ],
   prefetch: false,
-  // devToolbar: { enabled: false },
+  devToolbar: { enabled: false },
   server: {
     port: parseInt(ASTRO_PORT ?? "4321"),
     host: ASTRO_HOST ?? true,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "v3.accords-library.com",
-  "version": "3.0.0-beta.3",
+  "version": "3.0.0-beta.4",
   "scripts": {
     "dev": "astro dev",
     "start": "astro dev",

public/robots.txt

@@ -1,2 +1,4 @@
 User-agent: *
 Disallow: /
+Disallow: /*/api/
+Disallow: /*/dev/

src/pages/[locale]/api/hooks/collection-operation.ts

@@ -1,5 +1,9 @@
 import type { APIRoute } from "astro";
-import { Collections, type WebHookMessage } from "src/shared/payload/payload-sdk";
+import {
+  Collections,
+  WebHookOperationType,
+  type WebHookMessage,
+} from "src/shared/payload/payload-sdk";
 import {
   invalidateDataCache,
   refreshCurrencies,
@@ -15,14 +19,14 @@ export const POST: APIRoute = async ({ request }) => {
     return new Response(null, { status: 403, statusText: "Forbidden" });
   }
 
-  const message = (await request.json()) as WebHookMessage;
+  const { collection, operation, id } = (await request.json()) as WebHookMessage;
-  console.log("[Webhook] Received message from CMS:", message);
+  console.log("[Webhook] Received message from CMS:", { collection, Collections, id });
 
-  if (message.id) {
+  if (id && operation !== WebHookOperationType.create) {
-    await invalidateDataCache(message.id);
+    await invalidateDataCache(id);
   }
 
-  switch (message.collection) {
+  switch (collection) {
     case Collections.Wordings:
       await refreshWordings();
       break;

src/pages/[locale]/api/on-startup.ts

@@ -0,0 +1,7 @@
+import type { APIRoute } from "astro";
+import { initPayload } from "src/utils/payload";
+
+export const GET: APIRoute = async () => {
+  await initPayload();
+  return new Response(null, { status: 200, statusText: "Ok" });
+};

src/shared/analytics/analytics.ts

@@ -59,6 +59,7 @@ type AnalyticsBody = Record<string, unknown> & {
 };
 
 const track = async (body: AnalyticsBody) => {
+  if (!import.meta.env.ANALYTICS_URL) return;
   try {
     await fetch(import.meta.env.ANALYTICS_URL, {
       method: "POST",

src/shared/payload/payload-sdk.ts

@@ -2016,6 +2016,16 @@ export type PayloadImage = PayloadMedia & {
   height: number;
 };
 
+export type EndpointAllPaths = {
+  collectibles: string[];
+  pages: string[];
+  folders: string[];
+  videos: string[];
+  audios: string[];
+  images: string[];
+  recorders: string[];
+};
+
 // SDK
 
 type GetPayloadSDKParams = {
@@ -2042,7 +2052,7 @@ export const getPayloadSDK = ({
   responseCache,
 }: GetPayloadSDKParams) => {
   const refreshToken = async () => {
-    const loginUrl = payloadApiUrl(Collections.Recorders, "login");
+    const loginUrl = `${apiURL}/${Collections.Recorders}/login`;
     const loginResult = await fetch(loginUrl, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
@@ -2062,9 +2072,6 @@ export const getPayloadSDK = ({
     return token;
   };
 
-  const payloadApiUrl = (collection: Collections, endpoint?: string, isGlobal?: boolean): string =>
-    `${apiURL}/${isGlobal === undefined ? "" : "globals/"}${collection}${endpoint === undefined ? "" : `/${endpoint}`}`;
-
   const request = async (url: string): Promise<any> => {
     const cachedResponse = responseCache?.get(url);
     if (cachedResponse) {
@@ -2089,45 +2096,46 @@ export const getPayloadSDK = ({
 
   return {
     getConfig: async (): Promise<EndpointWebsiteConfig> =>
-      await request(payloadApiUrl(Collections.WebsiteConfig, `config`, true)),
+      await request(`${apiURL}/globals/${Collections.WebsiteConfig}/config`),
     getFolder: async (slug: string): Promise<EndpointFolder> =>
-      await request(payloadApiUrl(Collections.Folders, `slug/${slug}`)),
+      await request(`${apiURL}/${Collections.Folders}/slug/${slug}`),
     getLanguages: async (): Promise<Language[]> =>
-      await request(payloadApiUrl(Collections.Languages, `all`)),
+      await request(`${apiURL}/${Collections.Languages}/all`),
     getCurrencies: async (): Promise<Currency[]> =>
-      await request(payloadApiUrl(Collections.Currencies, `all`)),
+      await request(`${apiURL}/${Collections.Currencies}/all`),
     getWordings: async (): Promise<EndpointWording[]> =>
-      await request(payloadApiUrl(Collections.Wordings, `all`)),
+      await request(`${apiURL}/${Collections.Wordings}/all`),
     getPage: async (slug: string): Promise<EndpointPage> =>
-      await request(payloadApiUrl(Collections.Pages, `slug/${slug}`)),
+      await request(`${apiURL}/${Collections.Pages}/slug/${slug}`),
     getCollectible: async (slug: string): Promise<EndpointCollectible> =>
-      await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}`)),
+      await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}`),
     getCollectibleScans: async (slug: string): Promise<EndpointCollectibleScans> =>
-      await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/scans`)),
+      await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/scans`),
     getCollectibleScanPage: async (
       slug: string,
       index: string
     ): Promise<EndpointCollectibleScanPage> =>
-      await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/scans/${index}`)),
+      await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/scans/${index}`),
     getCollectibleGallery: async (slug: string): Promise<EndpointCollectibleGallery> =>
-      await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/gallery`)),
+      await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/gallery`),
     getCollectibleGalleryImage: async (
       slug: string,
       index: string
     ): Promise<EndpointCollectibleGalleryImage> =>
-      await request(payloadApiUrl(Collections.Collectibles, `slug/${slug}/gallery/${index}`)),
+      await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/gallery/${index}`),
     getChronologyEvents: async (): Promise<EndpointChronologyEvent[]> =>
-      await request(payloadApiUrl(Collections.ChronologyEvents, `all`)),
+      await request(`${apiURL}/${Collections.ChronologyEvents}/all`),
     getChronologyEventByID: async (id: string): Promise<EndpointChronologyEvent> =>
-      await request(payloadApiUrl(Collections.ChronologyEvents, `id/${id}`)),
+      await request(`${apiURL}/${Collections.ChronologyEvents}/id/${id}`),
     getImageByID: async (id: string): Promise<EndpointImage> =>
-      await request(payloadApiUrl(Collections.Images, `id/${id}`)),
+      await request(`${apiURL}/${Collections.Images}/id/${id}`),
     getAudioByID: async (id: string): Promise<EndpointAudio> =>
-      await request(payloadApiUrl(Collections.Audios, `id/${id}`)),
+      await request(`${apiURL}/${Collections.Audios}/id/${id}`),
     getVideoByID: async (id: string): Promise<EndpointVideo> =>
-      await request(payloadApiUrl(Collections.Videos, `id/${id}`)),
+      await request(`${apiURL}/${Collections.Videos}/id/${id}`),
     getRecorderByID: async (id: string): Promise<EndpointRecorder> =>
-      await request(payloadApiUrl(Collections.Recorders, `id/${id}`)),
+      await request(`${apiURL}/${Collections.Recorders}/id/${id}`),
+    getAllPaths: async (): Promise<EndpointAllPaths> => await request(`${apiURL}/all-paths`),
     request: async (url: string): Promise<any> => await request(url),
   };
 };

src/utils/payload.ts

@@ -22,8 +22,6 @@ export const payload = getPayloadSDK({
         console.log("[PayloadSDK] No token to be retrieved or the token expired");
         return undefined;
       }
-      const diffInMinutes = Math.floor((expiration - Date.now()) / 1000 / 60);
-      console.log("[PayloadSDK] Retrieved token from cache. TTL is", diffInMinutes, "minutes.");
       return token;
     },
     set: (newToken, newExpiration) => {
@@ -36,13 +34,10 @@ export const payload = getPayloadSDK({
   responseCache: {
     get: (url) => {
       const cachedResponse = responseCache.get(url);
-      if (!cachedResponse) {
+      if (cachedResponse) {
-        console.log("[ResponseCaching] No cached response found for", url);
-        return undefined;
-      }
         console.log("[ResponseCaching] Retrieved cache response for", url);
-
         return cachedResponse;
+      }
     },
     set: (url, response) => {
       const stringData = JSON.stringify(response);
@@ -114,3 +109,51 @@ export const refreshLocales = async () => {
 export const refreshWebsiteConfig = async () => {
   cache.config = await payload.getConfig();
 };
+
+let payloadInitialized = false;
+export const initPayload = async () => {
+  if (!payloadInitialized) {
+    const result = await payload.getAllPaths();
+
+    for (const slug of result.pages) {
+      await payload.getPage(slug);
+    }
+
+    for (const slug of result.folders) {
+      await payload.getFolder(slug);
+    }
+
+    for (const slug of result.collectibles) {
+      const collectible = await payload.getCollectible(slug);
+      if (collectible.scans) {
+        await payload.getCollectibleScans(slug);
+      }
+      if (collectible.gallery) {
+        await payload.getCollectibleGallery(slug);
+      }
+    }
+
+    for (const id of result.recorders) {
+      await payload.getRecorderByID(id);
+    }
+
+    for (const id of result.recorders) {
+      await payload.getRecorderByID(id);
+    }
+
+    for (const id of result.audios) {
+      await payload.getAudioByID(id);
+    }
+
+    for (const id of result.videos) {
+      await payload.getVideoByID(id);
+    }
+
+    for (const id of result.images) {
+      await payload.getImageByID(id);
+    }
+
+    payloadInitialized = true;
+    console.log("[ResponseCaching] Precaching completed!", responseCache.size, "responses cached");
+  }
+};