Improved caching system

This commit is contained in:
DrMint 2024-06-19 10:58:19 +02:00
parent fb49341b57
commit 1e0edd5c5c
11 changed files with 304 additions and 98 deletions

View File

@ -18,6 +18,8 @@ import { translatedFields } from "../../fields/translatedFields/translatedFields
import { beforeDuplicateAddCopyTo } from "../../hooks/beforeDuplicateAddCopyTo";
import { beforeDuplicatePiping } from "../../hooks/beforeDuplicatePiping";
import { beforeDuplicateUnpublish } from "../../hooks/beforeDuplicateUnpublish";
import { Collectible } from "../../types/collections";
import { isPayloadType } from "../../utils/asserts";
import { createEditor } from "../../utils/editor";
import { buildVersionedCollectionConfig } from "../../utils/versionedCollectionConfig";
import { RowLabel } from "./components/RowLabel";
@ -702,4 +704,14 @@ export const Collectibles = buildVersionedCollectionConfig({
],
},
],
custom: {
getBackPropagatedRelationships: ({ subitems, contents }: Collectible) => {
const result: string[] = [];
subitems?.forEach((subitem) => result.push(isPayloadType(subitem) ? subitem.id : subitem));
contents?.forEach(({ content: { relationTo, value } }) => {
if (relationTo === "pages") result.push(isPayloadType(value) ? value.id : value);
});
return result;
},
},
});

View File

@ -4,6 +4,8 @@ import { iconField } from "../../fields/iconField/iconField";
import { rowField } from "../../fields/rowField/rowField";
import { slugField } from "../../fields/slugField/slugField";
import { translatedFields } from "../../fields/translatedFields/translatedFields";
import { Folder } from "../../types/collections";
import { isPayloadType } from "../../utils/asserts";
import { buildCollectionConfig } from "../../utils/collectionConfig";
import { createEditor } from "../../utils/editor";
import { getBySlugEndpoint } from "./endpoints/getBySlugEndpoint";
@ -109,4 +111,21 @@ export const Folders = buildCollectionConfig({
hasMany: true,
},
],
custom: {
getBackPropagatedRelationships: ({ files, sections }: Folder) => {
const result: string[] = [];
files?.forEach(({ relationTo, value }) => {
if (relationTo === "collectibles" || relationTo === "pages") {
result.push(isPayloadType(value) ? value.id : value);
}
});
sections?.forEach(({ subfolders }) =>
subfolders?.forEach((folder) => {
result.push(isPayloadType(folder) ? folder.id : folder);
})
);
return result;
},
},
});

View File

@ -5,6 +5,8 @@ import { creditsField } from "../../fields/creditsField/creditsField";
import { imageField } from "../../fields/imageField/imageField";
import { rowField } from "../../fields/rowField/rowField";
import { translatedFields } from "../../fields/translatedFields/translatedFields";
import { Video } from "../../types/collections";
import { isPayloadType } from "../../utils/asserts";
import { buildCollectionConfig } from "../../utils/collectionConfig";
import { createEditor } from "../../utils/editor";
import { getByID } from "./endpoints/getByID";
@ -127,4 +129,12 @@ export const Videos = buildCollectionConfig({
],
}),
],
custom: {
getBackPropagatedRelationships: ({ platform, platformEnabled }: Video) => {
if (!platform || !platformEnabled) {
return [];
}
return [isPayloadType(platform.channel) ? platform.channel.id : platform.channel];
},
},
});

View File

@ -3,7 +3,7 @@ import { mustBeAdmin } from "../../accesses/collections/mustBeAdmin";
import { CollectionGroups, Collections } from "../../constants";
import { imageField } from "../../fields/imageField/imageField";
import { rowField } from "../../fields/rowField/rowField";
import { afterChangeWebhook } from "../../hooks/afterChangeWebhook";
import { globalAfterChangeWebhook } from "../../hooks/afterOperationWebhook";
import { getConfigEndpoint } from "./endpoints/getConfigEndpoint";
const fields = {
@ -32,7 +32,7 @@ export const WebsiteConfig: GlobalConfig = {
access: { update: mustBeAdmin, read: mustBeAdmin },
endpoints: [getConfigEndpoint],
hooks: {
afterChange: [afterChangeWebhook],
afterChange: [globalAfterChangeWebhook],
},
fields: [
rowField([

View File

@ -88,18 +88,12 @@ export enum AttributeTypes {
/* WEB HOOKS */
export interface WebHookMessage {
export type AfterOperationWebHookMessage = {
collection: Collections;
operation: WebHookOperationType;
id?: string;
}
export enum WebHookOperationType {
create = "create",
update = "update",
delete = "delete",
}
addedDependantIds: string[];
urls: string[];
};
/* RICH TEXT */
export type RichTextContent = {

View File

@ -22,6 +22,11 @@ export const getAllPathsEndpoint: Endpoint = {
depth: 0,
pagination: false,
user: req.user,
where: {
_status: {
equals: "published",
},
},
});
const pages = await payload.find({
@ -29,6 +34,11 @@ export const getAllPathsEndpoint: Endpoint = {
depth: 0,
pagination: false,
user: req.user,
where: {
_status: {
equals: "published",
},
},
});
const folders = await payload.find({
@ -66,6 +76,18 @@ export const getAllPathsEndpoint: Endpoint = {
user: req.user,
});
const chronologyEvents = await payload.find({
collection: Collections.ChronologyEvents,
depth: 0,
pagination: false,
user: req.user,
where: {
_status: {
equals: "published",
},
},
});
const result: EndpointAllPaths = {
collectibles: collectibles.docs.map(({ slug }) => slug),
pages: pages.docs.map(({ slug }) => slug),
@ -74,6 +96,7 @@ export const getAllPathsEndpoint: Endpoint = {
audios: audios.docs.map(({ id }) => id),
images: images.docs.map(({ id }) => id),
recorders: recorders.docs.map(({ id }) => id),
chronologyEvents: chronologyEvents.docs.map(({ id }) => id),
};
return res.status(200).send(result);

View File

@ -0,0 +1,47 @@
import payload, { GeneratedTypes } from "payload";
import { SanitizedCollectionConfig, SanitizedGlobalConfig } from "payload/types";
export const getAddedBackPropagationRelationships = async (
config: SanitizedCollectionConfig | SanitizedGlobalConfig,
doc: any,
previousDoc?: any
): Promise<string[]> => {
if (!("getBackPropagatedRelationships" in config.custom)) {
return [];
}
const getBackPropagatedRelationships: (doc: any) => string[] =
config.custom.getBackPropagatedRelationships;
if (!previousDoc) {
return getBackPropagatedRelationships(doc);
}
let currentIds: string[];
let previousIds: string[];
if (config.versions.drafts) {
const versions = await payload.findVersions({
collection: config.slug as keyof GeneratedTypes["collections"],
sort: "-updatedAt",
limit: 2,
where: {
and: [{ parent: { equals: doc.id } }, { "version._status": { equals: "published" } }],
},
});
const currentVersion = versions.docs[0]?.version;
const previousVersion = versions.docs[1]?.version;
if (!currentVersion) return [];
if (!previousVersion) return getBackPropagatedRelationships(currentVersion);
currentIds = getBackPropagatedRelationships(currentVersion);
previousIds = getBackPropagatedRelationships(previousVersion);
} else {
currentIds = getBackPropagatedRelationships(doc);
previousIds = getBackPropagatedRelationships(previousDoc);
}
return currentIds.filter((id) => !previousIds.includes(id));
};

View File

@ -1,24 +0,0 @@
import { AfterChangeHook } from "payload/dist/globals/config/types";
import { Collections, WebHookMessage, WebHookOperationType } from "../constants";
export const afterChangeWebhook: AfterChangeHook = async ({ doc, global }) => {
const url = `${process.env.WEB_HOOK_URI}/collection-operation`;
const message: WebHookMessage = {
collection: global.slug as Collections,
operation: WebHookOperationType.update,
};
fetch(url, {
headers: {
Authorization: `Bearer ${process.env.WEB_HOOK_TOKEN}`,
"Content-Type": "application/json",
},
body: JSON.stringify(message),
method: "POST",
}).catch((e) => {
console.warn("Error while sending webhook", url, e);
});
return doc;
};

View File

@ -1,49 +1,144 @@
import { AfterOperationHook } from "payload/dist/collections/config/types";
import { Collections, WebHookMessage, WebHookOperationType } from "../constants";
import {
AfterDeleteHook,
AfterChangeHook as CollectionAfterChangeHook,
} from "payload/dist/collections/config/types";
import { AfterChangeHook as GlobalAfterChangeHook } from "payload/dist/globals/config/types";
import { AfterOperationWebHookMessage, Collections } from "../constants";
import { getAddedBackPropagationRelationships } from "../fields/backPropagationField/backPropagationUtils";
import { getSDKEndpoint } from "../sdk";
import { Collectible } from "../types/collections";
const convertOperationToWebHookOperationType = (
operation: string
): WebHookOperationType | undefined => {
switch (operation) {
case "create":
return WebHookOperationType.create;
const getURLs = (collection: Collections, doc: any): string[] => {
switch (collection) {
case Collections.WebsiteConfig:
return [getSDKEndpoint.getConfigEndpoint()];
case "update":
case "updateByID":
return WebHookOperationType.update;
case Collections.Folders:
return [getSDKEndpoint.getFolderEndpoint(doc.slug)];
case "delete":
case "deleteByID":
return WebHookOperationType.delete;
case Collections.Languages:
return [getSDKEndpoint.getLanguagesEndpoint()];
default:
return undefined;
case Collections.Currencies:
return [getSDKEndpoint.getCurrenciesEndpoint()];
case Collections.Wordings:
return [getSDKEndpoint.getWordingsEndpoint()];
case Collections.Pages:
return [getSDKEndpoint.getPageEndpoint(doc.slug)];
case Collections.Collectibles: {
const { slug, gallery, scans, scansEnabled } = doc as Collectible;
const urls: string[] = [getSDKEndpoint.getCollectibleEndpoint(slug)];
if (gallery && gallery.length > 0) {
urls.push(getSDKEndpoint.getCollectibleGalleryEndpoint(slug));
urls.push(
...gallery.map((_, index) =>
getSDKEndpoint.getCollectibleGalleryImageEndpoint(slug, index.toString())
)
);
}
if (scans && scansEnabled) {
urls.push(getSDKEndpoint.getCollectibleScansEndpoint(slug));
// TODO: Add other pages for cover, obi, dustjacket...
if (scans.pages) {
urls.push(
...scans.pages.map(({ page }) =>
getSDKEndpoint.getCollectibleScanPageEndpoint(slug, page.toString())
)
);
}
}
return urls;
}
case Collections.ChronologyEvents:
return [
getSDKEndpoint.getChronologyEventsEndpoint(),
getSDKEndpoint.getChronologyEventByIDEndpoint(doc.id),
];
case Collections.Images:
return [getSDKEndpoint.getImageByIDEndpoint(doc.id)];
case Collections.Audios:
return [getSDKEndpoint.getAudioByIDEndpoint(doc.id)];
case Collections.Videos:
return [getSDKEndpoint.getVideoByIDEndpoint(doc.id)];
case Collections.Recorders:
return [getSDKEndpoint.getRecorderByIDEndpoint(doc.id)];
default: {
console.warn("Unrecognized collection", collection, "when sending webhook. No URL.");
return [];
}
}
};
export const afterOperationWebhook: AfterOperationHook = ({ result, collection, operation }) => {
const operationType = convertOperationToWebHookOperationType(operation);
if (!operationType) return result;
if (operationType === WebHookOperationType.update) {
if ("_status" in result && result._status === "draft") {
return result;
}
}
if (!("id" in result)) {
return result;
}
const message: WebHookMessage = {
collection: collection.slug as Collections,
operation: operationType,
id: result.id,
export const globalAfterChangeWebhook: GlobalAfterChangeHook = async ({
global,
doc,
previousDoc,
}) => {
const collection = global.slug as Collections;
await sendWebhookMessage({
collection,
addedDependantIds: await getAddedBackPropagationRelationships(global, doc, previousDoc),
urls: getURLs(collection, doc),
});
return doc;
};
const url = `${process.env.WEB_HOOK_URI}/collection-operation`;
export const collectionAfterChangeWebhook: CollectionAfterChangeHook = async ({
collection,
doc,
previousDoc,
operation,
}) => {
const collectionSlug = collection.slug as Collections;
console.log("afterChange", operation, collectionSlug, doc.id);
fetch(url, {
if ("_status" in doc && doc._status === "draft") {
return doc;
}
if (!("id" in doc)) {
return doc;
}
await sendWebhookMessage({
collection: collectionSlug,
id: doc.id,
addedDependantIds: await getAddedBackPropagationRelationships(collection, doc, previousDoc),
urls: getURLs(collectionSlug, doc),
});
return doc;
};
export const afterDeleteWebhook: AfterDeleteHook = async ({ collection, doc }) => {
const collectionSlug = collection.slug as Collections;
console.log("afterDelete", collection.slug, doc.id);
if (!("id" in doc)) {
return doc;
}
await sendWebhookMessage({
collection: collectionSlug,
id: doc.id,
addedDependantIds: [],
urls: getURLs(collectionSlug, doc),
});
return doc;
};
const sendWebhookMessage = async (message: AfterOperationWebHookMessage) => {
await fetch(`${process.env.WEB_HOOK_URI}/collection-operation`, {
headers: {
Authorization: `Bearer ${process.env.WEB_HOOK_TOKEN}`,
"Content-Type": "application/json",
@ -51,8 +146,6 @@ export const afterOperationWebhook: AfterOperationHook = ({ result, collection,
body: JSON.stringify(message),
method: "POST",
}).catch((e) => {
console.warn("Error while sending webhook", url, e);
console.warn("Error while sending webhook", e);
});
return result;
};

View File

@ -524,6 +524,7 @@ export type EndpointAllPaths = {
audios: string[];
images: string[];
recorders: string[];
chronologyEvents: string[];
};
// SDK
@ -544,6 +545,31 @@ type GetPayloadSDKParams = {
const logResponse = (res: Response) => console.log(res.status, res.statusText, res.url);
export const getSDKEndpoint = {
getConfigEndpoint: () => `/globals/${Collections.WebsiteConfig}/config`,
getFolderEndpoint: (slug: string) => `/${Collections.Folders}/slug/${slug}`,
getLanguagesEndpoint: () => `/${Collections.Languages}/all`,
getCurrenciesEndpoint: () => `/${Collections.Currencies}/all`,
getWordingsEndpoint: () => `/${Collections.Wordings}/all`,
getPageEndpoint: (slug: string) => `/${Collections.Pages}/slug/${slug}`,
getCollectibleEndpoint: (slug: string) => `/${Collections.Collectibles}/slug/${slug}`,
getCollectibleScansEndpoint: (slug: string) => `/${Collections.Collectibles}/slug/${slug}/scans`,
getCollectibleScanPageEndpoint: (slug: string, index: string) =>
`/${Collections.Collectibles}/slug/${slug}/scans/${index}`,
getCollectibleGalleryEndpoint: (slug: string) =>
`/${Collections.Collectibles}/slug/${slug}/gallery`,
getCollectibleGalleryImageEndpoint: (slug: string, index: string) =>
`/${Collections.Collectibles}/slug/${slug}/gallery/${index}`,
getChronologyEventsEndpoint: () => `/${Collections.ChronologyEvents}/all`,
getChronologyEventByIDEndpoint: (id: string) => `/${Collections.ChronologyEvents}/id/${id}`,
getImageByIDEndpoint: (id: string) => `/${Collections.Images}/id/${id}`,
getAudioByIDEndpoint: (id: string) => `/${Collections.Audios}/id/${id}`,
getVideoByIDEndpoint: (id: string) => `/${Collections.Videos}/id/${id}`,
getRecorderByIDEndpoint: (id: string) => `/${Collections.Recorders}/id/${id}`,
getAllPathsEndpoint: () => `/all-paths`,
getLoginEndpoint: () => `/${Collections.Recorders}/login`,
};
export const getPayloadSDK = ({
apiURL,
email,
@ -552,7 +578,7 @@ export const getPayloadSDK = ({
responseCache,
}: GetPayloadSDKParams) => {
const refreshToken = async () => {
const loginUrl = `${apiURL}/${Collections.Recorders}/login`;
const loginUrl = `${apiURL}${getSDKEndpoint.getLoginEndpoint()}`;
const loginResult = await fetch(loginUrl, {
method: "POST",
headers: { "Content-Type": "application/json" },
@ -572,13 +598,13 @@ export const getPayloadSDK = ({
return token;
};
const request = async (url: string): Promise<any> => {
const cachedResponse = responseCache?.get(url);
const request = async (endpoint: string): Promise<any> => {
const cachedResponse = responseCache?.get(endpoint);
if (cachedResponse) {
return cachedResponse;
}
const result = await fetch(url, {
const result = await fetch(`${apiURL}${endpoint}`, {
headers: {
Authorization: `JWT ${tokenCache?.get() ?? (await refreshToken())}`,
},
@ -590,52 +616,53 @@ export const getPayloadSDK = ({
}
const data = await result.json();
responseCache?.set(url, data);
responseCache?.set(endpoint, data);
return data;
};
return {
getConfig: async (): Promise<EndpointWebsiteConfig> =>
await request(`${apiURL}/globals/${Collections.WebsiteConfig}/config`),
await request(getSDKEndpoint.getConfigEndpoint()),
getFolder: async (slug: string): Promise<EndpointFolder> =>
await request(`${apiURL}/${Collections.Folders}/slug/${slug}`),
await request(getSDKEndpoint.getFolderEndpoint(slug)),
getLanguages: async (): Promise<Language[]> =>
await request(`${apiURL}/${Collections.Languages}/all`),
await request(getSDKEndpoint.getLanguagesEndpoint()),
getCurrencies: async (): Promise<Currency[]> =>
await request(`${apiURL}/${Collections.Currencies}/all`),
await request(getSDKEndpoint.getCurrenciesEndpoint()),
getWordings: async (): Promise<EndpointWording[]> =>
await request(`${apiURL}/${Collections.Wordings}/all`),
await request(getSDKEndpoint.getWordingsEndpoint()),
getPage: async (slug: string): Promise<EndpointPage> =>
await request(`${apiURL}/${Collections.Pages}/slug/${slug}`),
await request(getSDKEndpoint.getPageEndpoint(slug)),
getCollectible: async (slug: string): Promise<EndpointCollectible> =>
await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}`),
await request(getSDKEndpoint.getCollectibleEndpoint(slug)),
getCollectibleScans: async (slug: string): Promise<EndpointCollectibleScans> =>
await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/scans`),
await request(getSDKEndpoint.getCollectibleScansEndpoint(slug)),
getCollectibleScanPage: async (
slug: string,
index: string
): Promise<EndpointCollectibleScanPage> =>
await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/scans/${index}`),
await request(getSDKEndpoint.getCollectibleScanPageEndpoint(slug, index)),
getCollectibleGallery: async (slug: string): Promise<EndpointCollectibleGallery> =>
await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/gallery`),
await request(getSDKEndpoint.getCollectibleGalleryEndpoint(slug)),
getCollectibleGalleryImage: async (
slug: string,
index: string
): Promise<EndpointCollectibleGalleryImage> =>
await request(`${apiURL}/${Collections.Collectibles}/slug/${slug}/gallery/${index}`),
await request(getSDKEndpoint.getCollectibleGalleryImageEndpoint(slug, index)),
getChronologyEvents: async (): Promise<EndpointChronologyEvent[]> =>
await request(`${apiURL}/${Collections.ChronologyEvents}/all`),
await request(getSDKEndpoint.getChronologyEventsEndpoint()),
getChronologyEventByID: async (id: string): Promise<EndpointChronologyEvent> =>
await request(`${apiURL}/${Collections.ChronologyEvents}/id/${id}`),
await request(getSDKEndpoint.getChronologyEventByIDEndpoint(id)),
getImageByID: async (id: string): Promise<EndpointImage> =>
await request(`${apiURL}/${Collections.Images}/id/${id}`),
await request(getSDKEndpoint.getImageByIDEndpoint(id)),
getAudioByID: async (id: string): Promise<EndpointAudio> =>
await request(`${apiURL}/${Collections.Audios}/id/${id}`),
await request(getSDKEndpoint.getAudioByIDEndpoint(id)),
getVideoByID: async (id: string): Promise<EndpointVideo> =>
await request(`${apiURL}/${Collections.Videos}/id/${id}`),
await request(getSDKEndpoint.getVideoByIDEndpoint(id)),
getRecorderByID: async (id: string): Promise<EndpointRecorder> =>
await request(`${apiURL}/${Collections.Recorders}/id/${id}`),
getAllPaths: async (): Promise<EndpointAllPaths> => await request(`${apiURL}/all-paths`),
request: async (url: string): Promise<any> => await request(url),
await request(getSDKEndpoint.getRecorderByIDEndpoint(id)),
getAllPaths: async (): Promise<EndpointAllPaths> =>
await request(getSDKEndpoint.getAllPathsEndpoint()),
request: async (pathname: string): Promise<any> => await request(pathname),
};
};

View File

@ -1,16 +1,20 @@
import { GeneratedTypes } from "payload";
import { CollectionConfig } from "payload/types";
import { afterOperationWebhook } from "../hooks/afterOperationWebhook";
import { afterDeleteWebhook, collectionAfterChangeWebhook } from "../hooks/afterOperationWebhook";
import { formatToPascalCase } from "./string";
type CollectionConfigWithPlugins = CollectionConfig;
export type BuildCollectionConfig = Omit<
CollectionConfigWithPlugins,
"slug" | "typescript" | "labels"
"slug" | "typescript" | "labels" | "custom"
> & {
slug: keyof GeneratedTypes["collections"];
labels: { singular: string; plural: string };
custom?: {
getBackPropagatedRelationships?: (object: any) => string[];
[key: string]: unknown;
};
};
export const buildCollectionConfig = (config: BuildCollectionConfig): CollectionConfig => ({
@ -18,6 +22,7 @@ export const buildCollectionConfig = (config: BuildCollectionConfig): Collection
typescript: { interface: formatToPascalCase(config.labels.singular) },
hooks: {
...config.hooks,
afterOperation: [...(config.hooks?.afterOperation ?? []), afterOperationWebhook],
afterChange: [...(config.hooks?.afterChange ?? []), collectionAfterChangeWebhook],
afterDelete: [...(config.hooks?.afterDelete ?? []), afterDeleteWebhook],
},
});