Use SFTP instead of FTP

This commit is contained in:
DrMint 2024-04-06 10:36:11 +02:00
parent 8e36642715
commit b15fe7a0ce
10 changed files with 219 additions and 129 deletions

View File

@ -16,7 +16,8 @@ SEEDING_ADMIN_PASSWORD=somepassword
WEB_HOOK_TOKEN=webhooktoken5e6ea45ef4e66eaa151612bdcb599df
WEB_HOOK_URI=https://accords-library.com/some/path
FTP_USER=someuser
FTP_PASSWORD=somepassword
FTP_HOST=ftp.host.com
FTP_BASE_URL=https://ftp-base-url.com
SFTP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nxxxxxxxxxx..."
SFTP_USERNAME=someuser
SFTP_HOST=ftp.host.com
SFTP_DESTINATION_PATH_ROOT="/absolute/path/to/destination/root/folder"
SFTP_BASE_URL=https://ftp-base-url.com

175
package-lock.json generated
View File

@ -12,13 +12,12 @@
"@fontsource/vollkorn": "5.0.19",
"@payloadcms/bundler-webpack": "1.0.6",
"@payloadcms/db-mongodb": "1.4.4",
"@payloadcms/plugin-cloud-storage": "^1.1.2",
"@payloadcms/richtext-lexical": "0.8.0",
"basic-ftp": "^5.0.5",
"cross-env": "7.0.3",
"language-tags": "1.0.9",
"luxon": "3.4.4",
"payload": "2.12.1",
"payloadcms-sftp-storage": "^1.0.1",
"sharp": "0.33.3",
"styled-components": "6.1.8"
},
@ -4516,6 +4515,30 @@
"@types/node": "*"
}
},
"node_modules/@types/ssh2": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.0.tgz",
"integrity": "sha512-YcT8jP5F8NzWeevWvcyrrLB3zcneVjzYY9ZDSMAMboI+2zR1qYWFhwsyOFVzT7Jorn67vqxC0FRiw8YyG9P1ww==",
"dependencies": {
"@types/node": "^18.11.18"
}
},
"node_modules/@types/ssh2-sftp-client": {
"version": "9.0.3",
"resolved": "https://registry.npmjs.org/@types/ssh2-sftp-client/-/ssh2-sftp-client-9.0.3.tgz",
"integrity": "sha512-pkCiS/NYvfc8S6xq3TvHAIPhQvBcl9Z1kMFxS8yNsqxmg/8ozzglnT4TrfpYBR1hlBky3r+fYntdZ5WnvvlKoQ==",
"dependencies": {
"@types/ssh2": "*"
}
},
"node_modules/@types/ssh2/node_modules/@types/node": {
"version": "18.19.30",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.30.tgz",
"integrity": "sha512-453z1zPuJLVDbyahaa1sSD5C2sht6ZpHp5rgJNs+H8YGqhluCXcuOUmBYsAo0Tos0cHySJ3lVUGbGgLlqIkpyg==",
"dependencies": {
"undici-types": "~5.26.4"
}
},
"node_modules/@types/styled-components": {
"version": "5.1.34",
"resolved": "https://registry.npmjs.org/@types/styled-components/-/styled-components-5.1.34.tgz",
@ -5046,6 +5069,14 @@
"node": ">=8"
}
},
"node_modules/asn1": {
"version": "0.2.6",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
"integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
"dependencies": {
"safer-buffer": "~2.1.0"
}
},
"node_modules/atomic-sleep": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz",
@ -5152,12 +5183,12 @@
}
]
},
"node_modules/basic-ftp": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz",
"integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==",
"engines": {
"node": ">=10.0.0"
"node_modules/bcrypt-pbkdf": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
"integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
"dependencies": {
"tweetnacl": "^0.14.3"
}
},
"node_modules/big.js": {
@ -5484,6 +5515,15 @@
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
},
"node_modules/buildcheck": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
"integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==",
"optional": true,
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/builtins": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/builtins/-/builtins-5.0.1.tgz",
@ -6042,6 +6082,41 @@
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
},
"node_modules/concat-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
"integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
"engines": [
"node >= 6.0"
],
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.0.2",
"typedarray": "^0.0.6"
}
},
"node_modules/concat-stream/node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/concat-stream/node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"dependencies": {
"safe-buffer": "~5.2.0"
}
},
"node_modules/conf": {
"version": "10.2.0",
"resolved": "https://registry.npmjs.org/conf/-/conf-10.2.0.tgz",
@ -6213,6 +6288,20 @@
"node": ">=10"
}
},
"node_modules/cpu-features": {
"version": "0.0.9",
"resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz",
"integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==",
"hasInstallScript": true,
"optional": true,
"dependencies": {
"buildcheck": "~0.0.6",
"nan": "^2.17.0"
},
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/create-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
@ -7106,8 +7195,7 @@
"node_modules/err-code": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz",
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==",
"dev": true
"integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA=="
},
"node_modules/error-ex": {
"version": "1.3.2",
@ -10353,6 +10441,12 @@
"thenify-all": "^1.0.0"
}
},
"node_modules/nan": {
"version": "2.19.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz",
"integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==",
"optional": true
},
"node_modules/napi-build-utils": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
@ -11614,6 +11708,16 @@
"uuid": "dist/bin/uuid"
}
},
"node_modules/payloadcms-sftp-storage": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/payloadcms-sftp-storage/-/payloadcms-sftp-storage-1.0.1.tgz",
"integrity": "sha512-qgPIBsSHNPj0jMw08oodJcCH66fLZp9xDAOLXjbdJ7P4GrUF1cvYj7qTeyTETnBhglXPuT+8w1ZLKpY2ey3DQw==",
"dependencies": {
"@payloadcms/plugin-cloud-storage": "^1.1.2",
"@types/ssh2-sftp-client": "^9.0.3",
"ssh2-sftp-client": "^10.0.3"
}
},
"node_modules/peek-readable": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz",
@ -13187,7 +13291,6 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz",
"integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==",
"dev": true,
"dependencies": {
"err-code": "^2.0.2",
"retry": "^0.12.0"
@ -14111,7 +14214,6 @@
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
"integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
"dev": true,
"engines": {
"node": ">= 4"
}
@ -14845,6 +14947,40 @@
"node": ">= 10.x"
}
},
"node_modules/ssh2": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz",
"integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==",
"hasInstallScript": true,
"dependencies": {
"asn1": "^0.2.6",
"bcrypt-pbkdf": "^1.0.2"
},
"engines": {
"node": ">=10.16.0"
},
"optionalDependencies": {
"cpu-features": "~0.0.9",
"nan": "^2.18.0"
}
},
"node_modules/ssh2-sftp-client": {
"version": "10.0.3",
"resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-10.0.3.tgz",
"integrity": "sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==",
"dependencies": {
"concat-stream": "^2.0.0",
"promise-retry": "^2.0.1",
"ssh2": "^1.15.0"
},
"engines": {
"node": ">=16.20.2"
},
"funding": {
"type": "individual",
"url": "https://square.link/u/4g7sPflL"
}
},
"node_modules/ssri": {
"version": "10.0.5",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-10.0.5.tgz",
@ -15737,6 +15873,11 @@
"node": "*"
}
},
"node_modules/tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="
},
"node_modules/type": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz",
@ -15766,6 +15907,11 @@
"node": ">= 0.6"
}
},
"node_modules/typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="
},
"node_modules/typedarray-to-buffer": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
@ -15793,6 +15939,11 @@
"integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
"dev": true
},
"node_modules/undici-types": {
"version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
},
"node_modules/unique-filename": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz",

View File

@ -25,13 +25,12 @@
"@fontsource/vollkorn": "5.0.19",
"@payloadcms/bundler-webpack": "1.0.6",
"@payloadcms/db-mongodb": "1.4.4",
"@payloadcms/plugin-cloud-storage": "^1.1.2",
"@payloadcms/richtext-lexical": "0.8.0",
"basic-ftp": "^5.0.5",
"cross-env": "7.0.3",
"language-tags": "1.0.9",
"luxon": "3.4.4",
"payload": "2.12.1",
"payloadcms-sftp-storage": "1.0.1",
"sharp": "0.33.3",
"styled-components": "6.1.8"
},

View File

@ -14,7 +14,6 @@ const fields = {
translations: "translations",
translationsTitle: "title",
translationsDescription: "description",
translationsSubfile: "subfile",
thumbnail: "thumbnail",
duration: "duration",
tags: "tags",
@ -28,6 +27,7 @@ export const Audios = buildCollectionConfig({
group: CollectionGroups.Media,
defaultColumns: [
fields.filename,
fields.thumbnail,
fields.mimeType,
fields.filesize,
fields.translations,
@ -58,12 +58,6 @@ export const Audios = buildCollectionConfig({
type: "richText",
editor: createEditor({ inlines: true, lists: true, links: true }),
},
{
name: fields.translationsSubfile,
type: "upload",
relationTo: Collections.VideosSubtitles,
admin: { description: "The subtitle file needs to follow the WebVTT file format (.vtt)" },
},
],
}),
tagsField({ name: fields.tags }),

View File

@ -36,6 +36,7 @@ export const Videos = buildCollectionConfig({
group: CollectionGroups.Media,
defaultColumns: [
fields.filename,
fields.thumbnail,
fields.mimeType,
fields.filesize,
fields.translations,

View File

@ -1,4 +1,11 @@
import type { BreakBlock, Image, SectionBlock, TranscriptBlock } from "./types/collections";
import type {
Audio,
BreakBlock,
Image,
SectionBlock,
TranscriptBlock,
Video,
} from "./types/collections";
// END MOCKING SECTION
@ -141,10 +148,20 @@ export interface RichTextUploadNode extends RichTextNode {
}
export interface RichTextUploadImageNode extends RichTextUploadNode {
relationTo: "images" | "background-images";
relationTo: Collections.Images;
value: Image;
}
export interface RichTextUploadVideoNode extends RichTextUploadNode {
relationTo: Collections.Videos;
value: Video;
}
export interface RichTextUploadAudioNode extends RichTextUploadNode {
relationTo: Collections.Audios;
value: Audio;
}
export interface RichTextTextNode extends RichTextNode {
type: "text";
format: number;
@ -210,7 +227,13 @@ export const isNodeUploadNode = (node: RichTextNode): node is RichTextUploadNode
node.type === "upload";
export const isUploadNodeImageNode = (node: RichTextUploadNode): node is RichTextUploadImageNode =>
node.relationTo === "images" || node.relationTo === "background-images";
node.relationTo === Collections.Images;
export const isUploadNodeVideoNode = (node: RichTextUploadNode): node is RichTextUploadVideoNode =>
node.relationTo === Collections.Videos;
export const isUploadNodeAudioNode = (node: RichTextUploadNode): node is RichTextUploadAudioNode =>
node.relationTo === Collections.Audios;
export const isNodeListNode = (node: RichTextNode): node is RichTextListNode =>
node.type === "list";

View File

@ -3,6 +3,7 @@ import { mongooseAdapter } from "@payloadcms/db-mongodb";
import { cloudStorage } from "@payloadcms/plugin-cloud-storage";
import path from "path";
import { buildConfig } from "payload/config";
import { sftpAdapter } from "payloadcms-sftp-storage";
import { Audios } from "./collections/Audios/Audios";
import { ChronologyEvents } from "./collections/ChronologyEvents/ChronologyEvents";
import { Collectibles } from "./collections/Collectibles/Collectibles";
@ -25,15 +26,16 @@ import { Wordings } from "./collections/Wordings/Wordings";
import { Icon } from "./components/Icon";
import { Logo } from "./components/Logo";
import { Collections } from "./constants";
import { ftpAdapter } from "./plugins/ftpAdapter";
import { createEditor } from "./utils/editor";
const configuredFtpAdapter = ftpAdapter({
host: process.env.FTP_HOST ?? "",
user: process.env.FTP_USER ?? "",
password: process.env.FTP_PASSWORD ?? "",
secure: false,
endpoint: process.env.FTP_BASE_URL ?? "",
const configuredFtpAdapter = sftpAdapter({
connectOptions: {
host: process.env.SFTP_HOST,
username: process.env.SFTP_USERNAME,
privateKey: process.env.SFTP_PRIVATE_KEY,
},
destinationPathRoot: process.env.SFTP_DESTINATION_PATH_ROOT ?? "",
publicEndpoint: process.env.FTP_BASE_URL ?? "",
});
export default buildConfig({

View File

@ -1,79 +0,0 @@
import {
Adapter,
GenerateURL,
HandleDelete,
HandleUpload,
} from "@payloadcms/plugin-cloud-storage/dist/types";
import { Client } from "basic-ftp";
import path from "path";
import { Readable } from "stream";
import type { Configuration as WebpackConfig } from "webpack";
interface FTPAdapterConfig {
host: string;
user: string;
password: string;
secure: boolean;
endpoint: string;
}
export const ftpAdapter =
({ endpoint, host, password, secure, user }: FTPAdapterConfig): Adapter =>
({ collection }) => {
const generateURL: GenerateURL = ({ filename }) => `${endpoint}/${collection.slug}/${filename}`;
const handleDelete: HandleDelete = async ({ filename }) => {
const client = new Client();
client.ftp.verbose = true;
await client.access({
host,
user,
password,
secure,
});
await client.ensureDir(collection.slug);
await client.remove(filename);
client.close();
};
const handleUpload: HandleUpload = async ({ file }) => {
const client = new Client();
client.ftp.verbose = true;
await client.access({
host: process.env.FTP_HOST,
user: process.env.FTP_USER,
password: process.env.FTP_PASSWORD,
secure: false,
});
await client.ensureDir(collection.slug);
await client.uploadFrom(Readable.from(file.buffer), file.filename);
client.close();
};
const webpack = (existingWebpackConfig: WebpackConfig): WebpackConfig => {
const newConfig: WebpackConfig = {
...existingWebpackConfig,
resolve: {
...(existingWebpackConfig.resolve || {}),
alias: {
...(existingWebpackConfig.resolve?.alias ? existingWebpackConfig.resolve.alias : {}),
"./plugins/ftpAdapter": path.resolve(__dirname, "./mock.js"),
},
fallback: {
...(existingWebpackConfig.resolve?.fallback
? existingWebpackConfig.resolve.fallback
: {}),
stream: false,
},
},
};
return newConfig;
};
return {
generateURL,
handleDelete,
handleUpload,
staticHandler: () => {},
webpack,
};
};

View File

@ -1 +0,0 @@
export const ftpAdapter = () => {};

View File

@ -551,7 +551,6 @@ export interface Audio {
};
[k: string]: unknown;
} | null;
subfile?: string | VideoSubtitle | null;
id?: string | null;
}[];
tags?: (string | Tag)[] | null;
@ -597,19 +596,6 @@ export interface MediaThumbnail {
};
};
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "videos-subtitles".
*/
export interface VideoSubtitle {
id: string;
url?: string | null;
filename?: string | null;
mimeType?: string | null;
filesize?: number | null;
width?: number | null;
height?: number | null;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "videos".
@ -658,6 +644,19 @@ export interface Video {
width?: number | null;
height?: number | null;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "videos-subtitles".
*/
export interface VideoSubtitle {
id: string;
url?: string | null;
filename?: string | null;
mimeType?: string | null;
filesize?: number | null;
width?: number | null;
height?: number | null;
}
/**
* This interface was referenced by `Config`'s JSON-Schema
* via the `definition` "videos-channels".