From 37393ba2153d0874c8aa908c62148ff398a0b44a Mon Sep 17 00:00:00 2001 From: John Doe Date: Mon, 16 Sep 2024 08:11:32 +0000 Subject: [PATCH 1/8] ftp download corrections --- src/ts/plugins/server/FtpConnector.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index baace80b..c750a97b 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -16,7 +16,7 @@ */ import { Client } from 'basic-ftp' -import { PassThrough, Readable, Writable } from 'stream' +import { Readable } from 'stream' import { ConnectorFile, ConnectorFileContent, HostingConnector, StatusCallback, StorageConnector, contentToReadable, contentToString, toConnectorData, toConnectorEnum} from '../../server/connectors/connectors' import { requiredParam } from '../../server/utils/validation' import { WEBSITE_DATA_FILE, WEBSITE_META_DATA_FILE } from '../../constants' @@ -26,7 +26,8 @@ import { join } from 'path' import { tmpdir } from 'os' import { v4 as uuid } from 'uuid' import { JobManager } from '../../server/jobs' -import { mkdtemp } from 'fs/promises' +import { mkdtemp, rm, rmdir } from 'fs/promises' +import { createReadStream } from 'fs' /** * @fileoverview FTP connector for Silex @@ -273,9 +274,13 @@ export default class FtpConnector implements StorageConnector { } private async read(ftp: Client, path: string): Promise { - const stream = new PassThrough() - await ftp.downloadTo(stream, path) - return stream + const tempDir = await mkdtemp(join(tmpdir(), 'silex-tmp')) + const tempPath = join(tempDir, 'silex-ftp.tmp') + await ftp.downloadTo(tempPath, path) + const rStream = createReadStream(tempPath) + await rm(tempPath) + await rmdir(tempDir) + return rStream } private async readdir(ftp: Client, path: string): Promise { @@ -404,7 +409,7 @@ export default class FtpConnector implements StorageConnector { const ftp = await this.getClient(this.sessionData(session)) // Get stats for the website folder const folder = join(this.rootPath(session), websiteId) - const lastMod = await ftp.lastMod(folder) + //const lastMod = await ftp.lastMod(folder) // Read the meta data file to get the meta data set by the user const path = join(this.rootPath(session), websiteId, WEBSITE_META_DATA_FILE) const readable = await this.read(ftp, path) @@ -414,8 +419,8 @@ export default class FtpConnector implements StorageConnector { return { websiteId, //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), - createdAt: lastMod, - updatedAt: lastMod, + createdAt: new Date(NaN), + updatedAt: new Date(NaN), ...meta, } } catch(err) { @@ -570,7 +575,7 @@ export default class FtpConnector implements StorageConnector { const storageRootPath = this.rootPath(session) const ftp = await this.getClient(this.sessionData(session)) const dirPath = join(this.options.path, storageRootPath, id, this.options.assetsFolder, path) - const asset = this.read(ftp, dirPath) + const asset = await this.read(ftp, dirPath) this.closeClient(ftp) return asset } From b7cfff71c1aec864021cf843739bfc5cb9d97811 Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 06:53:45 +0000 Subject: [PATCH 2/8] ftp connector corrections after comments review --- src/ts/plugins/server/FtpConnector.ts | 4 ---- src/ts/types.ts | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index c750a97b..832988c5 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -409,7 +409,6 @@ export default class FtpConnector implements StorageConnector { const ftp = await this.getClient(this.sessionData(session)) // Get stats for the website folder const folder = join(this.rootPath(session), websiteId) - //const lastMod = await ftp.lastMod(folder) // Read the meta data file to get the meta data set by the user const path = join(this.rootPath(session), websiteId, WEBSITE_META_DATA_FILE) const readable = await this.read(ftp, path) @@ -418,9 +417,6 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, - //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), - createdAt: new Date(NaN), - updatedAt: new Date(NaN), ...meta, } } catch(err) { diff --git a/src/ts/types.ts b/src/ts/types.ts index 293cfa46..e2e310ee 100644 --- a/src/ts/types.ts +++ b/src/ts/types.ts @@ -308,8 +308,8 @@ export interface WebsiteMetaFileContent { */ export interface WebsiteMeta extends WebsiteMetaFileContent { websiteId: WebsiteId - createdAt: Date - updatedAt: Date + createdAt?: Date + updatedAt?: Date } /** From 723e3c505359083a312871a1d3a0e3e0c6ac548d Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 07:01:04 +0000 Subject: [PATCH 3/8] Revert "ftp connector corrections after comments review" This reverts commit b7cfff71c1aec864021cf843739bfc5cb9d97811. --- src/ts/plugins/server/FtpConnector.ts | 4 ++++ src/ts/types.ts | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index 832988c5..c750a97b 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -409,6 +409,7 @@ export default class FtpConnector implements StorageConnector { const ftp = await this.getClient(this.sessionData(session)) // Get stats for the website folder const folder = join(this.rootPath(session), websiteId) + //const lastMod = await ftp.lastMod(folder) // Read the meta data file to get the meta data set by the user const path = join(this.rootPath(session), websiteId, WEBSITE_META_DATA_FILE) const readable = await this.read(ftp, path) @@ -417,6 +418,9 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, + //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), + createdAt: new Date(NaN), + updatedAt: new Date(NaN), ...meta, } } catch(err) { diff --git a/src/ts/types.ts b/src/ts/types.ts index e2e310ee..293cfa46 100644 --- a/src/ts/types.ts +++ b/src/ts/types.ts @@ -308,8 +308,8 @@ export interface WebsiteMetaFileContent { */ export interface WebsiteMeta extends WebsiteMetaFileContent { websiteId: WebsiteId - createdAt?: Date - updatedAt?: Date + createdAt: Date + updatedAt: Date } /** From 73e65f4d49804f23edeb055e9f59142b3de1149b Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 07:02:14 +0000 Subject: [PATCH 4/8] ftp connector corrections after comments review --- src/ts/plugins/server/FtpConnector.ts | 5 ----- src/ts/types.ts | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index c750a97b..3e7bf695 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -409,8 +409,6 @@ export default class FtpConnector implements StorageConnector { const ftp = await this.getClient(this.sessionData(session)) // Get stats for the website folder const folder = join(this.rootPath(session), websiteId) - //const lastMod = await ftp.lastMod(folder) - // Read the meta data file to get the meta data set by the user const path = join(this.rootPath(session), websiteId, WEBSITE_META_DATA_FILE) const readable = await this.read(ftp, path) const meta = JSON.parse(await contentToString(readable)) as WebsiteMetaFileContent @@ -418,9 +416,6 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, - //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), - createdAt: new Date(NaN), - updatedAt: new Date(NaN), ...meta, } } catch(err) { diff --git a/src/ts/types.ts b/src/ts/types.ts index 293cfa46..e2e310ee 100644 --- a/src/ts/types.ts +++ b/src/ts/types.ts @@ -308,8 +308,8 @@ export interface WebsiteMetaFileContent { */ export interface WebsiteMeta extends WebsiteMetaFileContent { websiteId: WebsiteId - createdAt: Date - updatedAt: Date + createdAt?: Date + updatedAt?: Date } /** From b1806df553307a75c1cd9844bcc1475125d8fcbb Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 07:13:57 +0000 Subject: [PATCH 5/8] ftp connector corrections after comments review --- src/ts/plugins/server/FtpConnector.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index 3e7bf695..1778de1e 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -416,6 +416,7 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, + //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), ...meta, } } catch(err) { From 01316cd03280193c36323edb487505ebb8d24b90 Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 07:38:49 +0000 Subject: [PATCH 6/8] Revert "ftp connector corrections after comments review" This reverts commit b7cfff71c1aec864021cf843739bfc5cb9d97811. --- src/ts/plugins/server/FtpConnector.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index 1778de1e..3e7bf695 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -416,7 +416,6 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, - //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), ...meta, } } catch(err) { From f23298a4951237a9fd0f0eda3e4651031d11d7f3 Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 07:57:28 +0000 Subject: [PATCH 7/8] ftp connector 2 --- src/ts/plugins/server/FtpConnector.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index 3e7bf695..1778de1e 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -416,6 +416,7 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, + //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), ...meta, } } catch(err) { From 406be6d13fa6fc3cb02f203d13c13991e31ae0c5 Mon Sep 17 00:00:00 2001 From: John Doe Date: Tue, 24 Sep 2024 16:54:45 +0000 Subject: [PATCH 8/8] modif ftp connect2 --- src/ts/plugins/server/FtpConnector.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ts/plugins/server/FtpConnector.ts b/src/ts/plugins/server/FtpConnector.ts index 1778de1e..3e7bf695 100644 --- a/src/ts/plugins/server/FtpConnector.ts +++ b/src/ts/plugins/server/FtpConnector.ts @@ -416,7 +416,6 @@ export default class FtpConnector implements StorageConnector { // Return all meta return { websiteId, - //url: await this.getFileUrl(session, websiteId, WEBSITE_DATA_FILE_NAME), ...meta, } } catch(err) {