Skip to content

Commit

Permalink
Merge pull request #194 from autonomys/chore/remove-fs
Browse files Browse the repository at this point in the history
Separate into node and browser outputs
  • Loading branch information
clostao authored Dec 10, 2024
2 parents 2e8fe17 + a851443 commit ade7eac
Show file tree
Hide file tree
Showing 13 changed files with 1,504 additions and 211 deletions.
25 changes: 21 additions & 4 deletions packages/auto-drive/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,9 @@
"version": "1.0.12",
"license": "MIT",
"type": "module",
"main": "dist/index.js",
"repository": {
"type": "git",
"url": "https://github.com/autonomys/auto-sdk"
"type": "git",
"url": "https://github.com/autonomys/auto-sdk"
},
"author": {
"name": "Autonomys",
Expand All @@ -20,17 +19,35 @@
"node": ">=20.8.0"
},
"scripts": {
"build": "tsc"
"build": "rollup -c"
},
"exports": {
".": {
"browser": "./dist/browser.js",
"node": "./dist/node.js",
"types": "./dist/index.d.ts"
}
},
"devDependencies": {
"@prerenderer/rollup-plugin": "^0.3.12",
"@rollup/plugin-commonjs": "^28.0.1",
"@rollup/plugin-json": "^6.1.0",
"@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-typescript": "^12.1.1",
"@types/mime-types": "^2",
"rollup": "^4.28.0",
"rollup-plugin-jscc": "^2.0.0",
"rollup-plugin-terser": "^7.0.2",
"tslib": "^2.8.1",
"typescript": "^5.6.3"
},
"dependencies": {
"@autonomys/auto-dag-data": "^1.0.12",
"jszip": "^3.10.1",
"mime-types": "^2.1.35",
"process": "^0.11.10",
"rxjs": "^7.8.1",
"stream": "^0.0.3",
"zod": "^3.23.8"
},
"gitHead": "aead187672e1a487b54d0f6b973b6a546e2f7e99"
Expand Down
40 changes: 40 additions & 0 deletions packages/auto-drive/rollup.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import commonjs from '@rollup/plugin-commonjs'
import json from '@rollup/plugin-json'
import resolve from '@rollup/plugin-node-resolve'
import typescript from '@rollup/plugin-typescript'
import { terser } from 'rollup-plugin-terser'

export default [
{
input: 'src/node.ts',
output: {
file: 'dist/node.js',
format: 'esm',
sourcemap: true,
},
plugins: [
resolve({
preferBuiltins: false,
}),
commonjs(),
typescript({ tsconfig: './tsconfig.json' }),
json(),
],
},
{
input: 'src/browser.ts',
output: {
file: 'dist/browser.js',
format: 'esm',
sourcemap: true,
},
plugins: [
resolve({
preferBuiltins: false,
}),
commonjs(),
typescript({ tsconfig: './tsconfig.json' }),
json(),
],
},
]
12 changes: 3 additions & 9 deletions packages/auto-drive/src/api/models/folderTree.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import fs from 'fs'
import JSZip from 'jszip'
import { z } from 'zod'

Expand Down Expand Up @@ -93,16 +92,11 @@ export const constructFromInput = (input: File[]): FolderTree => {
const addFilesToZip = (
folder: JSZip,
folderNode: FolderTreeFolder,
files: Record<string, File | string>,
files: Record<string, File>,
) => {
folderNode.children.forEach((child) => {
if (child.type === 'file') {
const file = files[child.id]
if (typeof file === 'string') {
folder.file(child.name, fs.createReadStream(file))
} else {
folder.file(child.name, file)
}
folder.file(child.name, files[child.id])
} else if (child.type === 'folder') {
const subFolder = folder.folder(child.name)
if (!subFolder) {
Expand All @@ -115,7 +109,7 @@ const addFilesToZip = (

export const constructZipBlobFromTreeAndPaths = async (
tree: FolderTree,
files: Record<string, File | string>,
files: Record<string, File>,
) => {
if (tree.type === 'file') {
throw new Error('Cannot construct zip from file')
Expand Down
137 changes: 2 additions & 135 deletions packages/auto-drive/src/api/wrappers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@ import {
EncryptionAlgorithm,
stringToCid,
} from '@autonomys/auto-dag-data'
import fs from 'fs'
import mime from 'mime-types'
import { asyncByChunk, asyncFromStream, fileToIterable } from '../utils/async.js'
import { getFiles } from '../utils/folder.js'
import { progressToPercentage } from '../utils/misc.js'
import { PromisedObservable } from '../utils/observable.js'
import {
Expand All @@ -24,14 +22,10 @@ import {
} from './calls/index.js'
import { AutoDriveApi } from './connection.js'
import { GenericFile } from './models/file.js'
import {
constructFromFileSystemEntries,
constructFromInput,
constructZipBlobFromTreeAndPaths,
} from './models/folderTree.js'
import { constructFromInput, constructZipBlobFromTreeAndPaths } from './models/folderTree.js'
import { UploadChunksStatus, UploadFileStatus, UploadFolderStatus } from './models/uploads.js'

type UploadFileOptions = {
export type UploadFileOptions = {
password?: string
compression?: boolean
}
Expand All @@ -57,48 +51,6 @@ const uploadFileChunks = (
})
}

/**
* Uploads a file to the server with optional encryption and compression.
*
* This function reads a file from the specified file path, optionally encrypts it
* using the provided password, and compresses it using the specified algorithm if requested.
* It then uploads the file in chunks to the server, creating an upload session and
* completing it once all chunks have been successfully uploaded.
*
* @param {AutoDriveApi} api - The API instance used to send requests.
* @param {string} filePath - The path to the file to be uploaded.
* @param {UploadFileOptions} options - Options for the upload process.
* @param {string} [options.password] - The password for encryption (optional).
* @param {boolean} [options.compression=true] - Whether to compress the file (optional).
* @param {number} [uploadChunkSize] - The size of each chunk to upload (optional).
* @returns {PromisedObservable<UploadFileStatus>} - An observable that emits the upload status.
* @throws {Error} - Throws an error if the upload fails at any stage.
*/
export const uploadFileFromFilepath = (
api: AutoDriveApi,
filePath: string,
{ password, compression = true }: UploadFileOptions,
uploadChunkSize?: number,
): PromisedObservable<UploadFileStatus> => {
const name = filePath.split('/').pop()!

return uploadFile(
api,
{
read: () => fs.createReadStream(filePath),
name,
mimeType: mime.lookup(name) || undefined,
size: fs.statSync(filePath).size,
path: filePath,
},
{
password,
compression,
},
uploadChunkSize,
)
}

/**
* Uploads a file to the server with optional encryption and compression.
*
Expand Down Expand Up @@ -237,91 +189,6 @@ export const uploadFile = (
})
}

/**
* Uploads an entire folder to the server.
*
* This function retrieves all files within the specified folder,
* constructs a file tree representation, and initiates the upload
* process. It also handles optional compression and encryption of the files during
* the upload.
*
* If a password is provided, the files will be zipped before uploading.
*
* @param {AutoDriveApi} api - The API instance used to send requests.
* @param {string} folderPath - The path of the folder to be uploaded.
* @param {Object} options - Optional parameters for the upload.
* @param {number} [options.uploadChunkSize] - The size of each chunk to be uploaded.
* @param {string} [options.password] - An optional password for encrypting the files.
* @returns {Promise<PromisedObservable<UploadFileStatus | UploadFolderStatus>>} - A promise that resolves to an observable that tracks the upload progress.
* @throws {Error} - Throws an error if the upload fails at any stage.
*/
export const uploadFolderFromFolderPath = async (
api: AutoDriveApi,
folderPath: string,
{ uploadChunkSize, password }: { uploadChunkSize?: number; password?: string } = {},
): Promise<PromisedObservable<UploadFileStatus | UploadFolderStatus>> => {
const files = await getFiles(folderPath)
const fileTree = constructFromFileSystemEntries(files)

if (password) {
const filesMap = Object.fromEntries(files.map((file) => [file, file]))
const zipBlob = await constructZipBlobFromTreeAndPaths(fileTree, filesMap)
const name = folderPath.split('/').pop()!
return uploadFile(
api,
{
read: () => fileToIterable(zipBlob),
name: `${name}.zip`,
mimeType: 'application/zip',
size: zipBlob.size,
path: name,
},
{
password,
compression: true,
},
)
}

return new PromisedObservable<UploadFolderStatus>(async (subscriber) => {
const folderUpload = await createFolderUpload(api, {
fileTree,
uploadOptions: {
compression: {
algorithm: CompressionAlgorithm.ZLIB,
level: 9,
},
},
})

const genericFiles: GenericFile[] = files.map((file) => ({
read: () => fs.createReadStream(file),
name: file.split('/').pop()!,
mimeType: mime.lookup(file.split('/').pop()!) || undefined,
size: fs.statSync(file).size,
path: file,
}))

const totalSize = genericFiles.reduce((acc, file) => acc + file.size, 0)

let progress = 0
for (const file of genericFiles) {
await uploadFileWithinFolderUpload(api, folderUpload.id, file, uploadChunkSize).forEach((e) =>
subscriber.next({
type: 'folder',
progress: progressToPercentage(progress + e.uploadBytes, totalSize),
}),
)
progress += file.size
}

const result = await completeUpload(api, { uploadId: folderUpload.id })

subscriber.next({ type: 'folder', progress: 100, cid: stringToCid(result.cid) })
subscriber.complete()
})
}

/**
* Uploads an entire folder to the server.
*
Expand Down
File renamed without changes.
2 changes: 2 additions & 0 deletions packages/auto-drive/src/fs/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export * from './utils.js'
export * from './wrappers.js'
79 changes: 79 additions & 0 deletions packages/auto-drive/src/fs/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { WriteStream, createReadStream } from 'fs'
import fs from 'fs/promises'
import JSZip from 'jszip'
import path from 'path'
import { FolderTree, FolderTreeFolder } from '../api/models/folderTree.js'

export const getFiles = async (folderPath: string): Promise<string[]> => {
const stat = await fs.stat(folderPath)

if (stat.isDirectory()) {
const files = await fs.readdir(folderPath)
const promises = files.map((file) => getFiles(path.join(folderPath, file)))
const allFiles = await Promise.all(promises)
return allFiles.flat()
} else {
return [folderPath]
}
}

export const createWriteStreamAdapter = (
nodeWriteStream: WriteStream,
): WritableStream<Uint8Array> => {
return new WritableStream({
write(chunk) {
return new Promise((resolve, reject) => {
nodeWriteStream.write(chunk, (err) => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
},
close() {
nodeWriteStream.end()
},
abort(err) {
nodeWriteStream.destroy(err)
},
})
}

const addFilesFromFilepathsToZip = (
folder: JSZip,
folderNode: FolderTreeFolder,
files: Record<string, string>,
) => {
folderNode.children.forEach((child) => {
if (child.type === 'file') {
const file = files[child.id]
if (typeof file === 'string') {
folder.file(child.name, createReadStream(file))
} else {
folder.file(child.name, file)
}
} else if (child.type === 'folder') {
const subFolder = folder.folder(child.name)
if (!subFolder) {
throw new Error('Failed to create folder in zip')
}
addFilesFromFilepathsToZip(subFolder, child as FolderTreeFolder, files)
}
})
}

export const constructZipFromTreeAndFileSystemPaths = async (
tree: FolderTree,
files: Record<string, string>,
) => {
if (tree.type === 'file') {
throw new Error('Cannot construct zip from file')
}

const zip = new JSZip()
addFilesFromFilepathsToZip(zip, tree as FolderTreeFolder, files)

return zip.generateAsync({ type: 'blob' })
}
Loading

0 comments on commit ade7eac

Please sign in to comment.