Merge pull request #1 from Carimus/feature/temp-and-perm-urls

[Feat] Temporary and Permanent URLs

.eslintrc.json

@@ -32,6 +32,7 @@
         "node/prefer-global/url": ["error", "always"],
         "node/no-unsupported-features/es-syntax": "off",
         "@typescript-eslint/explicit-function-return-type": "error",
+        "@typescript-eslint/no-explicit-any": "off",
         "curly": ["error", "all"],
         "max-len": ["error", { "code": 140, "ignoreUrls": true }],
         "no-undefined": "error",

README.md

@@ -33,38 +33,35 @@ available `Disk` methods to perform operations.
 For example (in typescript):
 
 ```typescript
-import { Disk, LocalDisk, S3Disk, DiskDriver } from '@carimus/node-disks';
+import { Disk, LocalDisk, S3Disk, pipeStreams } from '@carimus/node-disks';
 
-const foo: Disk = new LocalDisk({ driver: DiskDriver.Local, root: '/tmp' });
-const bar: Disk = new S3Disk({ driver: DiskDriver.S3, bucket: 'test' });
+const foo: Disk = new LocalDisk({ root: '/tmp' });
+const bar: Disk = new S3Disk({ bucket: 'test' });
 
 // Wrap everything in a self-executing async function.
 (async () => {
     // Write a file to the foo disk
     await foo.write('foo.txt', 'This is a foo file');
 
+    // Log out the contents of foo.txt
+    console.log(await foo.read('foo.txt'));
+
     // Stream the file from the foo disk to the bar disk as bar.txt
     const fooReadStream = await foo.createReadStream('foo.txt');
     const barWriteStream = await bar.createWriteStream('bar.txt');
-
-    // Initiate the piping and wait for it to finish
-    fooReadStream.pipe(barWriteStream);
-    await new Promise((resolve, reject) => {
-        barWriteStream.on('end', () => resolve('end'));
-        barWriteStream.on('finish', () => resolve('finish'));
-        barWriteStream.on('error', (error) => reject(error));
-    });
+    await pipeStreams(fooReadStream, barWriteStream);
 
     // Get a listing of the bar disk contents and store it on the foo disk.
     const s3Listing = await bar.list();
     await foo.write('s3listing.json', JSON.stringify(s3Listing, null, 2));
+
+    // Delete the files we created
+    await foo.delete('foo.txt');
+    await foo.delete('s3listing.json');
+    await bar.delete('bar.txt');
 })();
 ```
 
-**Important Note:** Providing `driver` to the `LocalDisk` and `S3Disk` constructors isn't functionally
-necessary; it's necessary only to satisfy typescript right now until we separate the driver from the options since
-the driver is only a concern of the `DiskManager`.
-
 ### Option B: Use a Disk Manager
 
 `node-disks` also ships with a `DiskManager` that you can provide a single, declarative
@@ -77,16 +74,21 @@ done:
 import { DiskManager, Disk, DiskDriver } from '@carimus/node-disks';
 
 const diskManager = new DiskManager({
+    // `default` MUST be an alias to another disk. All other keys can be aliases (strings) or objects.
     default: 'foo',
     foo: {
         driver: DiskDriver.Local,
-        root: '/tmp',
+        config: {
+            root: '/tmp',
+        },
     },
     bar: {
         driver: DiskDriver.S3,
-        bucket: 'test',
+        config: {
+            bucket: 'test',
+        },
     },
-    baz: 'bar', // You can alias disks as well! `default` above is an alias.
+    baz: 'bar', // Alias the baz disk name to the bar S3 disk.
 });
 
 const foo: Disk = diskManager.getDisk('foo');
@@ -95,44 +97,63 @@ const bar: Disk = diskManager.getDisk('bar');
 // Use `foo` and `bar` not worrying about their implementation like in Option A.
 ```
 
-## Supported Drivers
+## Supported Drivers and Options
+
+### Common
+
+All drivers inherit from and implement the abstract [`Disk`](#disk-abstract-class) class and as such inherit some
+default base functionality and options.
+
+#### Common Disk Options
+
+| Name                   | Type    | Description                                                                                                |
+| ---------------------- | ------- | ---------------------------------------------------------------------------------------------------------- |
+| `url`                  | string  | Optional; If the disk supports URLs, this is the base URL to prepend to paths                              |
+| `temporaryUrlExpires`  | number  | Optional; If the disk supports temp URLs, how many seconds after generation do they expire? Default 1 day. |
+| `temporaryUrlFallback` | boolean | Optional; If the disk supports URLs but not temp URLs, should it by default fallback to permanent URLs?    |
 
 ### Memory
 
-**Driver name:** `'memory'`
+**Driver name:** `'memory'` (`DiskDriver.Memory`)
 
 **`Disk` class:** `MemoryDisk`
 
+**`DiskConfig` interface:** [`DiskConfig`](./src/lib/types.ts#L4) (no extra config outside of common `DiskConfig`)
+
 An in-memory disk whose contents will be forgotten when the node process ends. Each instance of the `MemoryDisk` has
 its own isolated filesystem.
 
-#### Options
+#### Memory Disk Options
 
 Takes no options.
 
 ### Local
 
-**Driver name:** `'local'`
+**Driver name:** `'local'` (`DiskDriver.Local`)
 
 **`Disk` class:** `LocalDisk`
 
+**`DiskConfig` interface:** [`LocalDiskConfig`](./src/drivers/local/types.ts#L3)
+
 A disk that uses the local filesystem.
 
-#### Options:
+#### Local Disk Options
 
 | Name   | Type   | Description                                                                                         |
 | ------ | ------ | --------------------------------------------------------------------------------------------------- |
 | `root` | string | Required; The absolute path to thee directory where files should be stored on the local filesystem. |
 
 ### S3
 
-**Driver name:** `'s3'`
+**Driver name:** `'s3'` (`DiskDriver.S3`)
 
 **`Disk` class:** `S3Disk`
 
+**`DiskConfig` interface:** [`S3DiskConfig`](./src/drivers/s3/types.ts#L4)
+
 A disk that uses a remote AWS S3 bucket.
 
-#### Options:
+#### S3 Disk Options
 
 | Name           | Type   | Description                                                                                                               |
 | -------------- | ------ | ------------------------------------------------------------------------------------------------------------------------- |
@@ -145,12 +166,9 @@ A disk that uses a remote AWS S3 bucket.
 
 ## API
 
-### `Disk`
+### [`Disk`](./src/lib/Disk.ts) Abstract Class
 
-More detailed docs are TODO. [Check out the source](./src/lib/Disk.ts)
-for inline documentation and types.
-
-Important public methods:
+#### Methods
 
 -   `async read(path: string): Promise<Buffer>` to read a file into memory
 -   `async createReadStream(path: string): Promise<Readable>` to obtain a readable stream to a file
@@ -161,18 +179,68 @@ Important public methods:
     directory on the disk.
 -   `getName(): string | null` to get the name of the disk if it was created with one. The `DiskManager` will
     automatically and appropriately set this to the actual resolved name of the disk from the config.
+-   `getUrl(path: string): string | null` to get a URL to an object if the disk supports it and is configured
+    appropriately. If the disk doesn't support URLs, `null` is returned.
+-   `getTemporaryUrl(path: string, expires: number, fallback: boolean): string | null` to get a temporary URL to an
+    object if the disk supports it and is configured appropriately. This method by default won't fallback to
+    generating permanent URLs but can if `fallback` is explicitly passed as `true` or if the disk is configured with
+    `temporaryUrlFallback` set to `true`. If the disk doesn't support temporary URLs and can't fallback to permanent
+    URLs, `null` is returned.
+-   `isTemporaryUrlValid(temporaryUrl: string, against: number | Date = Date.now()): boolean | null` to determine if
+    a temporary URL generated with `getTemporaryUrl` is valid (i.e. unexpired). Will return `null` if the URL can't
+    be determined either way or if the disk does not support temporary URLs.
+
+### [`MemoryDisk`](./src/drivers/memory/MemoryDisk.ts) Class (extends [`Disk`](#disk-abstract-class))
+
+#### Methods
 
-### `DiskManager`
+-   `constructor(config: MemoryDiskConfig, name?: string)` to create the disk.
+    -   See [Memory Disk Options](#memory-disk-options) above for a list of config options you can pass
+        to this disk.
 
-More detailed docs are TODO. [Check out the source](./src/lib/DiskManager.ts)
-for inline documentation and types.
+### [`LocalDisk`](./src/drivers/local/LocalDisk.ts) Class (extends [`Disk`](#disk-abstract-class))
+
+#### Methods
+
+-   `constructor(config: LocalDiskConfig, name?: string)` to create the disk.
+    -   See [`LocalDiskConfig` Options](#local-disk-options) above for a list of config options you can pass
+        to this disk.
+
+### [`S3Disk`](./src/drivers/s3/S3Disk.ts) Class (extends [`Disk`](#disk-abstract-class))
+
+#### Methods
+
+-   `constructor(config: S3DiskConfig, name?: string, s3Client?: AWS.S3)` to create the disk, optionally taking a
+    pre-initialized S3 client instead of creating a new one.
+    -   See [`S3DiskConfig` Options](#s3-disk-options) above for a list of config options you can pass
+        to this disk.
+
+### [`DiskManager`](./src/lib/manager/DiskManager.ts) Class
+
+#### Methods
 
 -   `constructor(config: DiskManagerConfig)` create the disk manager with a map of disk names to
-    their configuration object containing at least a `driver` property and then additionally
-    whatever required config options that specific driver needs. Or a string, which is treated as
-    an alias for another disk.
--   `async geDisk(name: string, options: DiskManagerOptions): Disk` to get a disk by name (allowing
-    for aliases in config).
+    their specification object containing at least a `driver` property and then additionally a `config`
+    property containing whatever required config options that specific driver needs. Or a string, which
+    is treated as an alias for another disk in the `DiskManagerConfig`
+-   `async getDisk(name: string, options: GetDiskOptions): Disk` to get a disk by name. Some disks require runtime
+    options that aren't optimal to pass through config. Those are provided in the second argument here.
+
+### `DiskListingObject` Interface
+
+#### Properties
+
+-   `name`: the name of the file or directory
+-   `type`: the type (file or directory), see [`DiskObjectType`](#diskobjecttype-enum)
+
+### `DiskObjectType` Enum
+
+#### Values
+
+Indicates the type of an object on the disk.
+
+-   `DiskObjectType.File`: a file
+-   `DiskObjectType.Directory`: a directory
 
 ### Utils
 
@@ -199,12 +267,6 @@ This library also exports some helper methods:
     -   Proper errors from bad permissions for `MemoryDisk`/`LocalDisk`
     -   Multiple writes to the same file do truncate
     -   Listings always include directories first
--   [ ] Document the `Disk` API.
--   [ ] Document the `DiskManager` API.
--   [ ] Support `rimraf` for directories.
--   [ ] Support `force` for delete which doesn't to mimic `rm -f` which doesn't fail if the file isn't found.
--   [ ] Separate driver from remaining options so that the `driver` options doesn't have to be passed to `*Disk`
-        constructors.
 -   [ ] Wrap all unknown errors in an `UnknownDiskError` (maybe using `VError`?)
 -   [ ] Ensure that when memfs is used, we always use the posix path module even on a win32 host FS (or otherwise
         verify that on win32, memfs uses win32 paths).
@@ -214,6 +276,9 @@ This library also exports some helper methods:
 -   [ ] Support additional basic filesystem operations:
     -   [ ] Copy
     -   [ ] Move
+    -   [ ] Delete many (globs??)
+    -   [ ] Delete directories (`rimraf`)
+    -   [ ] Support `force` delete option for not failing when file isn't found and does rimraf for directories.
 
 ## Development
 

package.json

@@ -40,10 +40,11 @@
         "fs-extra": "^7.0.1",
         "ramda": "0.25.0",
         "stream-to-array": "^2.3.0",
+        "url-join": "^4.0.0",
         "verror": "^1.10.0"
     },
     "peerDependencies": {
-        "memfs": "^2.15.2"
+        "memfs": "^2.0.0"
     },
     "devDependencies": {
         "@commitlint/cli": "^7.5.2",
@@ -52,6 +53,7 @@
         "@types/jest": "^24.0.11",
         "@types/ramda": "types/npm-ramda#dist",
         "@types/stream-to-array": "^2.3.0",
+        "@types/url-join": "^4.0.0",
         "@types/verror": "^1.10.3",
         "@typescript-eslint/eslint-plugin": "^1.5.0",
         "@typescript-eslint/parser": "^1.5.0",

src/drivers/memory/MemoryDisk.test.ts

@@ -1,9 +1,8 @@
 import { MemoryDisk } from './MemoryDisk';
-import { DiskDriver } from '../..';
-import { streamToBuffer } from '../../lib/utils';
+import { streamToBuffer } from '../..';
 
 test("memory disk's basic methods work", async () => {
-    const disk = new MemoryDisk({ driver: DiskDriver.Memory });
+    const disk = new MemoryDisk();
     expect(await disk.list()).toHaveLength(0);
     await disk.write('/test.txt', 'this is a test');
     expect((await disk.read('/test.txt')).toString('utf8')).toBe(
@@ -15,7 +14,7 @@ test("memory disk's basic methods work", async () => {
 });
 
 test("memory disk's stream methods work", async () => {
-    const disk = new MemoryDisk({ driver: DiskDriver.Memory });
+    const disk = new MemoryDisk();
     expect(await disk.list()).toHaveLength(0);
 
     // Create a write stream, write to it, and wait for it to close.
@@ -36,10 +35,54 @@ test("memory disk's stream methods work", async () => {
 });
 
 test('memory disks use isolated filesystems', async () => {
-    const disk1 = new MemoryDisk({ driver: DiskDriver.Memory });
-    const disk2 = new MemoryDisk({ driver: DiskDriver.Memory });
+    const disk1 = new MemoryDisk();
+    const disk2 = new MemoryDisk();
 
     await disk1.write('/test.txt', 'this is a test');
     expect(await disk1.list()).toHaveLength(1);
     expect(await disk2.list()).toHaveLength(0);
 });
+
+test('memory disk can generate URLs if one is provided in config', async () => {
+    const diskWithoutUrls = new MemoryDisk();
+    const diskWithUrls = new MemoryDisk({ url: 'http://localhost:1234' });
+    const diskWithUrlsAndTempFallback = new MemoryDisk({
+        url: 'http://localhost:1234',
+        temporaryUrlFallback: true,
+    });
+
+    // A memory disk without a url config should return null permanent and temporary URLs
+    expect(diskWithoutUrls.getUrl('test.txt')).toBe(null);
+    expect(diskWithoutUrls.getTemporaryUrl('test.txt')).toBe(null);
+
+    // A disk with a url config should return urls that are the paths appended to the base url
+    expect(diskWithUrls.getUrl('test.txt')).toBe(
+        'http://localhost:1234/test.txt',
+    );
+    expect(diskWithUrls.getUrl('/test.txt')).toBe(
+        'http://localhost:1234/test.txt',
+    );
+    expect(diskWithUrls.getUrl('/abc/1/2/3/test')).toBe(
+        'http://localhost:1234/abc/1/2/3/test',
+    );
+
+    // A memory disk with url configured but no temp fallback enabled by default should return null temp URLs unless
+    // explicitly instructed to fallback.
+    expect(diskWithUrls.getTemporaryUrl('test.txt')).toBe(null);
+    expect(diskWithUrls.getTemporaryUrl('test.txt', 1000, false)).toBe(null);
+    expect(diskWithUrls.getTemporaryUrl('test.txt', 1000, true)).toBe(
+        'http://localhost:1234/test.txt',
+    );
+
+    // A memory disk with url configured and temp fallback enabled should return temp URLs unless explicitly
+    // instructed not to fallback.
+    expect(diskWithUrlsAndTempFallback.getTemporaryUrl('test.txt')).toBe(
+        'http://localhost:1234/test.txt',
+    );
+    expect(
+        diskWithUrlsAndTempFallback.getTemporaryUrl('test.txt', 1000, false),
+    ).toBe(null);
+    expect(
+        diskWithUrlsAndTempFallback.getTemporaryUrl('test.txt', 1000, true),
+    ).toBe('http://localhost:1234/test.txt');
+});