Merge pull request #2 from Chaphasilor/dev

New options, some fixes

.gitignore

@@ -102,3 +102,6 @@ dist
 
 # TernJS port file
 .tern-port
+
+# ignore for now
+test.js

README.md

@@ -10,8 +10,11 @@ Open Directory Downloader is a CLI tool for indexing so-called *Open Directories
 
 | Wrapper Version | Supported OpenDirectoryDownloader Versions | Included Version |
 | --- | --- | --- |
+| 1.1.0 | >1.9.3.5 | 1.9.3.6 |
 | 1.0.0 | >1.9.3.1 | 1.9.3.3 |
 
+Some intermediary releases might not be fully supported. It is recommended to use the versions that are included by default.
+
 ## Installation
 
 ```shell
@@ -21,23 +24,31 @@ npm i open-directory-downloader
 Upon installation, the latest supported OpenDirectoryDownloader binary will automatically be downloaded from GitHub.  
 If this should fail, a local installation can be used instead (see below).
 
+For changelogs, please look at the [GitHub release page](https://github.com/Chaphasilor/open-directory-downloader/releases).
+
 ## Class OpenDirectoryDownloader
 
 ### OpenDirectoryDownloader([executablePath, outputDirectory])
 
 - `executablePath` (`String`) (optional): The full path to the OpenDirectoryDownloader executable.  
   Allows you to use your own installation of OpenDirectoryDownloader.
-- `outputDirectory` (`String`) (optional): The full path to the directory where OpenDirectoryDownloader save its scan files.  
+- `outputDirectory` (`String`) (optional): The full path to the directory where OpenDirectoryDownloader saves its scan files.  
   Required if `executablePath` is set.
 - Returns: An instance of `OpenDirectoryDownloader`
 
 *Constructor*
 
-### OpenDirectoryDownloader.scanUrl(url[, keepJson = false])  
+### OpenDirectoryDownloader.scanUrl(url[, options])  
 
-- `url` (`String`) (required): The URL to scan/index.
-- `keepJson` (`Boolean`) (optional): If set to `true`, the JSON-file generated by the OpenDirectoryDownloader binary won't be deleted after the scan.
-- Returns: Promise<[ScanResult](#scanresult)>
+- `url` (`String` or `URL` Object) (required): The URL to scan/index.
+- `options` (`Object`) (optional): Additional options for the scan
+  - `outputFile` (`String`) (optional, defaults to the escaped URL): The name of the output file(s).
+  - `keepJsonFile` (`Boolean`) (optional, default is `false`): Keep the JSON-file generated by the OpenDirectoryDownloader binary after the scan?
+  - `keepUrlFile` (`Boolean`) (optional, default is `false`): Keep the text-file generated by the OpenDirectoryDownloader binary after the scan?
+  - `performSpeedtest` (`Boolean`) (optional, default is `false`): Perform a speed test after the scan is done? (usually takes a few seconds)
+  - `uploadUrlFile` (`Boolean`) (optional, default is `false`): Automatically upload the file containing all the found URLs to GoFile?
+- Returns: Promise<Resolves to `ScanResult` | Rejects to `Array<Error[,ScanResult]>`>  
+  If the promise rejects, it will return an array where the first element is always an `Error` object and there might also be a second element, which is a `ScanResult` but without the `ScanResult.scan` property.
 
 *Initiates the scan of the provided URL*
 
@@ -46,8 +57,10 @@ If this should fail, a local installation can be used instead (see below).
 - `scannedUrl` The URL that was scanned
 - `scan` The parsed JSON-Object created by the OpenDirectoryDownloader binary.  
   Can be very large depending on the size of the Open Directory.
-- `scanFile` The full path to the created JSON-file.  
-  Only available if `keepJson` was set to `true`.
+- `jsonFile` The full path to the created JSON-file.  
+  Only available if `keepJsonFile` was set to `true`.
+- `urlFile` The full path to the created text-file.  
+  Only available if `keepUrlFile` was set to `true`.
 - `reddit` The markdown-formatted table containing the stats for the Open Directory.  
   Does not include the credits.
 - `credits` The markdown signature containing a link to KoalaBear84/OpenDirectoryDownloader
@@ -67,7 +80,8 @@ odd.scanUrl(url, true)
   // {
   //   scannedUrl: `https://example.com/files`,
   //   scan: {...},
-  //   scanFile: `/path/to/scan/file`,
+  //   jsonFile: `/full/path/to/scan/file`,
+  //   urlFile: `/full/path/to/url/file`,
   //   reddit: `
   //     |**Url:** https://example.com/files/||[Urls file](https://gofile.io/?c=XXXXX)|
   //     |:-|-:|-:|

config.js

@@ -1,8 +1,8 @@
 module.exports = {
   GitHubReleasesUrl: `https://api.github.com/repos/KoalaBear84/OpenDirectoryDownloader/releases`,
   OpenDirectoryDownloaderVersion: {
-    version: `v1.9.3.3`,
-    releaseId: `36663866`
+    version: `v1.9.3.6`,
+    releaseId: `36853483`
   },
   OpenDirectoryDownloaderFolder: `${__dirname}/ODD`,
   OpenDirectoryDownloaderOutputFolder: `${__dirname}/ODD/Scans`,

open-directory-downloader.js

@@ -5,33 +5,63 @@ const CONFIG = require(`./config`)
 
 module.exports = class OpenDirectoryDownloader {
 
-  constructor(executablePath, outputDirectory) {
+  constructor(executablePath = CONFIG.OpenDirectoryDownloaderPath, outputDirectory = CONFIG.OpenDirectoryDownloaderOutputFolder) {
 
-    this.executable = executablePath || CONFIG.OpenDirectoryDownloaderPath;
-    this.outputDir = outputDirectory || CONFIG.OpenDirectoryDownloaderOutputFolder;
+    this.executable = executablePath;
+    this.outputDir = outputDirectory;
 
   }
 
-  scanUrl(url, keepJson = false) {
+  /**
+   * Initiates the scan of the provided URL
+   * @param {URL|String} url The URL to scan. Has to be a supported Open Directory
+   * @param {Object} [options] Additional options for the scan
+   * @param {String} [options.outputFile] The name of the output file(s). Defaults to the (safely formatted) URL.
+   * @param {Boolean} [options.keepJsonFile=false] Keep the JSON file created by the OpenDirectoryDownloader binary after the scan is done?
+   * @param {Boolean} [options.keepUrlFile=false] Keep the text file created by the OpenDirectoryDownloader binary after the scan is done?
+   * @param {Boolean} [options.performSpeedtest=false] Perform a speed test after the scan is done? (usually takes a few seconds)
+   * @param {Boolean} [options.uploadUrlFile=false] Automatically upload the file containing all the found URLs to GoFile?
+   */
+  scanUrl(url, options = {}) {
     return new Promise((resolve, reject) => {
+
+      options.keepJsonFile = options.keepJsonFile || false
+      options.keepUrlFile = options.keepUrlFile || false
+      options.performSpeedtest = options.performSpeedtest || false
+      options.uploadUrlFile = options.uploadUrlFile || false
 
-      const oddProcess = spawn(this.executable, [`-u ${url}`, `--quit`, `--json`, `--upload-urls`, `--speedtest`]);
+      // both String and URL implement the toString() method, so just use that instead of detecting the type
+      let processArgs = [`-u ${url}`, `--quit`, `--json`, ]
+      if (options.performSpeedtest) {
+        processArgs.push(`--upload-urls`)
+      }
+      if (options.performSpeedtest) {
+        processArgs.push(`--speedtest`)
+      }
+      if (options.outputFile && options.outputFile.length > 0) {
+        processArgs.push(`--output-file`)
+        processArgs.push(options.outputFile)
+      }
+      const oddProcess = spawn(this.executable, processArgs);
 
       let output = ``;
       let error = ``;
+
+      oddProcess.stdout.setEncoding(`utf8`)
+      oddProcess.stderr.setEncoding(`utf8`)
 
       oddProcess.stdout.on('data', (data) => {
         // console.log(`stdout: ${data}`);
         output += data;
       });
 
       oddProcess.stderr.on('data', (data) => {
-        console.warn(`Error from ODD: ${data}`);
+        // console.warn(`Error from ODD: ${data}`);
         error += data;
       });
 
       oddProcess.on(`error`, (err) => {
-        return reject(err);
+        return reject([err]);
       })
 
       oddProcess.on('close', (code) => {
@@ -40,14 +70,14 @@ module.exports = class OpenDirectoryDownloader {
           reject(new Error(`ODD exited with code ${code}: ${error}`));
         }
 
-        if (output.split(`Finshed indexing`).length <= 1) {
+        if (output.split(`Finished indexing`).length <= 1) {
           return reject(new Error(`ODD never finished indexing!`));
         }
 
-        // const finalResults = output.split(`Finshed indexing`)[1];
-        const finalResults = output.split(`Saving URL list to file...`)[1];
+        // const finalResults = output.split(`Finished indexing`)[1];
+        const finalResults = output.split(`Saving URL list to file..`)[1];
 
-        console.log(`finalResults:`, finalResults);
+        // console.log(`finalResults:`, finalResults);
 
         const redditOutputStartString = `|`;
         const redditOutputEndString = `^(Created by [KoalaBear84's OpenDirectory Indexer](https://github.com/KoalaBear84/OpenDirectoryDownloader/))`;
@@ -57,41 +87,60 @@ module.exports = class OpenDirectoryDownloader {
 
         let sessionRegexResults = finalResults.match(/Saved\ session:\ (.*)/);
         if (!sessionRegexResults || sessionRegexResults.length <= 1) {
-          return reject(new Error(`JSON session file not found!`));
+          return reject([new Error(`JSON session file not found!`)]);
         }
         let jsonFile = sessionRegexResults[1]; // get first capturing group. /g modifier has to be missing!
 
         let urlListRegexResults = finalResults.match(/Saved URL list to file:\ (.*)/);
         if (!urlListRegexResults || urlListRegexResults.length <= 1) {
-          return reject(new Error(`URL list file not found!`));
+          return reject([new Error(`URL list file not found!`)]);
         }
         let urlFile = urlListRegexResults[1];
-        fs.unlinkSync(`${this.outputDir}/${urlFile}`);
+        if (!options.keepUrlFile) {
+          try {
+            fs.unlinkSync(urlFile);
+          } catch (err) {
+            // console.error(`Failed to delete URL list file:`, err)
+            // fail silently in production, because this isn't a critical error
+            // could be changed once https://github.com/KoalaBear84/OpenDirectoryDownloader/issues/64 is fixed
+          }
+        }
 
         let results;
         try {
 
-          results = JSON.parse(fs.readFileSync(`${this.outputDir}/${jsonFile}`));
-          if (!keepJson) {
-            fs.unlinkSync(`${this.outputDir}/${jsonFile}`);
+          results = JSON.parse(fs.readFileSync(jsonFile));
+          if (!options.keepJsonFile) {
+            try {
+              fs.unlinkSync(jsonFile);
+            } catch (err) {
+              // console.error(`Failed to delete JSON file:`, err)
+              // fail silently in production, because this isn't a critical error
+              // could be changed once https://github.com/KoalaBear84/OpenDirectoryDownloader/issues/64 is fixed
+            }
           }
 
         } catch (err) {
-          console.error(`err:`, err);
-
-          resolve({
-            scannedUrl: url,
-            scanFile: `${this.outputDir}/${jsonFile}`,
-            reddit: redditOutput,
-            credits,
-          })
 
+          // console.error(`Error while reading in the scan results:`, err);
+          reject([
+            new Error(`Error while reading in the scan results`),
+            {
+              scannedUrl: url.toString(),
+              jsonFile: options.keepJsonFile ? jsonFile :  undefined,
+              urlFile: options.keepUrlFile ? urlFile :  undefined,
+              reddit: redditOutput,
+              credits,
+            },
+          ])
+
         }
 
         resolve({
-          scannedUrl: url,
+          scannedUrl: url.toString(),
           scan: results,
-          scanFile: keepJson ? `${this.outputDir}/${jsonFile}` :  undefined,
+          jsonFile: options.keepJsonFile ? jsonFile :  undefined,
+          urlFile: options.keepUrlFile ? urlFile :  undefined,
           reddit: redditOutput,
           credits,
         })

package.json

@@ -1,6 +1,6 @@
 {
   "name": "open-directory-downloader",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "A NodeJS wrapper around KoalaBear84/OpenDirectoryDownloader",
   "main": "open-directory-downloader.js",
   "scripts": {