feat(download): improved error handling for cURL command (pelias#488)

michaelkirk-pelias · Sep 16, 2021 · 989ac38 · 989ac38
1 parent d01caab
commit 989ac38
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 2 deletions.
diff --git a/utils/download_all.js b/utils/download_all.js
@@ -45,7 +45,28 @@ function downloadBundle(targetDir, config, sourceUrl, callback) {
           const s3Options = config.imports.openaddresses.s3Options || '';
           child_process.exec(`aws s3 cp ${sourceUrl} ${tmpZipFile} --only-show-errors ${s3Options}`, callback);
         } else {
-          child_process.exec(`curl -s -L -X GET --referer ${referer} -o ${tmpZipFile} ${sourceUrl}`, callback);
+          const flags = [
+            '--request GET',                // HTTP GET
+            '--silent',                     // be quiet
+            '--location',                   // follow redirects
+            '--fail',                       // exit with a non-zero code for >=400 responses
+            '--write-out "%{http_code}"',   // print status code to STDOUT
+            `--referer ${referer}`,         // set referer header
+            `--output ${tmpZipFile}`,       // set output filepath
+            '--retry 5',                    // retry this number of times before giving up
+            '--retry-connrefused',          // consider ECONNREFUSED as a transient error
+            '--retry-delay 5'               // sleep this many seconds between retry attempts
+          ].join(' ');
+
+          // the `--fail*` flags cause an error to be returned as the first arg with `error.code`
+          // as the process exit status, the `-w "%{http_code}"` flag writes the HTTP status to STDOUT.
+          child_process.exec(`curl ${flags} ${sourceUrl}`, (error, stdout) => {
+            if (!error) { return callback(); }
+
+            // provide a more user-friendly error message
+            error.message = `cURL request failed, HTTP status: ${stdout}, exit code: ${error.code}`;
+            callback(error);
+          });
         }
       },
       // unzip file into target directory

diff --git a/utils/download_filtered.js b/utils/download_filtered.js
@@ -76,7 +76,28 @@ function downloadSource(targetDir, file, main_callback) {
       // download the zip file into the temp directory
       (callback) => {
         logger.debug(`downloading ${file.url}`);
-        child_process.exec(`curl -s -L -X GET --referer ${referer} -o ${file.zip} ${file.url}`, callback);
+        const flags = [
+          '--request GET',                // HTTP GET
+          '--silent',                     // be quiet
+          '--location',                   // follow redirects
+          '--fail',                       // exit with a non-zero code for >=400 responses
+          '--write-out "%{http_code}"',   // print status code to STDOUT
+          `--referer ${referer}`,         // set referer header
+          `--output ${file.zip}`,         // set output filepath
+          '--retry 5',                    // retry this number of times before giving up
+          '--retry-connrefused',          // consider ECONNREFUSED as a transient error
+          '--retry-delay 5'               // sleep this many seconds between retry attempts
+        ].join(' ');
+
+        // the `--fail*` flags cause an error to be returned as the first arg with `error.code`
+        // as the process exit status, the `-w "%{http_code}"` flag writes the HTTP status to STDOUT.
+        child_process.exec(`curl ${flags} ${file.url}`, (error, stdout) => {
+          if (!error) { return callback(); }
+
+          // provide a more user-friendly error message
+          error.message = `cURL request failed, HTTP status: ${stdout}, exit code: ${error.code}`;
+          callback(error);
+        });
       },
       // unzip file into target directory
       (callback) => {