rebuild test if theres an execution error

README.md

@@ -5,6 +5,7 @@ To use `nessie`, you'll need to install:
 - [rustup](https://doc.rust-lang.org/cargo/getting-started/installation.html) 
 - [nodejs](https://nodejs.org/en/download/)
 - npm (installs with nodejs)
+- [yarn](https://yarnpkg.com/) (you can install this with npm)
 
 Note: `nessie` has only been tested on linux.
 Some of the functionality might require some minor adapting to work for other OSs (e.g., the timeout for executing tests uses the linux `timeout` utility).
@@ -43,6 +44,9 @@ The pipeline of `nessie` is as follows:
   - for each test, collect the output as feedback and use this to build up a database of valid extension points of previous tests
   - output these tests to `testing-dir` (the default value is a directory `test` in the `nessie` root directory)
 
+*Note*: if using the source directory of a module with `lib-src-dir`, the `get_api_specs` script will try and install with the `yarn` package manager if `yarn.lock` exists, and tries to run `npm run build` if the `build` script is present. 
+However, if the module has a custom build setup then you'll need to go in `get_api_specs` and change the build/setup code.
+
 
 #### Example: generating 100 tests for `jsonfile` package
 
@@ -59,6 +63,8 @@ cd node-jsonfile
 npm install
 cd ..
 ```
+`jsonfile` doesn't have a build script and uses `npm` (i.e. not `yarn`) so at this point it's set up.
+
 Then, run `nessie` using the local source of `jsonfile`.
 ```
 cargo run -- --lib-name jsonfile --num-tests 100 --lib-src-dir node-jsonfile
@@ -72,9 +78,6 @@ To find the coverage these generated tests achieve on the local `jsonfile` sourc
 
 First, install `mocha` and `nyc` test library and coverage tools.
 ```
-cd node-jsonfile
-rm test/* # get rid of the existing test suite, or just move it
-cp ../test/*.js test # copy over our generated tests
 nyc mocha test/metatest.js
 ```
 The resulting output displays the coverage of all source files in `node-jsonfile`.

get_api_specs.sh

@@ -30,7 +30,13 @@ if [ -z $lib_src_dir ]; then
 	npm install $lib_name
 else
 	cd $lib_src_dir
-	npm install
+	if [ -f "yarn.lock" ]; then
+		yarn
+	else 
+		npm install
+	fi
+	# note: if there's a custom build for the module, you may need to edit this
+	npm run build --if-present
 	cd $cur_dir/js_tools
 fi
 node api_info.js --lib_name=$lib_name --lib_src_dir=$lib_src_dir --import_code_file=$import_code_file

src/decisions.rs

@@ -271,7 +271,7 @@ impl<'cxt> TestGenDB {
                         .into_string()
                         .unwrap(),
                         Err(_) => self.toy_dir_base.clone() + "/" 
-                                + &self.gen_random_string_val(false).get_string_rep(None, None, false),}
+                                + &self.gen_random_string_val(false).get_string_rep(None, None, false).replace("\"", ""),}
                     + "\""
             }
             _ => {

src/testgen.rs

@@ -17,7 +17,8 @@ pub fn run_testgen_phase<'cxt>(
     testgen_db: &'cxt mut TestGenDB,
     num_tests: i32,
 ) -> Result<(), DFError> {
-    for cur_test_id in 1..=num_tests.try_into().unwrap() {
+    let mut cur_test_id = 1;
+    while cur_test_id <= num_tests.try_into().unwrap() {
         // get a random extension type
         let ext_type: ExtensionType = rand::thread_rng().gen();
 
@@ -29,7 +30,21 @@ pub fn run_testgen_phase<'cxt>(
             consts::FRESH_TEST_IF_CANT_EXTEND,
         )?;
 
-        let test_results = cur_test.execute()?;
+        // if there's an error in a test execution (e.g., timeout), just keep going with the
+        // rest of the tests but don't add this test to the valid pool
+        // HEURISTIC: don't increment the test ID number. Technically this makes the worst
+        // case complexity infinite, but in practice this doesn't happen enough to be a problem.
+        // Revisit if this ends up being a problem with other packages.
+        let test_results = match cur_test.execute() {
+            Ok(res) => res,
+            Err(_) => {
+                println!(
+                    "Execution error in generating test {:?} -- retrying",
+                    cur_test_id
+                );
+                continue;
+            }
+        };
 
         // after running the test, reprint file without all the instrumentation
         // and as part of a mocha test suite
@@ -41,6 +56,8 @@ pub fn run_testgen_phase<'cxt>(
         testgen_db.set_cur_test_index(cur_test_id);
         testgen_db.add_extension_points_for_test(&cur_test, &test_results);
         println!("Test: {:?} of {:?}", cur_test_id, num_tests);
+
+        cur_test_id = cur_test_id + 1;
     }
     // print the runner for the mocha test suite
     write_meta_test(testgen_db.test_dir_path.clone(), num_tests)?;