Builder records both test pass/fail results and grades

Partial first step toward #52.

src/builder.js

@@ -226,6 +226,10 @@ exports.build = function(spec, progressCallback, resultCallback) {
     grade: [ 'hidden', function(results, next) {
       let result = { json: { public: results.public.result, hidden: results.hidden.result } };
       grader.grade(spec, results.builddir, result, buildOutputBase(spec, results.builder, 'grade'), next);
+    } ],
+    passfail: [ 'hidden', function(results, next) {
+      let result = { json: { public: results.public.result, hidden: results.hidden.result } };
+      exports.passfail(spec, results.builddir, result, buildOutputBase(spec, results.builder, 'passfail'), next);
     } ]
   }, function(err, results) {
     log.info('build complete');
@@ -246,6 +250,8 @@ exports.build = function(spec, progressCallback, resultCallback) {
     if (results.grade) {
       results.grade = [ results.grade.score, results.grade.outof ];
     }
+    // and don't store pass/fail
+    delete results.passfail;
 
     fs.writeFile(buildResultFile(spec), JSON.stringify(results), function(fserr) {
       if (fserr) { log.error({ err: fserr, results }, 'error writing results'); }
@@ -260,7 +266,32 @@ exports.build = function(spec, progressCallback, resultCallback) {
       }
     });
   });
-}
+};
+
+exports.passfail = function(spec, builddir, build, output, callback) {
+  let passfail = { spec, public: {}, hidden: {} };
+  for (let [ report, testsuites ] of [
+    [ passfail.public, build.json.public && build.json.public.testsuites || [] ],
+    [ passfail.hidden, build.json.hidden && build.json.hidden.testsuites || [] ],
+  ]) {
+    for (let suite of testsuites) {
+      let pkg = report[suite.package] || (report[suite.package] = {});
+      let cls = pkg[suite.name] = Object.assign({}, suite);
+      [ 'hostname' ].forEach(reject => delete cls[reject]);
+      cls.testcases = {};
+      for (let testcase of suite.testcases) {
+        let test = cls.testcases[testcase.name] = Object.assign({}, testcase);
+        [ 'classname', 'name', 'time' ].forEach(reject => delete test[reject]);
+      }
+    }
+  }
+  fs.writeFile(output + '.json', JSON.stringify(passfail), function(err) {
+    if (err) {
+      err.dmesg = 'error writing pass/fail report';
+    }
+    callback(err, passfail);
+  });
+};
 
 // command-line build
 if (require.main === module) {

test/builder.js

@@ -236,6 +236,7 @@ describe('builder', function() {
         results.public.should.be.type('boolean');
         results.hidden.should.be.type('boolean');
         results.grade.should.be.an.instanceof(Array);
+        should.not.exist(results.passfail);
         results.started.should.be.within(start, finish);
         results.finished.should.be.within(start, finish);
         done(err);
@@ -287,6 +288,18 @@ describe('builder', function() {
         });
       });
     });
+    it('should record test results', function(done) {
+      builder.build(spec, function() { }, function(err, results) {
+        fs.readFile(path.join(resultdir, results.builder, 'passfail.json'), { encoding: 'utf8' }, function(fserr, data) {
+          let json = JSON.parse(data);
+          json.public['']['FakePublic'].testcases['publicPass'].should.eql({});
+          json.public['']['FakePublic'].testcases['publicFail'].should.eql({ failure: 'Fake failure!' });
+          json.hidden['']['FakeHidden'].testcases['hiddenPass'].should.eql({});
+          json.hidden['']['FakeHidden'].testcases['hiddenError'].should.eql({ error: 'Fake error!' });
+          done(err || fserr);
+        });
+      });
+    });
     it('should schedule build directory for deletion', function(done) {
       sandbox.useFakeTimers('setTimeout');
       builder.build(spec, function() { }, function(err, results) {

test/fixtures/builder/build/should-record-test-results/build/build.xml

@@ -0,0 +1,45 @@
+<project name="abcplayer">
+
+  <target name="compile">
+  </target>
+
+  <target name="public">
+    <echoxml file="TEST-0-fakery.xml">
+      <testsuite name="FakePublic" tests="2" failures="1" errors="0">
+        <properties>
+          <property name="didit.desc" value="Fake public tests"/>
+        </properties>
+        <testcase name="publicPass"/>
+        <testcase name="publicFail">
+          <failure type="Fake">Fake failure!</failure>
+        </testcase>
+      </testsuite>
+    </echoxml>
+
+    <junitreport>
+      <fileset dir="." includes="TEST-*.xml"/>
+    </junitreport>
+  </target>
+
+  <target name="hidden">
+    <echoxml file="TEST-0-fakery.xml">
+      <testsuite name="FakeHidden" tests="2" failures="0" errors="1">
+        <properties>
+          <property name="didit.desc" value="Fake hidden tests"/>
+        </properties>
+        <testcase name="hiddenPass"/>
+        <testcase name="hiddenError">
+          <error type="Fake">Fake error!</error>
+        </testcase>
+        <testcase name="hiddenSkip">
+          <skipped/>
+        </testcase>
+      </testsuite>
+    </echoxml>
+
+    <junitreport>
+      <fileset dir="." includes="TEST-*.xml"/>
+    </junitreport>
+  </target>
+
+</project>