Merge pull request #211 from codaco/feature/filter-query-backend

Implement filter & query backend

.eslintignore

@@ -1,2 +1,3 @@
 src/renderer/ui
 src/main/utils/shared-api/node_modules
+src/main/utils/network-query

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "src/main/utils/shared-api"]
 	path = src/main/utils/shared-api
 	url = https://github.com/codaco/secure-comms-api.git
+[submodule "src/main/utils/network-query"]
+	path = src/main/utils/network-query
+	url = https://github.com/codaco/networkQuery.git

jsdoc.conf.json

@@ -1,7 +1,7 @@
 {
   "source": {
     "includePattern": ".+\\.js$",
-    "excludePattern": "__tests__"
+    "excludePattern": "__tests__|utils\/network-query"
   },
   "plugins": ["plugins/markdown"]
 }

package.json

@@ -173,6 +173,7 @@
     "setupTestFrameworkScriptFile": "<rootDir>/config/jest/setupTestFramework.js",
     "testPathIgnorePatterns": [
       "<rootDir>[/\\\\](build|docs|node_modules|scripts|release-builds)[/\\\\]",
+      "<rootDir>/src/main/utils/network-query",
       "<rootDir>/src/renderer/ui"
     ],
     "testEnvironment": "node",

src/main/data-managers/ExportManager.js

@@ -8,8 +8,12 @@ const SessionDB = require('./SessionDB');
 const { archive } = require('../utils/archive');
 const { writeFile } = require('../utils/promised-fs');
 const { RequestError, ErrorMessages } = require('../errors/RequestError');
-const { unionOfNetworks } = require('../utils/formatters/network');
 const { makeTempDir, removeTempDir } = require('../utils/formatters/dir');
+const {
+  filterNetworkEntities,
+  filterNetworksWithQuery,
+  unionOfNetworks,
+} = require('../utils/formatters/network');
 const {
   formats,
   formatsAreValid,
@@ -104,6 +108,11 @@ class ExportManager {
    * @param {Object} options.filter before formatting, apply this filter to each network (if
    *                                `exportNetworkUnion` is false) or the merged network (if
    *                                `exportNetworkUnion` is true)
+   * @param {Object} networkFilter a user-supplied filter configuration which will be run on each
+   *                               member of each network, to filter out unwanted nodes & edges.
+   * @param {Object} networkInclusionQuery a user-supplied query configuration which will be run on
+   *                                       each interview's network to determine if it should be
+   *                                       exported
    * @param {boolean} options.useDirectedEdges used by the formatters. May be removed in the future
    *                                           if information is encapsulated in network.
    * @return {Promise} A `promise` that resloves to a filepath (string). The promise is decorated
@@ -113,7 +122,14 @@ class ExportManager {
    */
   createExportFile(
     protocol,
-    { destinationFilepath, exportFormats, exportNetworkUnion, useDirectedEdges/* , filter */ } = {},
+    {
+      destinationFilepath,
+      exportFormats,
+      exportNetworkUnion,
+      entityFilter,
+      networkInclusionQuery,
+      useDirectedEdges,
+    } = {},
   ) {
     if (!protocol) {
       return Promise.reject(new RequestError(ErrorMessages.NotFound));
@@ -131,6 +147,14 @@ class ExportManager {
       variableRegistry: protocol.variableRegistry,
     };
 
+    // Export flow:
+    // 1. fetch all networks produced for this protocol's interviews
+    // 2. optional: run user query to select networks are exported
+    // 3. optional: merge all networks into a single union for export
+    // 4. optional: filter each network based on user-supplied rules
+    // 5. [TODO: #199] optional: insert ego with edges into each network
+    // 6. export each format for each network
+    // 7. save ZIP file to requested location
     const exportPromise = makeTempDir()
       .then((dir) => {
         tmpDir = dir;
@@ -141,12 +165,14 @@ class ExportManager {
       .then(() => this.sessionDB.findAll(protocol._id, null, null))
       // TODO: may want to preserve some session metadata for naming?
       .then(sessions => sessions.map(session => session.data))
+      .then(allNetworks => filterNetworksWithQuery(allNetworks, networkInclusionQuery))
       .then(networks => (exportNetworkUnion ? [unionOfNetworks(networks)] : networks))
-      .then((networks) => {
+      .then(networks => filterNetworkEntities(networks, entityFilter))
+      .then((filteredNetworks) => {
         // TODO: evaluate & test. It'll be easier to track progress when run concurrently,
         // but this may run into memory issues.
         promisedExports = flatten(
-          networks.map((network, i) =>
+          filteredNetworks.map((network, i) =>
             exportFormats.map(format =>
               exportFile(`${i + 1}`, format, tmpDir, network, exportOpts))));
         return Promise.all(promisedExports);

src/main/utils/formatters/__tests__/network-test.js

@@ -1,7 +1,45 @@
 /* eslint-env jest */
-const { getNodeAttributes, nodeAttributesProperty, unionOfNetworks } = require('../network');
+const {
+  filterNetworkEntities,
+  filterNetworksWithQuery,
+  getNodeAttributes,
+  nodeAttributesProperty,
+  unionOfNetworks,
+} = require('../network');
 
 describe('network format helpers', () => {
+  describe('filtering & querying', () => {
+    const ruleConfig = {
+      rules: [
+        {
+          type: 'alter',
+          options: { type: 'person', operator: 'EXACTLY', attribute: 'age', value: 20 },
+          assert: { operator: 'GREATER_THAN', value: 0 }, // for query only
+        },
+      ],
+      join: 'OR',
+    };
+
+    describe('filterNetworksWithQuery()', () => {
+      it('includes nodes from attribute query', () => {
+        const a = { nodes: [{ type: 'person', [nodeAttributesProperty]: { age: 20 } }], edges: [] };
+        const b = { nodes: [{ type: 'person', [nodeAttributesProperty]: { age: 20 } }], edges: [] };
+        const c = { nodes: [{ type: 'person', [nodeAttributesProperty]: { age: 21 } }], edges: [] };
+        const networks = [a, b, c];
+        expect(filterNetworksWithQuery(networks, ruleConfig)).toEqual([a, b]);
+      });
+    });
+
+    describe('filterNetworkEntities()', () => {
+      it('includes nodes matching attributes', () => {
+        const alice = { type: 'person', [nodeAttributesProperty]: { age: 20 } };
+        const bob = { type: 'person', [nodeAttributesProperty]: { age: 21 } };
+        const networks = [{ nodes: [alice, bob], edges: [] }];
+        expect(filterNetworkEntities(networks, ruleConfig)[0].nodes).toEqual([alice]);
+      });
+    });
+  });
+
   describe('unionOfNetworks', () => {
     it('joins nodes of two networks', () => {
       const a = { nodes: [{ id: 1 }], edges: [] };

src/main/utils/formatters/network.js

@@ -1,3 +1,6 @@
+const getQuery = require('../network-query/query').default;
+const getFilter = require('../network-query/filter').default;
+
 // TODO: share with other places this is defined
 const nodePrimaryKeyProperty = '_uid';
 
@@ -12,7 +15,33 @@ const unionOfNetworks = networks =>
     return union;
   }, { nodes: [], edges: [] });
 
+/**
+ * Run the query on each network; filter for those which meet the criteria (i.e., where the query
+ * evaluates to `true`).
+ * @param  {Object[]} networks An array of NC networks
+ * @param  {Object} inclusionQueryConfig a query definition with asserting rules
+ * @return {Object[]} a subset of the networks
+ */
+const filterNetworksWithQuery = (networks, inclusionQueryConfig) =>
+  (inclusionQueryConfig ? networks.filter(getQuery(inclusionQueryConfig)) : networks);
+
+/**
+ * Filter each network based on the filter config.
+ * @param  {Object[]} networks An array of NC networks
+ * @param  {Object} filterConfig a filter definition with rules
+ * @return {Object[]} a copy of `networks`, each possibly containing a subset of the original
+ */
+const filterNetworkEntities = (networks, filterConfig) => {
+  if (!filterConfig || !filterConfig.rules || !filterConfig.rules.length) {
+    return networks;
+  }
+  const filter = getFilter(filterConfig);
+  return networks.map(network => filter(network));
+};
+
 module.exports = {
+  filterNetworkEntities,
+  filterNetworksWithQuery,
   getNodeAttributes,
   nodeAttributesProperty,
   nodePrimaryKeyProperty,

src/renderer/components/Filter/Rule/AddButton.js

@@ -23,6 +23,7 @@ class AddButton extends PureComponent {
       >
         <button
           className="rule-add-button__open"
+          type="button"
           disabled
         >
           +

src/renderer/components/Filter/Rule/AlterRule.js

@@ -117,7 +117,7 @@ class AlterRule extends PureComponent {
             </div>
           )}
         </div>
-        <button className="rule__delete" onClick={() => onDeleteRule(id)} />
+        <button type="button" className="rule__delete" onClick={() => onDeleteRule(id)} />
       </div>
     );
   }

src/renderer/components/Filter/Rule/EdgeRule.js

@@ -117,7 +117,7 @@ class EdgeRule extends PureComponent {
             </div>
           )}
         </div>
-        <button className="rule__delete" onClick={() => onDeleteRule(id)} />
+        <button type="button" className="rule__delete" onClick={() => onDeleteRule(id)} />
       </div>
     );
   }

src/renderer/components/Filter/Rule/EgoRule.js

@@ -104,7 +104,7 @@ class EgoRule extends PureComponent {
           </div>
         }
         { !hasPersonType && <div>No &quot;Person&quot; node type found!</div> }
-        <button className="rule__delete" onClick={() => onDeleteRule(id)} />
+        <button type="button" className="rule__delete" onClick={() => onDeleteRule(id)} />
       </div>
     );
   }

src/renderer/containers/ExportScreen.js

@@ -35,13 +35,13 @@ class ExportScreen extends Component {
       exportFormat: 'graphml',
       exportNetworkUnion: false,
       csvTypes: new Set(Object.keys(availableCsvTypes)),
-      filter: defaultFilter,
+      entityFilter: defaultFilter,
       useDirectedEdges: true,
     };
   }
 
-  handleFilterChange = (filter) => {
-    this.setState({ filter });
+  handleFilterChange = (entityFilter) => {
+    this.setState({ entityFilter });
   }
 
   handleFormatChange = (evt) => {
@@ -114,7 +114,7 @@ class ExportScreen extends Component {
       exportFormat,
       exportNetworkUnion,
       csvTypes,
-      filter,
+      entityFilter,
       useDirectedEdges,
     } = this.state;
 
@@ -123,7 +123,7 @@ class ExportScreen extends Component {
         exportFormats: (exportFormat === 'csv' && [...csvTypes]) || [exportFormat],
         exportNetworkUnion,
         destinationFilepath,
-        filter,
+        entityFilter,
         useDirectedEdges,
       })
       .then(() => showConfirmation('Export complete'))
@@ -132,7 +132,7 @@ class ExportScreen extends Component {
   }
 
   render() {
-    const { protocol, protocolsHaveLoaded } = this.props;
+    const { protocol, protocolsHaveLoaded, variableRegistry } = this.props;
 
     if (protocolsHaveLoaded && !protocol) { // This protocol doesn't exist
       return <Redirect to="/" />;
@@ -248,11 +248,11 @@ class ExportScreen extends Component {
         </div>
         <div className="export__section">
           <h3>Filtering</h3>
-          <p>Optionally filter the network(s) before export.</p>
+          <p>Include nodes and edges that meet the following criteria:</p>
           <Filter
-            filter={this.state.filter}
+            filter={this.state.entityFilter}
             onChange={this.handleFilterChange}
-            variableRegistry={protocol.variableRegistry}
+            variableRegistry={variableRegistry}
           />
         </div>
         <Button type="submit" disabled={exportInProgress}>Export</Button>
@@ -267,16 +267,19 @@ ExportScreen.propTypes = {
   protocolsHaveLoaded: PropTypes.bool.isRequired,
   showConfirmation: PropTypes.func.isRequired,
   showError: PropTypes.func.isRequired,
+  variableRegistry: PropTypes.object,
 };
 
 ExportScreen.defaultProps = {
   apiClient: null,
   protocol: null,
+  variableRegistry: null,
 };
 
 const mapStateToProps = (state, ownProps) => ({
   protocolsHaveLoaded: selectors.protocolsHaveLoaded(state),
   protocol: selectors.currentProtocol(state, ownProps),
+  variableRegistry: selectors.transposedRegistry(state, ownProps),
 });
 
 const mapDispatchToProps = dispatch => ({

src/renderer/containers/__tests__/ExportScreen-test.js

@@ -107,7 +107,7 @@ describe('<ExportScreen />', () => {
       const filterInstance = subject.find('Connect(FilterGroup)');
       const mockFilter = { join: null, rules: [{ mock: true }] };
       filterInstance.simulate('change', mockFilter);
-      expect(subject.state('filter')).toEqual(mockFilter);
+      expect(subject.state('entityFilter')).toEqual(mockFilter);
     });
   });
 });

src/renderer/ducks/modules/__tests__/protocols-test.js

@@ -1,5 +1,5 @@
 /* eslint-env jest */
-import reducer, { actionCreators, actionTypes } from '../protocols';
+import reducer, { actionCreators, actionTypes, selectors } from '../protocols';
 import AdminApiClient from '../../../utils/adminApiClient';
 
 jest.mock('../../../utils/adminApiClient');
@@ -95,4 +95,42 @@ describe('the protocols module', () => {
       });
     });
   });
+
+  describe('selectors', () => {
+    const { currentProtocol, protocolsHaveLoaded, transposedRegistry } = selectors;
+
+    describe('currentProtocol', () => {
+      it('returns the current protocol object', () => {
+        const state = { protocols: [{ id: '1' }] };
+        const props = { match: { params: { id: '1' } } };
+        expect(currentProtocol(state, props)).toEqual(state.protocols[0]);
+      });
+    });
+
+    describe('protocolsHaveLoaded', () => {
+      it('indicates protocols have loaded', () => {
+        expect(protocolsHaveLoaded({ protocols: null })).toBe(false);
+        expect(protocolsHaveLoaded({ protocols: [{ id: '1' }] })).toBe(true);
+      });
+    });
+
+    describe('transposedRegistry', () => {
+      it('returns a modified registry', () => {
+        const variableRegistry = { node: { 'node-type-id': { name: 'person', variables: {} } } };
+        const state = { protocols: [{ id: '1', variableRegistry }] };
+        const props = { match: { params: { id: '1' } } };
+        const transposed = transposedRegistry(state, props);
+        expect(transposed).toHaveProperty('node');
+        expect(transposed.node).toHaveProperty('person');
+      });
+
+      it('does not require edge variables', () => {
+        const variableRegistry = { node: { 'node-type-id': { name: 'person', variables: {} } }, edge: { 'edge-type-id': {} } };
+        const state = { protocols: [{ id: '1', variableRegistry }] };
+        const props = { match: { params: { id: '1' } } };
+        const transposed = transposedRegistry(state, props);
+        expect(transposed.edge).toEqual({});
+      });
+    });
+  });
 });