From eb8e867a8cbaa011a03088c0c0b4d21fa5041f78 Mon Sep 17 00:00:00 2001 From: Bryan Fox Date: Wed, 12 Dec 2018 12:12:18 -0500 Subject: [PATCH 1/6] Add networkQuery submodule Ignore networkQuery in preflight tasks --- .eslintignore | 1 + .gitmodules | 3 +++ jsdoc.conf.json | 2 +- package.json | 1 + src/main/utils/network-query | 1 + 5 files changed, 7 insertions(+), 1 deletion(-) create mode 160000 src/main/utils/network-query diff --git a/.eslintignore b/.eslintignore index 666d75046..3772eac0b 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,2 +1,3 @@ src/renderer/ui src/main/utils/shared-api/node_modules +src/main/utils/network-query diff --git a/.gitmodules b/.gitmodules index 4860685c1..899665caa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "src/main/utils/shared-api"] path = src/main/utils/shared-api url = https://github.com/codaco/secure-comms-api.git +[submodule "src/main/utils/network-query"] + path = src/main/utils/network-query + url = https://github.com/codaco/networkQuery.git diff --git a/jsdoc.conf.json b/jsdoc.conf.json index 180e915e7..4828c5f27 100644 --- a/jsdoc.conf.json +++ b/jsdoc.conf.json @@ -1,7 +1,7 @@ { "source": { "includePattern": ".+\\.js$", - "excludePattern": "__tests__" + "excludePattern": "__tests__|utils\/network-query" }, "plugins": ["plugins/markdown"] } diff --git a/package.json b/package.json index 71f00a762..8236c363c 100644 --- a/package.json +++ b/package.json @@ -173,6 +173,7 @@ "setupTestFrameworkScriptFile": "/config/jest/setupTestFramework.js", "testPathIgnorePatterns": [ "[/\\\\](build|docs|node_modules|scripts|release-builds)[/\\\\]", + "/src/main/utils/network-query", "/src/renderer/ui" ], "testEnvironment": "node", diff --git a/src/main/utils/network-query b/src/main/utils/network-query new file mode 160000 index 000000000..0c6b34cef --- /dev/null +++ b/src/main/utils/network-query @@ -0,0 +1 @@ +Subproject commit 0c6b34cef59839b672512d8ec9e4c4e01badf02a From cf3c2f5589996a9119420279a7eb1b98fd48223f Mon Sep 17 00:00:00 2001 From: Bryan Fox Date: Wed, 12 Dec 2018 12:23:38 -0500 Subject: [PATCH 2/6] Add filter & query steps to export backend --- src/main/data-managers/ExportManager.js | 34 ++++++++++++++-- .../formatters/__tests__/network-test.js | 40 ++++++++++++++++++- src/main/utils/formatters/network.js | 29 ++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/main/data-managers/ExportManager.js b/src/main/data-managers/ExportManager.js index 6e4af1315..2f6474454 100644 --- a/src/main/data-managers/ExportManager.js +++ b/src/main/data-managers/ExportManager.js @@ -8,8 +8,12 @@ const SessionDB = require('./SessionDB'); const { archive } = require('../utils/archive'); const { writeFile } = require('../utils/promised-fs'); const { RequestError, ErrorMessages } = require('../errors/RequestError'); -const { unionOfNetworks } = require('../utils/formatters/network'); const { makeTempDir, removeTempDir } = require('../utils/formatters/dir'); +const { + filterNetworkEntities, + filterNetworksWithQuery, + unionOfNetworks, +} = require('../utils/formatters/network'); const { formats, formatsAreValid, @@ -104,6 +108,11 @@ class ExportManager { * @param {Object} options.filter before formatting, apply this filter to each network (if * `exportNetworkUnion` is false) or the merged network (if * `exportNetworkUnion` is true) + * @param {Object} networkFilter a user-supplied filter configuration which will be run on each + * member of each network, to filter out unwanted nodes & edges. + * @param {Object} networkInclusionQuery a user-supplied query configuration which will be run on + * each interview's network to determine if it should be + * exported * @param {boolean} options.useDirectedEdges used by the formatters. May be removed in the future * if information is encapsulated in network. * @return {Promise} A `promise` that resloves to a filepath (string). The promise is decorated @@ -113,7 +122,14 @@ class ExportManager { */ createExportFile( protocol, - { destinationFilepath, exportFormats, exportNetworkUnion, useDirectedEdges/* , filter */ } = {}, + { + destinationFilepath, + exportFormats, + exportNetworkUnion, + entityFilter, + networkInclusionQuery, + useDirectedEdges, + } = {}, ) { if (!protocol) { return Promise.reject(new RequestError(ErrorMessages.NotFound)); @@ -131,6 +147,14 @@ class ExportManager { variableRegistry: protocol.variableRegistry, }; + // Export flow: + // 1. fetch all networks produced for this protocol's interviews + // 2. optional: run user query to select networks are exported + // 3. optional: merge all networks into a single union for export + // 4. optional: filter each network based on user-supplied rules + // 5. [TODO: #199] optional: insert ego with edges into each network + // 6. export each format for each network + // 7. save ZIP file to requested location const exportPromise = makeTempDir() .then((dir) => { tmpDir = dir; @@ -141,12 +165,14 @@ class ExportManager { .then(() => this.sessionDB.findAll(protocol._id, null, null)) // TODO: may want to preserve some session metadata for naming? .then(sessions => sessions.map(session => session.data)) + .then(allNetworks => filterNetworksWithQuery(allNetworks, networkInclusionQuery)) .then(networks => (exportNetworkUnion ? [unionOfNetworks(networks)] : networks)) - .then((networks) => { + .then(networks => filterNetworkEntities(networks, entityFilter)) + .then((filteredNetworks) => { // TODO: evaluate & test. It'll be easier to track progress when run concurrently, // but this may run into memory issues. promisedExports = flatten( - networks.map((network, i) => + filteredNetworks.map((network, i) => exportFormats.map(format => exportFile(`${i + 1}`, format, tmpDir, network, exportOpts)))); return Promise.all(promisedExports); diff --git a/src/main/utils/formatters/__tests__/network-test.js b/src/main/utils/formatters/__tests__/network-test.js index 67f74cc53..33a61d1b9 100644 --- a/src/main/utils/formatters/__tests__/network-test.js +++ b/src/main/utils/formatters/__tests__/network-test.js @@ -1,7 +1,45 @@ /* eslint-env jest */ -const { getNodeAttributes, nodeAttributesProperty, unionOfNetworks } = require('../network'); +const { + filterNetworkEntities, + filterNetworksWithQuery, + getNodeAttributes, + nodeAttributesProperty, + unionOfNetworks, +} = require('../network'); describe('network format helpers', () => { + describe('filtering & querying', () => { + const ruleConfig = { + rules: [ + { + type: 'alter', + options: { type: 'person', operator: 'EXACTLY', attribute: 'age', value: 20 }, + assert: { operator: 'GREATER_THAN', value: 0 }, // for query only + }, + ], + join: 'OR', + }; + + describe('filterNetworksWithQuery()', () => { + it('includes nodes from attribute query', () => { + const a = { nodes: [{ type: 'person', [nodeAttributesProperty]: { age: 20 } }], edges: [] }; + const b = { nodes: [{ type: 'person', [nodeAttributesProperty]: { age: 20 } }], edges: [] }; + const c = { nodes: [{ type: 'person', [nodeAttributesProperty]: { age: 21 } }], edges: [] }; + const networks = [a, b, c]; + expect(filterNetworksWithQuery(networks, ruleConfig)).toEqual([a, b]); + }); + }); + + describe('filterNetworkEntities()', () => { + it('includes nodes matching attributes', () => { + const alice = { type: 'person', [nodeAttributesProperty]: { age: 20 } }; + const bob = { type: 'person', [nodeAttributesProperty]: { age: 21 } }; + const networks = [{ nodes: [alice, bob], edges: [] }]; + expect(filterNetworkEntities(networks, ruleConfig)[0].nodes).toEqual([alice]); + }); + }); + }); + describe('unionOfNetworks', () => { it('joins nodes of two networks', () => { const a = { nodes: [{ id: 1 }], edges: [] }; diff --git a/src/main/utils/formatters/network.js b/src/main/utils/formatters/network.js index 8d3c07eea..f7db1231d 100644 --- a/src/main/utils/formatters/network.js +++ b/src/main/utils/formatters/network.js @@ -1,3 +1,6 @@ +const getQuery = require('../network-query/query').default; +const getFilter = require('../network-query/filter').default; + // TODO: share with other places this is defined const nodePrimaryKeyProperty = '_uid'; @@ -12,7 +15,33 @@ const unionOfNetworks = networks => return union; }, { nodes: [], edges: [] }); +/** + * Run the query on each network; filter for those which meet the criteria (i.e., where the query + * evaluates to `true`). + * @param {Object[]} networks An array of NC networks + * @param {Object} inclusionQueryConfig a query definition with asserting rules + * @return {Object[]} a subset of the networks + */ +const filterNetworksWithQuery = (networks, inclusionQueryConfig) => + (inclusionQueryConfig ? networks.filter(getQuery(inclusionQueryConfig)) : networks); + +/** + * Filter each network based on the filter config. + * @param {Object[]} networks An array of NC networks + * @param {Object} filterConfig a filter definition with rules + * @return {Object[]} a copy of `networks`, each possibly containing a subset of the original + */ +const filterNetworkEntities = (networks, filterConfig) => { + if (!filterConfig) { + return networks; + } + const filter = getFilter(filterConfig); + return networks.map(network => filter(network)); +}; + module.exports = { + filterNetworkEntities, + filterNetworksWithQuery, getNodeAttributes, nodeAttributesProperty, nodePrimaryKeyProperty, From af6446892856213dc4800877ac770ef963e2f8a9 Mon Sep 17 00:00:00 2001 From: Bryan Fox Date: Thu, 13 Dec 2018 08:56:44 -0500 Subject: [PATCH 3/6] Transpose variable registry for filter components Filter components from Architect work with IDs, not names, but our imported data has been transposed from ID->name. By transposing a copy of the registry, we don't need to modify the filtering components. --- src/renderer/containers/ExportScreen.js | 7 ++-- .../ducks/modules/__tests__/protocols-test.js | 32 +++++++++++++++++- src/renderer/ducks/modules/protocols.js | 33 +++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/renderer/containers/ExportScreen.js b/src/renderer/containers/ExportScreen.js index e4d00dec2..4988e316a 100644 --- a/src/renderer/containers/ExportScreen.js +++ b/src/renderer/containers/ExportScreen.js @@ -132,7 +132,7 @@ class ExportScreen extends Component { } render() { - const { protocol, protocolsHaveLoaded } = this.props; + const { protocol, protocolsHaveLoaded, variableRegistry } = this.props; if (protocolsHaveLoaded && !protocol) { // This protocol doesn't exist return ; @@ -252,7 +252,7 @@ class ExportScreen extends Component { @@ -267,16 +267,19 @@ ExportScreen.propTypes = { protocolsHaveLoaded: PropTypes.bool.isRequired, showConfirmation: PropTypes.func.isRequired, showError: PropTypes.func.isRequired, + variableRegistry: PropTypes.object, }; ExportScreen.defaultProps = { apiClient: null, protocol: null, + variableRegistry: null, }; const mapStateToProps = (state, ownProps) => ({ protocolsHaveLoaded: selectors.protocolsHaveLoaded(state), protocol: selectors.currentProtocol(state, ownProps), + variableRegistry: selectors.transposedRegistry(state, ownProps), }); const mapDispatchToProps = dispatch => ({ diff --git a/src/renderer/ducks/modules/__tests__/protocols-test.js b/src/renderer/ducks/modules/__tests__/protocols-test.js index 23c26468d..484593cfe 100644 --- a/src/renderer/ducks/modules/__tests__/protocols-test.js +++ b/src/renderer/ducks/modules/__tests__/protocols-test.js @@ -1,5 +1,5 @@ /* eslint-env jest */ -import reducer, { actionCreators, actionTypes } from '../protocols'; +import reducer, { actionCreators, actionTypes, selectors } from '../protocols'; import AdminApiClient from '../../../utils/adminApiClient'; jest.mock('../../../utils/adminApiClient'); @@ -95,4 +95,34 @@ describe('the protocols module', () => { }); }); }); + + describe('selectors', () => { + const { currentProtocol, protocolsHaveLoaded, transposedRegistry } = selectors; + + describe('currentProtocol', () => { + it('returns the current protocol object', () => { + const state = { protocols: [{ id: '1' }] }; + const props = { match: { params: { id: '1' } } }; + expect(currentProtocol(state, props)).toEqual(state.protocols[0]); + }); + }); + + describe('protocolsHaveLoaded', () => { + it('indicates protocols have loaded', () => { + expect(protocolsHaveLoaded({ protocols: null })).toBe(false); + expect(protocolsHaveLoaded({ protocols: [{ id: '1' }] })).toBe(true); + }); + }); + + describe('transposedRegistry', () => { + it('returns a modified registry', () => { + const variableRegistry = { node: { 'node-type-id': { name: 'person', variables: {} } } }; + const state = { protocols: [{ id: '1', variableRegistry }] }; + const props = { match: { params: { id: '1' } } }; + const transposed = transposedRegistry(state, props); + expect(transposed).toHaveProperty('node'); + expect(transposed.node).toHaveProperty('person'); + }); + }); + }); }); diff --git a/src/renderer/ducks/modules/protocols.js b/src/renderer/ducks/modules/protocols.js index 152d68112..000ac6ec4 100644 --- a/src/renderer/ducks/modules/protocols.js +++ b/src/renderer/ducks/modules/protocols.js @@ -38,6 +38,38 @@ const currentProtocol = (state, props) => { return protocols && id && protocols.find(p => p.id === id); }; +// Transpose one section of the registry ('node' or 'edge') from IDs to names +const transposedRegistrySection = (section = {}) => + Object.values(section).reduce((sectionRegistry, definition) => { + const displayVariable = definition.variables[definition.displayVariable]; + + const variables = Object.values(definition.variables).reduce((acc, variable) => { + acc[variable.name] = variable; + return acc; + }, {}); + sectionRegistry[definition.name] = { // eslint-disable-line no-param-reassign + ...definition, + displayVariable: displayVariable && displayVariable.name, + variables, + }; + return sectionRegistry; + }, {}); + +// Transpose all types & variable IDs to names +// Imported data is transposed; this allows utility components from Architect to work as-is. +const transposedRegistry = (state, props) => { + const protocol = currentProtocol(state, props); + if (!protocol) { + return null; + } + + const registry = protocol.variableRegistry || {}; + return { + edge: transposedRegistrySection(registry.edge), + node: transposedRegistrySection(registry.node), + }; +}; + const protocolsHaveLoaded = state => state.protocols !== initialState; const loadProtocolsDispatch = () => ({ @@ -89,6 +121,7 @@ const actionTypes = { const selectors = { currentProtocol, protocolsHaveLoaded, + transposedRegistry, }; export { From b34149d1828e1da78bc0cf417b1ccc3ff374e08a Mon Sep 17 00:00:00 2001 From: Bryan Fox Date: Thu, 13 Dec 2018 09:16:24 -0500 Subject: [PATCH 4/6] Fix button types --- src/renderer/components/Filter/Rule/AddButton.js | 1 + src/renderer/components/Filter/Rule/AlterRule.js | 2 +- src/renderer/components/Filter/Rule/EdgeRule.js | 2 +- src/renderer/components/Filter/Rule/EgoRule.js | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/renderer/components/Filter/Rule/AddButton.js b/src/renderer/components/Filter/Rule/AddButton.js index d3ca5cbd7..cb2266517 100644 --- a/src/renderer/components/Filter/Rule/AddButton.js +++ b/src/renderer/components/Filter/Rule/AddButton.js @@ -23,6 +23,7 @@ class AddButton extends PureComponent { >