diff --git a/.github/actions/deploy-with-medic-conf/README.md b/.github/actions/deploy-with-medic-conf/README.md index 6ffcdc724fb..8d732f21d11 100644 --- a/.github/actions/deploy-with-medic-conf/README.md +++ b/.github/actions/deploy-with-medic-conf/README.md @@ -13,7 +13,7 @@ on: ['deployment'] jobs: deployment: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb06157f5fc..2cb7031d671 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ jobs: build: name: Compile the app - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - name: Get Docker Hub username @@ -57,7 +57,7 @@ jobs: config-tests: needs: build name: Config Tests - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: grunt-cmd: [ 'exec:test-config-standard', 'exec:test-config-default'] @@ -78,7 +78,7 @@ jobs: publish: needs: [tests,config-tests] name: Publish branch build - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 @@ -98,7 +98,7 @@ jobs: tests: needs: build name: ${{ matrix.grunt-cmd }} on node ${{ matrix.node-version }} - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 env: NODE: ${{ matrix.node-version }} diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 1f9af342a0a..1d6407179af 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -10,7 +10,7 @@ env: jobs: cleanup: name: Deleting old published artefacts - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/scalability.yml b/.github/workflows/scalability.yml index 36f35aeafe5..649521e58b8 100644 --- a/.github/workflows/scalability.yml +++ b/.github/workflows/scalability.yml @@ -8,7 +8,7 @@ on: jobs: build: name: Set up Medic on EC2 and Execute Jmeter on another EC2 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Set TAG_NAME var run: echo "TAG_NAME=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV @@ -25,4 +25,4 @@ jobs: - name: Execute Jmeter env: SCALABILITY_ARN: ${{ secrets.SCALABILITY_ARN }} - run: cd tests/scalability && ./start_jmeter_ec2.sh \ No newline at end of file + run: cd tests/scalability && ./start_jmeter_ec2.sh diff --git a/Gruntfile.js b/Gruntfile.js index 4e97d2c746e..8fce93912d6 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -129,6 +129,7 @@ module.exports = function(grunt) { test: { files: { ['http://admin:pass@localhost:4984/medic-test']: 'build/ddocs/medic.json', + ['http://admin:pass@localhost:4984/medic-test-logs']: 'build/ddocs/medic/_attachments/ddocs/logs.json', }, }, staging: { @@ -540,11 +541,12 @@ module.exports = function(grunt) { 'patch webapp/node_modules/moment/locale/hi.js < webapp/patches/moment-hindi-use-euro-numerals.patch', // patch enketo to always mark the /inputs group as relevant - 'patch webapp/node_modules/enketo-core/src/js/Form.js < webapp/patches/enketo-inputs-always-relevant.patch', + 'patch webapp/node_modules/enketo-core/src/js/form.js < webapp/patches/enketo-inputs-always-relevant_form.patch', + 'patch webapp/node_modules/enketo-core/src/js/relevant.js < webapp/patches/enketo-inputs-always-relevant_relevant.patch', - // patch enketo so forms with no active pages are considered valid - // https://github.com/medic/medic/issues/5484 - 'patch webapp/node_modules/enketo-core/src/js/page.js < webapp/patches/enketo-handle-no-active-pages.patch', + // patch enketo to fix repeat name collision bug - this should be removed when upgrading to a new version of enketo-core + // https://github.com/enketo/enketo-core/issues/815 + 'patch webapp/node_modules/enketo-core/src/js/calculate.js < webapp/patches/enketo-repeat-name-collision.patch', // patch messageformat to add a default plural function for languages not yet supported by make-plural #5705 'patch webapp/node_modules/messageformat/lib/plurals.js < webapp/patches/messageformat-default-plurals.patch', diff --git a/TESTING.md b/TESTING.md index 6b3ff8c3a0d..87b0abfdf8d 100644 --- a/TESTING.md +++ b/TESTING.md @@ -138,8 +138,7 @@ Github actions will artifact all files in tests/logs. This is the directory any ### Test Architecture -Our github actions spin up an ubuntu-18.04 machine. Installs software and then launches Couchdb and Horticulturalist in a docker container. This is needed to run our applications in the specific node versions we support while allowing our test code to run in versions of node it they support. This creates a paradigm to keep in mind when writing tests. Tests run on the ubuntu machine. Any test code that starts a server or runs an executable is running outside of the horti container. The ports are exposed for all our services and horti has access to the cht-core root via a volume. Horti can also talk to the host by getting the gateway of the docker network. - +Our GitHub actions spin up an ubuntu-22.04 machine. Installs software and then launches Couchdb and Horticulturalist in a docker container. This is needed to run our applications in the specific node versions we support while allowing our test code to run in versions of node it they support. This creates a paradigm to keep in mind when writing tests. Tests run on the ubuntu machine. Any test code that starts a server or runs an executable is running outside of the horti container. The ports are exposed for all our services and horti has access to the cht-core root via a volume. Horti can also talk to the host by getting the gateway of the docker network. ### Glossary diff --git a/admin/package-lock.json b/admin/package-lock.json index 27aead1b423..12600beeb87 100644 --- a/admin/package-lock.json +++ b/admin/package-lock.json @@ -158,9 +158,9 @@ "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM=" }, "google-libphonenumber": { - "version": "3.2.27", - "resolved": "https://registry.npmjs.org/google-libphonenumber/-/google-libphonenumber-3.2.27.tgz", - "integrity": "sha512-et3QlrfWemNPhyUfXZmJG8TfzitfAN71ygNI15+B35zNge/7vyZxkpDsc13oninkf8RAtN2kNEzvMr4L1n3vfQ==" + "version": "3.2.30", + "resolved": "https://registry.npmjs.org/google-libphonenumber/-/google-libphonenumber-3.2.30.tgz", + "integrity": "sha512-Kx2/AqmY0P6863vOhkiCFqfAxfY3jagMe916ByU38JRKiRCqSHGJW1qTOZNV4+ag8Xda69dk6w8VwEeswVy44w==" }, "gsm": { "version": "0.1.4", diff --git a/admin/package.json b/admin/package.json index 0b85f978df8..3265448df2f 100644 --- a/admin/package.json +++ b/admin/package.json @@ -19,7 +19,7 @@ "bikram-sambat-bootstrap": "^1.5.0", "bootstrap": "^3.4.1", "font-awesome": "^4.7.0", - "google-libphonenumber": "^3.2.27", + "google-libphonenumber": "^3.2.30", "gsm": "^0.1.4", "jquery": "^3.5.1", "lodash": "^4.17.19", diff --git a/admin/src/css/theme.less b/admin/src/css/theme.less index feb99af3300..929e3a20734 100644 --- a/admin/src/css/theme.less +++ b/admin/src/css/theme.less @@ -185,3 +185,33 @@ ul { .people-section { margin-bottom: 30px; } + +.modal { + height: fit-content; + max-height: 70%; + top: 20%; + left: 25%; + width: 50%; + margin: 0px; + padding: 0px; +} + +.modal-dialog { + width: auto; + margin: 0px; + padding: 0px; +} + +.modal-content { + width: auto; +} + +.summary-box { + border-bottom-width:0px !important; + padding-bottom:20px !important; +} + +.summary-number { + font-size: 7rem; + margin-bottom: -15px; +} \ No newline at end of file diff --git a/admin/src/js/controllers/edit-user.js b/admin/src/js/controllers/edit-user.js index 70af95be825..db88f3bd916 100644 --- a/admin/src/js/controllers/edit-user.js +++ b/admin/src/js/controllers/edit-user.js @@ -386,7 +386,7 @@ angular return UpdateUser($scope.editUserModel.username, updates); } - return CreateUser(updates); + return CreateUser.createSingleUser(updates); }) .then(() => { $scope.setFinished(); diff --git a/admin/src/js/controllers/main.js b/admin/src/js/controllers/main.js index bff9ac58353..8801b3b2fef 100644 --- a/admin/src/js/controllers/main.js +++ b/admin/src/js/controllers/main.js @@ -3,11 +3,12 @@ angular.module('controllers').controller('MainCtrl', $log, $scope, $state, - $translate, $window, Auth, + Language, Location, - Session + Session, + SetLanguage ) { 'ngInject'; @@ -23,7 +24,7 @@ angular.module('controllers').controller('MainCtrl', }); }; - $translate.use('en'); + Language().then(locale => SetLanguage(locale)); $scope.authorized = false; $scope.navbarCollapsed = true; Auth.any([['can_configure'], ['can_view_outgoing_messages'], ['can_export_all']]) diff --git a/admin/src/js/controllers/multiple-user.js b/admin/src/js/controllers/multiple-user.js new file mode 100644 index 00000000000..7416a2cdc89 --- /dev/null +++ b/admin/src/js/controllers/multiple-user.js @@ -0,0 +1,150 @@ +angular.module('controllers').controller('MultipleUserCtrl', function( + $scope, + $uibModalInstance, + $window, + CreateUser, + DB +) { + + 'use strict'; + 'ngInject'; + + $scope.status = { uploading: false }; + $scope.displayAddMultipleModal = true; + $scope.displayUnavailableModal = false; + $scope.displayUploadConfirm = false; + $scope.displayProcessingStatus = false; + $scope.displayFinishSummary = false; + $scope.outputFileUrl = ''; + $scope.processTotal = ''; + const USER_LOG_DOC_ID = 'bulk-user-upload-'; + + $scope.onCancel = function () { + $scope.clearScreen(); + $uibModalInstance.dismiss(); + }; + + const getLogsByType = (docPrefix) => { + return DB({ logsDB: true }) + .allDocs({ + startkey: docPrefix, + endkey: docPrefix + '\ufff0', + include_docs: true + }) + .then(result => { + if (!result || !result.rows || !result.rows.length) { + return; + } + return result.rows + .map(row => row.doc) + .sort((a, b) => new Date(b.bulk_uploaded_on) - new Date(a.bulk_uploaded_on)); + }); + }; + + const prepareStringForCSV = (str) => { + if (!str) { + return str; + } + return '"' + str.replace(/"/g, '""') + '"'; + }; + + const convertToCSV = doc => { + if (!doc || !doc.data) { + return; + } + const eol = '\r\n'; + const delimiter = ','; + const columns = ['import.status:excluded', 'import.message:excluded', 'import.username:excluded']; + let output = columns.join(delimiter) + eol; + + doc.data.forEach(record => { + if (record.import) { + output += [ + prepareStringForCSV(record.import.status), + prepareStringForCSV(record.import.message), + prepareStringForCSV(record.username), + ].join(delimiter) + eol; + } + }); + + const file = new Blob([output], { type: 'text/csv;charset=utf-8;' }); + return URL.createObjectURL(file); + }; + + $scope.processUpload = function () { + $scope.clearScreen(); + $scope.displayProcessingStatus = true; + return $scope.uploadedData + .text() + .then(data => CreateUser.createMultipleUsers(data)) + .then(() => getLogsByType(USER_LOG_DOC_ID)) + .then(docs => { + if (!docs || !docs.length) { + // eslint-disable-next-line no-console + console.error('CreateMultipleUser : Error getting logs by type'); + $scope.setError('CreateMultipleUser : Error getting logs by type'); + } else { + $scope.uploadProcessLog = docs[0]; + $scope.processTotal = $scope.uploadProcessLog.progress.parsing.total; + $scope.successUsersNumber = $scope.uploadProcessLog.progress.saving.successful; + $scope.ignoredUsersNumber = $scope.uploadProcessLog.progress.saving.ignored; + $scope.failedUsersNumber = $scope.uploadProcessLog.progress.saving.failed; + $scope.$apply(); + $scope.outputFileUrl = convertToCSV( $scope.uploadProcessLog); + $scope.showFinishSummary(); + } + }) + .catch(error => { + // eslint-disable-next-line no-console + console.error(error, 'CreateMultipleUser : Error processing data after upload'); + $scope.setError(error, 'CreateMultipleUser : Error processing data after upload'); + }); + }; + + $scope.showFinishSummary = function () { + $scope.clearScreen(); + $scope.displayFinishSummary = true; + $scope.$apply(); + }; + + $scope.showDisplayUploadConfirm = function () { + $scope.clearScreen(); + $scope.displayUploadConfirm = true; + $scope.$apply(); + }; + + $scope.backToAppManagement = function () { + $window.location.href = '/admin/#/users'; + $window.location.reload(); + $scope.clearScreen(); + $uibModalInstance.dismiss(); + $scope.$apply(); + }; + + $scope.clearScreen = function () { + $scope.displayAddMultipleModal = false; + $scope.displayUnavailableModal = false; + $scope.displayUploadConfirm = false; + $scope.displayProcessingStatus = false; + $scope.displayFinishSummary = false; + }; + + const upload = function() { + const files = $('#users-upload .uploader')[0].files; + if (!files || files.length === 0) { + return; + } + $scope.usersFilename = files[0].name; + $scope.uploadedData = files[0]; + $scope.showDisplayUploadConfirm(); + }; + + angular.element(function () { + $('#users-upload .uploader').on('change', upload); + $('#users-upload .choose').on('click', function(e) { + e.preventDefault(); + $('#users-upload .uploader').click(); + }); + }); +} +); diff --git a/admin/src/js/controllers/upgrade.js b/admin/src/js/controllers/upgrade.js index 1c43ecf09f4..32c584617ad 100644 --- a/admin/src/js/controllers/upgrade.js +++ b/admin/src/js/controllers/upgrade.js @@ -40,6 +40,9 @@ angular.module('controllers').controller('UpgradeCtrl', return DB().get('_design/medic') .then(function(ddoc) { $scope.currentDeploy = ddoc.deploy_info; + + const currentVersion = Version.currentVersion($scope.currentDeploy); + $scope.isUsingFeatureRelease = !!currentVersion && typeof currentVersion.featureRelease !== 'undefined'; }); }; @@ -94,7 +97,21 @@ angular.module('controllers').controller('UpgradeCtrl', endkey: [ 'release', 'medic', 'medic', minVersion.major, minVersion.minor, minVersion.patch], descending: true, limit: 50 - }) + }), + featureReleases: !$scope.isUsingFeatureRelease ? builds({ + startkey: [minVersion.featureRelease, 'medic', 'medic', {}], + endkey: [ + minVersion.featureRelease, + 'medic', + 'medic', + minVersion.major, + minVersion.minor, + minVersion.patch, + minVersion.beta, + ], + descending: true, + limit: 50, + }) : [], }).then(function(results) { $scope.versions = results; }); diff --git a/admin/src/js/controllers/users.js b/admin/src/js/controllers/users.js index b7af055d855..652018985ee 100644 --- a/admin/src/js/controllers/users.js +++ b/admin/src/js/controllers/users.js @@ -52,6 +52,14 @@ angular.module('controllers').controller('UsersCtrl', }); }; + $scope.showAddMultipleUsersModal = function() { + Modal({ + templateUrl: 'templates/multiple_user_modal.html', + controller: 'MultipleUserCtrl', + model: {}, + }); + }; + $scope.$on('UsersUpdated', function() { $scope.updateList(); }); diff --git a/admin/src/js/directives/modal.js b/admin/src/js/directives/modal.js index aea188ba0f5..a92eefa5213 100644 --- a/admin/src/js/directives/modal.js +++ b/admin/src/js/directives/modal.js @@ -52,7 +52,7 @@ angular.module('directives').directive('mmModal', function() { disableSubmit: '=', // string: (optional) the expression which, if true, will show the delete button - showDelete: '=' + showDelete: '=', } }; }); diff --git a/admin/src/js/main.js b/admin/src/js/main.js index b8f4589a3b1..28377f84c19 100644 --- a/admin/src/js/main.js +++ b/admin/src/js/main.js @@ -66,6 +66,7 @@ require('./controllers/targets-edit'); require('./controllers/upgrade'); require('./controllers/upgrade-confirm'); require('./controllers/users'); +require('./controllers/multiple-user'); angular.module('directives', ['ngSanitize']); require('./directives/file-model'); diff --git a/admin/src/js/services/create-user.js b/admin/src/js/services/create-user.js index 425e383a82e..6eb34581baf 100644 --- a/admin/src/js/services/create-user.js +++ b/admin/src/js/services/create-user.js @@ -1,38 +1,67 @@ -angular.module('services').factory('CreateUser', - function( - $http, - $log, - $q - ) { - 'ngInject'; - 'use strict'; - - /** - * Creates a user from a collection of updates - * - * Updates are in the style of the /api/v1/users/{username} service, see - * its documentation for more details. - * - * @param {Object} updates Updates you wish to make - */ - return function(updates) { - const url = '/api/v1/users'; - - if (!updates.username) { - return $q.reject('You must provide a username to create a user'); - } - - $log.debug('CreateUser', url, updates); - - return $http({ - method: 'POST', - url: url, - data: updates, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json' +(function () { + + 'use strict'; + const URL = '/api/v2/users'; + + angular.module('services').factory('CreateUser', + function ( + $http, + $log, + $q + ) { + 'ngInject'; + + /** + * Creates a user from a collection of updates + * + * Updates are in the style of the /api/v2/users/{username} service, see + * its documentation for more details. + * + * @param {Object} updates Updates you wish to make + */ + const createSingleUser = (updates) => { + if (!updates.username) { + return $q.reject('You must provide a username to create a user'); } - }); - }; - } + + $log.debug('CreateSingleUser', URL, updates); + + return $http({ + method: 'POST', + url: URL, + data: updates, + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + }); + }; + + /** + * Creates a user from a collection of updates + * + * @param {Object} data content of the csv file + */ + const createMultipleUsers = (data) => { + $log.debug('CreateMultipleUsers', URL, data); + + return $http({ + method: 'POST', + url: URL, + data: data, + headers: { + 'Content-Type': 'text/csv', + 'Accept': 'application/json' + } + }); + }; + + return { + createSingleUser, + createMultipleUsers + }; + } + ); + +}() ); diff --git a/admin/src/js/services/db.js b/admin/src/js/services/db.js index 3cb35b4546d..cdf730cf637 100644 --- a/admin/src/js/services/db.js +++ b/admin/src/js/services/db.js @@ -5,6 +5,7 @@ const DISALLOWED_CHARS = /[^a-z0-9_$()+/-]/g; const USER_DB_SUFFIX = 'user'; const META_DB_SUFFIX = 'meta'; const USERS_DB_SUFFIX = 'users'; +const MEDIC_LOGS_DB_SUFFIX = 'logs'; angular.module('inboxServices').factory('DB', function( @@ -31,14 +32,19 @@ angular.module('inboxServices').factory('DB', return username.replace(DISALLOWED_CHARS, match => `(${match.charCodeAt(0)})`); }; - const getDbName = (remote, meta, usersMeta) => { + const getDbName = (remote, meta, usersMeta, logsDB) => { const parts = []; if (remote) { parts.push(Location.url); } else { parts.push(Location.dbName); } - if ((!remote || meta) && !usersMeta) { + + if (logsDB) { + parts.push(MEDIC_LOGS_DB_SUFFIX); + } + + if ((!remote || meta) && !usersMeta && !logsDB) { parts.push(USER_DB_SUFFIX); parts.push(getUsername(remote)); } else if (usersMeta) { @@ -63,8 +69,8 @@ angular.module('inboxServices').factory('DB', return clone; }; - const get = ({ remote=isOnlineOnly, meta=false, usersMeta=false }={}) => { - const name = getDbName(remote, meta, usersMeta); + const get = ({ remote=isOnlineOnly, meta=false, usersMeta=false, logsDB=false }={}) => { + const name = getDbName(remote, meta, usersMeta, logsDB); if (!cache[name]) { cache[name] = pouchDB(name, getParams(remote, meta, usersMeta)); } diff --git a/admin/src/js/services/get-summaries.js b/admin/src/js/services/get-summaries.js index 12441f3ac35..b5b22922300 100644 --- a/admin/src/js/services/get-summaries.js +++ b/admin/src/js/services/get-summaries.js @@ -93,7 +93,6 @@ angular.module('inboxServices').factory('GetSummaries', contact_type: doc.contact_type, contact: doc.contact && doc.contact._id, lineage: getLineage(doc.parent), - simprints_id: doc.simprints_id, date_of_death: doc.date_of_death, muted: doc.muted }; diff --git a/admin/src/js/services/language.js b/admin/src/js/services/language.js index 10956652f2c..10a441cb3af 100644 --- a/admin/src/js/services/language.js +++ b/admin/src/js/services/language.js @@ -24,16 +24,8 @@ const moment = require('moment'); SetLanguageCookie ) { 'ngInject'; - - const setDatepickerLanguage = function(language) { - const availableCalendarLanguages = Object.keys($.fn.datepicker.dates); - const calendarLanguage = availableCalendarLanguages.indexOf(language) >= 0 ? language : 'en'; - $.fn.datepicker.defaults.language = calendarLanguage; - }; - return function(code, setLanguageCookie) { moment.locale([code, 'en']); - setDatepickerLanguage(code); $translate.use(code); if (setLanguageCookie !== false) { diff --git a/admin/src/js/services/location.js b/admin/src/js/services/location.js index b3077e811ca..eb0f3c44f0a 100644 --- a/admin/src/js/services/location.js +++ b/admin/src/js/services/location.js @@ -5,7 +5,8 @@ angular.module('inboxServices').factory('Location', 'ngInject'; const location = $window.location; - const dbName = 'medic'; + const isTestEnv = $window.localStorage.getItem('isTestEnv'); + const dbName = isTestEnv ? 'medic-test' : 'medic'; const path = '/'; const adminPath = '/admin/'; const port = location.port ? ':' + location.port : ''; diff --git a/admin/src/js/services/version.js b/admin/src/js/services/version.js index 95f45adf8f4..9995d27cde4 100644 --- a/admin/src/js/services/version.js +++ b/admin/src/js/services/version.js @@ -16,8 +16,10 @@ angular.module('services').factory('Version', }; const versionInformation = function(versionString) { + // TODO: replace this regex with named capture groups once we deprecate node 8 + // /^(?
\d+)\.(?
\d+)\.(?
\d+)(?
-FR(?:-\w+)+)?(?:-beta\.(?
\d+))?$/ const versionMatch = versionString && - versionString.match(/^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-beta\.([0-9]+))?$/); + versionString.match(/^(\d+)\.(\d+)\.(\d+)(-FR(?:-\w+)+)?(?:-beta\.(\d+))?$/); if (versionMatch) { const version = { @@ -26,8 +28,16 @@ angular.module('services').factory('Version', patch: parseInt(versionMatch[3]) }; + if (versionMatch[5] !== undefined) { + version.beta = parseInt(versionMatch[5]); + } + if (versionMatch[4] !== undefined) { - version.beta = parseInt(versionMatch[4]); + version.featureRelease = versionMatch[4].slice(1); // remove leading dash '-' + + if (version.beta) { + version.featureRelease += '-beta'; + } } return version; diff --git a/admin/src/templates/edit_user.html b/admin/src/templates/edit_user.html index 5a5f6e13f00..b92696d2a2d 100644 --- a/admin/src/templates/edit_user.html +++ b/admin/src/templates/edit_user.html @@ -7,6 +7,7 @@ on-cancel="cancel()" on-submit="editUser()" > +
diff --git a/admin/src/templates/modal.html b/admin/src/templates/modal.html index 28620a7b56a..f7b77679302 100644 --- a/admin/src/templates/modal.html +++ b/admin/src/templates/modal.html @@ -4,7 +4,7 @@
{{titleKey}}
-