From 23d20e82f6a55cdde89b6fd6973db5ab54d4c40a Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 19 Dec 2024 17:01:17 +0100 Subject: [PATCH] wip part 2 --- api/src/services/setup/check-install.js | 3 +- api/src/services/setup/upgrade-steps.js | 2 +- .../services/setup/view-indexer-progress.js | 15 +++--- api/src/services/setup/view-indexer.js | 50 ++++++++++++++++--- .../nouveau/contacts_by_freetext/index.js | 1 + 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/api/src/services/setup/check-install.js b/api/src/services/setup/check-install.js index e8ceb6beaf..599a7ec2de 100644 --- a/api/src/services/setup/check-install.js +++ b/api/src/services/setup/check-install.js @@ -9,14 +9,13 @@ const startupLog = require('./startup-log'); const checkInstallForDb = async (database) => { const check = {}; const allDdocs = await ddocsService.getDdocs(database); - // console.log("allDdocs", allDdocs); const bundledDdocs = await upgradeUtils.getBundledDdocs(database); - console.log("bundledDdocs", bundledDdocs); const liveDdocs = allDdocs.filter(ddoc => !ddocsService.isStaged(ddoc._id)); const stagedDdocs = allDdocs.filter(ddoc => ddocsService.isStaged(ddoc._id)); const liveDdocsCheck = ddocsService.compareDdocs(bundledDdocs, liveDdocs); + console.log("liveDdocsCheck.different", liveDdocsCheck.different); check.missing = liveDdocsCheck.missing; check.different = liveDdocsCheck.different; diff --git a/api/src/services/setup/upgrade-steps.js b/api/src/services/setup/upgrade-steps.js index 220d2a9637..67b572b87a 100644 --- a/api/src/services/setup/upgrade-steps.js +++ b/api/src/services/setup/upgrade-steps.js @@ -102,7 +102,7 @@ const indexStagedViews = async () => { const viewsToIndex = await viewIndexer.getViewsToIndex(); const viewIndexingPromise = viewIndexer.indexViews(viewsToIndex); const stopQueryingIndexers = viewIndexerProgress.log(); - const nouveauIndexingPromise = Promise.resolve(); // TODO + const nouveauIndexingPromise = Promise.resolve(); await Promise.all([viewIndexingPromise, nouveauIndexingPromise]); stopQueryingIndexers(); }; diff --git a/api/src/services/setup/view-indexer-progress.js b/api/src/services/setup/view-indexer-progress.js index 86e98b7e7e..68a1f80c85 100644 --- a/api/src/services/setup/view-indexer-progress.js +++ b/api/src/services/setup/view-indexer-progress.js @@ -56,6 +56,7 @@ const updateRunningTasks = (indexers, activeTasks = []) => { indexers.push(indexer); } + console.log(`${task.node}-${task.pid} progress`, task.progress); indexer.tasks[`${task.node}-${task.pid}`] = task.progress; }); }; @@ -90,17 +91,13 @@ const getIndexers = async (indexers = []) => { try { const activeTasks = await db.activeTasks(); const tasks = activeTasks.filter(task => { - const isViewIndexingTask = task.type === 'indexer'; - if (isViewIndexingTask) { - return DDOC_PREFIX.test(String(task.design_document)); + const isFirstNouveauIndexingTask = task.type === 'search_indexer' && + task.design_document === '_design/medic-nouveau'; + if (isFirstNouveauIndexingTask) { + return true; } - const isNouveauIndexingTask = task.type === 'search_indexer'; - if (isNouveauIndexingTask) { - return task.design_document === '_design/medic-nouveau'; - } - - return false; + return DDOC_PREFIX.test(String(task.design_document)); }); // We assume all previous tasks have finished. console.log("indexers 1", indexers); diff --git a/api/src/services/setup/view-indexer.js b/api/src/services/setup/view-indexer.js index 6dac3ff347..fc82ec4e42 100644 --- a/api/src/services/setup/view-indexer.js +++ b/api/src/services/setup/view-indexer.js @@ -12,13 +12,17 @@ let continueIndexing; const indexViews = async (viewsToIndex) => { continueIndexing = true; + console.log("viewsToIndex", viewsToIndex); if (!Array.isArray(viewsToIndex)) { + console.log("not array"); await upgradeLogService.setIndexed(); return; } await upgradeLogService.setIndexing(); const indexResult = await Promise.all(viewsToIndex.map(indexView => indexView())); + console.log("indexResult", indexResult); + console.log("continueIndexing", continueIndexing); if (continueIndexing) { await upgradeLogService.setIndexed(); } @@ -38,14 +42,19 @@ const getViewsToIndex = async () => { for (const database of DATABASES) { const stagedDdocs = await ddocsService.getStagedDdocs(database); stagedDdocs.forEach(ddoc => { - if (!ddoc.views || !_.isObject(ddoc.views)) { - return; + if (ddoc.views && !_.isObject(ddoc.views)) { + const ddocViewIndexPromises = Object + .keys(ddoc.views) + .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName)); + viewsToIndex.push(...ddocViewIndexPromises); } - const ddocViewIndexPromises = Object - .keys(ddoc.views) - .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName)); - viewsToIndex.push(...ddocViewIndexPromises); + if (ddoc.nouveau && !_.isObject(ddoc.nouveau)) { + const ddocNouveauIndexPromises = Object + .keys(ddoc.nouveau) + .map(indexName => indexNouveauIndex.bind({}, database.name, ddoc._id, indexName)); + viewsToIndex.push(...ddocNouveauIndexPromises); + } }); } return viewsToIndex; @@ -79,7 +88,36 @@ const indexView = async (dbName, ddocId, viewName) => { } while (continueIndexing); }; +/** + * Returns a promise that resolves when a nouveau index is indexed. + * Retries querying the index until no error is thrown + * @param {String} dbName + * @param {String} ddocId + * @param {String} indexName + * @return {Promise} + */ +const indexNouveauIndex = async (dbName, ddocId, indexName) => { + do { + try { + return await request.get({ + uri: `${environment.serverUrl}/${dbName}/_design/${ddocId}/_nouveau/${indexName}`, + json: true, + qs: { q: '*:*', limit: 1 }, + }); + } catch (requestError) { + if (!continueIndexing) { + return; + } + + if (!requestError || !requestError.error || !SOCKET_TIMEOUT_ERROR_CODE.includes(requestError.error.code)) { + throw requestError; + } + } + } while (continueIndexing); +}; + const stopIndexing = () => { + console.trace("************ stopIndexing"); continueIndexing = false; }; diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 2502f61245..7fb443356c 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -1,4 +1,5 @@ function (doc) { + 'different 2'; var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; var toIndex = '';