From 8bce0846913573d56999d654e65319b8f4efb2f4 Mon Sep 17 00:00:00 2001
From: Sean Tracey <seanmtracey@gmail.com>
Date: Thu, 1 Jun 2017 16:29:03 +0100
Subject: [PATCH] Get all of the tagged items known to UPP.

---
 bin/lib/search-topics.js | 81 ++++++++++++++++++++++++++--------------
 routes/index.js          |  2 -
 2 files changed, 53 insertions(+), 30 deletions(-)

diff --git a/bin/lib/search-topics.js b/bin/lib/search-topics.js
index e1559b2..117af79 100644
--- a/bin/lib/search-topics.js
+++ b/bin/lib/search-topics.js
@@ -1,36 +1,61 @@
 const debug = require('debug')('bin:lib:search-topics');
 const fetch = require('node-fetch');
 
-module.exports = function(topic){
+module.exports = function(topic, offset = 0, recursive = false){
 	const SAPI_URL_MINUS_KEY = `${process.env.CAPI_ENDPOINT}/search/v1`;
 	const SAPI_URL = `${SAPI_URL_MINUS_KEY}?apiKey=${process.env.CAPI_KEY}`;
+	
+	const results = [];
 
-	const SEARCH_BODY = {
-		'queryString': `topics:${topic}`,
-  		'queryContext' : {
-  			'curations' : [ 'ARTICLES', 'BLOGS' ]
-  		},
-  		'resultContext' : {
-  			'maxResults' : '100',
-  			'offset' : '0',
-  			'aspects' : [ 'title', 'location', 'summary', 'lifecycle', 'metadata']
-  		}
-  	};
-
-	return fetch(SAPI_URL, {
-			'method'       : 'POST', 
-			'body'         :  JSON.stringify(SEARCH_BODY),
-			'headers'      : {
-				'accept'       : 'application/json',
-				'content-type' : 'application/json'
-			}
-		})
-		.then(res => {
-			if(res.status !== 200){
-				throw `An error occurred retrieving ${SAPI_URL} with body=${JSON.stringify(SEARCH_BODY)},\nres=${JSON.stringify(res)}`;
-			} else {
-				return res.json();
+	function search(topic, offset = 0){
+		
+		const SEARCH_BODY = {
+			'queryString': `topics:${topic}`,
+			'queryContext' : {
+				'curations' : [ 'ARTICLES', 'BLOGS' ]
+			},
+			'resultContext' : {
+				'maxResults' : '100',
+				'offset' : offset,
+				'aspects' : [ 'title', 'location', 'summary', 'lifecycle', 'metadata']
 			}
-		})
-	;
+		};
+		
+		return fetch(SAPI_URL, {
+				'method'       : 'POST', 
+				'body'         :  JSON.stringify(SEARCH_BODY),
+				'headers'      : {
+					'accept'       : 'application/json',
+					'content-type' : 'application/json'
+				}
+			})
+			.then(res => {
+				if(res.status !== 200){
+					throw `An error occurred retrieving ${SAPI_URL} with body=${JSON.stringify(SEARCH_BODY)},\nres=${JSON.stringify(res)}`;
+				} else {
+					return res.json();
+				}
+			})
+			.then(data => {
+
+				debug(data);
+	
+				if(data.results[0].indexCount < offset + 100){
+					debug('Got em all. Number of results:', results.length);
+					return results;
+				} else {
+					debug('Doing another search offset from:', offset + 100);
+					data.results[0].results.forEach(result => {
+						results.push(result);
+					});
+					return search(topic, offset + 100);
+				}
+
+			})
+		;
+
+	}
+
+	return search(topic, offset)
+
 }
diff --git a/routes/index.js b/routes/index.js
index 4bb333c..ecf7fe6 100644
--- a/routes/index.js
+++ b/routes/index.js
@@ -23,8 +23,6 @@ router.get('/', (req, res) => {
 			searchTopics('audio-articles')
 				.then(taggedArticles => {
 
-					taggedArticles = taggedArticles.results[0].results;
-
 					const readiedAssets = data.Items.filter(item => {
 							// Items that have been deleted from the database still have their UUID
 							// and enabled values saved, so that if they're reabsorbed, a previously