From 5e6c2b1da42b126c9e49b422205fb21a69637d49 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Fri, 19 Jan 2024 18:02:21 +0100
Subject: [PATCH 01/11] WIP add script to rebuilt recent vocabs

---
 package.json         |   3 +-
 src/common.js        |  14 ++++-
 src/rebuildVocabs.js | 125 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 140 insertions(+), 2 deletions(-)
 create mode 100644 src/rebuildVocabs.js

diff --git a/package.json b/package.json
index 077a859..35797fe 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,8 @@
     "test:unit": "jest --forceExit unit",
     "test:int": "jest --forceExit int",
     "test:docker": "jest --forceExit docker",
-    "lint:js": "eslint src --ext .jsx,.js --quiet"
+    "lint:js": "eslint src --ext .jsx,.js --quiet",
+    "rebuild-vocabs": "node src/rebuildVocabs.js"
   },
   "repository": {
     "type": "git",
diff --git a/src/common.js b/src/common.js
index 2bdbafa..b0a47b7 100644
--- a/src/common.js
+++ b/src/common.js
@@ -80,6 +80,17 @@ const isValid = (hook, event) => {
   return false
 }
 
+/**
+ * @param {Object} payload
+ * @param {string} SECRET
+ * @returns {string} signed payload
+*/
+const securePayload = (payload, SECRET) => {
+  const hmac = crypto.createHmac("sha1", SECRET)
+  const digest = "sha1=" + hmac.update(JSON.stringify(payload)).digest("hex")
+  return digest
+}
+
 const isSecured = (signature, payload, SECRET) => {
   // Is not secured if all the parameters are not present
   if (!signature || !payload || !SECRET) {
@@ -215,5 +226,6 @@ module.exports = {
   isSecured,
   getRepositoryFiles,
   parseHook,
-  checkStdOutForError
+  checkStdOutForError,
+  securePayload
 }
diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
new file mode 100644
index 0000000..e28df7a
--- /dev/null
+++ b/src/rebuildVocabs.js
@@ -0,0 +1,125 @@
+const fs = require("fs-extra")
+const path = require('path');
+const { securePayload } = require("./common.js")
+require("dotenv").config()
+
+const { PORT, SECRET, BUILD_URL, DOCKER_IMAGE, DOCKER_TAG, PULL_IMAGE_SECRET } =
+  process.env
+
+/**
+ * @typedef {Object} BuildInfo
+ * @property {string} id
+ * @property {Object} body
+ * @property {Date} date
+ * @property {string} ref
+ * @property {string} repository
+*/
+
+// get all json data from dist/build folder
+const readBuildDir = async () => {
+  const directoryPath = './dist/build';
+  try {
+    const files = await fs.readdir(directoryPath);
+    const jsonFiles = files.filter(file => path.extname(file) === '.json');
+
+    const jsonObjects = await Promise.all(jsonFiles.map(async file => {
+      const filePath = path.join(directoryPath, file);
+      const data = await fs.readFile(filePath, 'utf8');
+      return JSON.parse(data);
+    }));
+
+    return jsonObjects;
+  } catch (err) {
+    console.error('Error:', err);
+    throw err; // or handle error as needed
+  }
+}
+
+// map with body, ref, date, repository
+
+// create an array only containing the most recent webhook requests
+/**
+ * @param {BuildInfo[]} buildInfo
+ */
+const sortBuildInfo = (buildInfo) => {
+  const reposToBuild = buildInfo.filter(b => {
+    if (buildInfo.some(e => e.ref === b.ref && e.repository === b.repository && new Date(e.date).getTime() > new Date(b.date).getTime())) {
+      return false
+    } else if (buildInfo.some(e => e.ref === b.ref && e.repository === b.repository && new Date(e.date).getTime() <= new Date(b.date).getTime())) {
+      return true
+    }
+  })
+  return reposToBuild
+}
+
+/**
+ * send fetch request to webhook
+ * @param {BuildInfo} buildInfo
+ * @returns {string} buildId
+ */
+const sendBuildRequest = async (buildInfo) => {
+  const payload = {
+    repository: {
+      full_name: buildInfo.repository
+    },
+    ref: buildInfo.ref
+  }
+  const signature = securePayload(payload, SECRET)
+  const headers = {
+    "x-hub-signature": signature,
+    "Accept": "application/json",
+    "Content-Type": "application/json",
+    "x-github-event": "push",
+  }
+  try {
+    const response = await fetch(`http://localhost:${PORT}/build`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(payload)
+    })
+    if (!response.ok) {
+      throw new Error("Network response was not ok!")
+    }
+    const respBody = await response.text()
+    // get url from response
+    const url = new URL(respBody.substring(17))
+    const id = url.searchParams.get("id")
+
+    return id
+  } catch (error) {
+    console.error("Error sending request", error)
+  }
+}
+
+//
+const checkBuildStatus = (id) => {
+  const getData = async () => {
+    try {
+      const response = fs.readFileSync(`./dist/build/${id}.json`)
+      const json = JSON.parse(response)
+      // renderData(json)
+      // TODO do sth with data, check build status
+      if (json.status === "complete" || json.status === "error") {
+        console.log(`ID: ${id} Finish with status: ${json.status}`)
+        return
+      } else {
+        throw new Error("Not completed")
+      }
+    } catch (error) {
+      setTimeout(() => {
+        // console.warn("error", error)
+        getData()
+      }, 2000)
+    }
+  }
+  getData()
+}
+
+const main = async () => {
+  const buildInfo = await readBuildDir()
+  const sortedBuildInfo = sortBuildInfo(buildInfo)
+  const buildIds = await Promise.all(sortedBuildInfo.map((b) => sendBuildRequest(b)))
+  buildIds.forEach(id => checkBuildStatus(id))
+}
+
+main()

From 5d5f5dd3090ed1fa59fab78367204337cfbf19b8 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Mon, 22 Jan 2024 11:07:07 +0100
Subject: [PATCH 02/11] Add info about how to rebuild a vocabulary (#19)

---
 Dockerfile           |  3 +--
 README.md            |  8 ++++++++
 src/rebuildVocabs.js | 21 +++++++++++++--------
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 098d51f..a307652 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,8 +25,7 @@ RUN chown -R node:node /app
 COPY --chown=node:node .env.example .env
 COPY --chown=node:node . .
 
-# don't run prepare step with husky
-RUN npm pkg delete scripts.prepare
+RUN npm config set update-notifier false
 
 RUN npm i --only=production
 
diff --git a/README.md b/README.md
index 261687e..5ca29e5 100644
--- a/README.md
+++ b/README.md
@@ -63,6 +63,14 @@ In order to wire this up with GitHub, this has to be available to the public. Yo
 
 To restart and rebuild the service, e.g. after a `git pull` do `docker compose up --build --force-recreate`.
 
+## Rebuilding all vocabularies
+
+To rebuild all vocabularies:
+
+1. Make a backup of the dist-folder: `cp -R ./dist ./dist-backup`
+1. Make sure to have built docker image:  `docker build -t skohub-webhook .`
+1. Then mount the dist folder of the webhook container and rebuilt the vocabs: `docker run --network=host -v ./dist:/app/dist skohub-webhook:latest "npm run rebuild-vocabs"`
+
 ## Connecting to our webhook server
 
 Feel free to clone https://github.com/literarymachine/skos.git to poke around. Go to https://github.com/YOUR_GITHUB_USER/skos/settings/hooks/new to set up the web hook (get in touch to receive the secret). Edit https://github.com/YOUR_GITHUB_USER/skos/edit/master/hochschulfaecher.ttl and commit the changes to master. This will trigger a build and expose it at https://test.skohub.io/YOUR_GITHUB_USER/skos/w3id.org/class/hochschulfaecher/scheme.
diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
index e28df7a..2f27f16 100644
--- a/src/rebuildVocabs.js
+++ b/src/rebuildVocabs.js
@@ -13,6 +13,7 @@ const { PORT, SECRET, BUILD_URL, DOCKER_IMAGE, DOCKER_TAG, PULL_IMAGE_SECRET } =
  * @property {Date} date
  * @property {string} ref
  * @property {string} repository
+ * @property {string} status
 */
 
 // get all json data from dist/build folder
@@ -31,15 +32,14 @@ const readBuildDir = async () => {
     return jsonObjects;
   } catch (err) {
     console.error('Error:', err);
-    throw err; // or handle error as needed
+    throw err;
   }
 }
 
-// map with body, ref, date, repository
-
-// create an array only containing the most recent webhook requests
 /**
+ * Create an array only containing the most recent webhook requests
  * @param {BuildInfo[]} buildInfo
+ * @returns {BuildInfo[]} sorted build information
  */
 const sortBuildInfo = (buildInfo) => {
   const reposToBuild = buildInfo.filter(b => {
@@ -93,21 +93,26 @@ const sendBuildRequest = async (buildInfo) => {
 
 //
 const checkBuildStatus = (id) => {
+  const maxAttempts = 30
+  let attempts = 0
   const getData = async () => {
     try {
       const response = fs.readFileSync(`./dist/build/${id}.json`)
+      /** @type {BuildInfo} */
       const json = JSON.parse(response)
-      // renderData(json)
-      // TODO do sth with data, check build status
       if (json.status === "complete" || json.status === "error") {
-        console.log(`ID: ${id} Finish with status: ${json.status}`)
+        console.log(`${json.repository}, ${json.ref}: Finish with status: ${json.status} (ID: ${id})`)
         return
       } else {
         throw new Error("Not completed")
       }
     } catch (error) {
+      if (attempts > maxAttempts) {
+        console.log(`${json.repository}, ${json.ref}: did not finish after ${attempts} attempts. Aborting. Error: ${error} (ID: ${id})`)
+        return
+      }
       setTimeout(() => {
-        // console.warn("error", error)
+        attempts++;
         getData()
       }, 2000)
     }

From cc404dc0abb77397c552511f696d24643b6e3146 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Mon, 22 Jan 2024 15:49:34 +0100
Subject: [PATCH 03/11] Use buildURL instead of localhost #30

---
 src/rebuildVocabs.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
index 2f27f16..961dd91 100644
--- a/src/rebuildVocabs.js
+++ b/src/rebuildVocabs.js
@@ -3,7 +3,7 @@ const path = require('path');
 const { securePayload } = require("./common.js")
 require("dotenv").config()
 
-const { PORT, SECRET, BUILD_URL, DOCKER_IMAGE, DOCKER_TAG, PULL_IMAGE_SECRET } =
+const { SECRET, BUILD_URL } =
   process.env
 
 /**
@@ -72,7 +72,7 @@ const sendBuildRequest = async (buildInfo) => {
     "x-github-event": "push",
   }
   try {
-    const response = await fetch(`http://localhost:${PORT}/build`, {
+    const response = await fetch(BUILD_URL, {
       method: "POST",
       headers,
       body: JSON.stringify(payload)
@@ -82,7 +82,10 @@ const sendBuildRequest = async (buildInfo) => {
     }
     const respBody = await response.text()
     // get url from response
-    const url = new URL(respBody.substring(17))
+    const responseUrl = respBody.substring(17).startsWith("http")
+      ? respBody.substring(17)
+      : "https://" + respBody.substring(17)
+    const url = new URL(responseUrl)
     const id = url.searchParams.get("id")
 
     return id

From fd6fde3bdaf343a8c002804a9140950042f501be Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Mon, 22 Jan 2024 16:19:03 +0100
Subject: [PATCH 04/11] Add function to add protocol to URLs

---
 src/rebuildVocabs.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
index 961dd91..ad49abb 100644
--- a/src/rebuildVocabs.js
+++ b/src/rebuildVocabs.js
@@ -6,6 +6,13 @@ require("dotenv").config()
 const { SECRET, BUILD_URL } =
   process.env
 
+function protocolizeUrl(url) {
+  return url.startsWith("http")
+    ? url
+    : "https://" + url
+
+}
+
 /**
  * @typedef {Object} BuildInfo
  * @property {string} id
@@ -72,7 +79,7 @@ const sendBuildRequest = async (buildInfo) => {
     "x-github-event": "push",
   }
   try {
-    const response = await fetch(BUILD_URL, {
+    const response = await fetch(protocolizeUrl(BUILD_URL), {
       method: "POST",
       headers,
       body: JSON.stringify(payload)
@@ -82,9 +89,8 @@ const sendBuildRequest = async (buildInfo) => {
     }
     const respBody = await response.text()
     // get url from response
-    const responseUrl = respBody.substring(17).startsWith("http")
-      ? respBody.substring(17)
-      : "https://" + respBody.substring(17)
+    const responseUrl = protocolizeUrl(respBody.substring(17))
+    console.log("Build Urls:", responseUrl)
     const url = new URL(responseUrl)
     const id = url.searchParams.get("id")
 

From 47e128a4bce81203ce474778d371782f5105c3bc Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Tue, 23 Jan 2024 10:08:44 +0100
Subject: [PATCH 05/11] Test if branch and repo still exist before trying to
 build #30

---
 src/rebuildVocabs.js | 61 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 52 insertions(+), 9 deletions(-)

diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
index ad49abb..7b27efe 100644
--- a/src/rebuildVocabs.js
+++ b/src/rebuildVocabs.js
@@ -59,6 +59,36 @@ const sortBuildInfo = (buildInfo) => {
   return reposToBuild
 }
 
+/**
+ * @param {string} repository
+ * @param {string} ref
+ */
+async function checkIfBranchExists(repository, ref) {
+  const branchName = ref.split("/").slice(-1)[0]
+  const result = await fetch(`https://api.github.com/repos/${repository}/branches`)
+    .then(response => {
+      if (!response.ok) {
+        throw new Error('Network response was not ok');
+      }
+      return response.json();
+    })
+    .then(data => {
+      const branchExists = data.some(branch => branch.name === branchName);
+      if (branchExists) {
+        console.log(`Branch "${branchName}" exists!`);
+        return true
+      } else {
+        console.log(`Branch "${branchName}" does not exist.`);
+        return false
+      }
+    })
+    .catch(error => {
+      console.error(`Fetch error for repo ${repository} and branch ${branchName}, ${error}`);
+      return false
+    });
+  return result
+}
+
 /**
  * send fetch request to webhook
  * @param {BuildInfo} buildInfo
@@ -94,30 +124,41 @@ const sendBuildRequest = async (buildInfo) => {
     const url = new URL(responseUrl)
     const id = url.searchParams.get("id")
 
-    return id
+    return {
+      id,
+      repository: buildInfo.repository,
+      ref: buildInfo.ref
+    }
   } catch (error) {
     console.error("Error sending request", error)
   }
 }
 
-//
-const checkBuildStatus = (id) => {
+/**
+ * @param {{
+ * id: string
+ * repository: string
+ * ref: string
+ * }} buildInfo
+ */
+const checkBuildStatus = (buildInfo) => {
   const maxAttempts = 30
   let attempts = 0
+  let json = {}
   const getData = async () => {
     try {
-      const response = fs.readFileSync(`./dist/build/${id}.json`)
+      const response = fs.readFileSync(`./dist/build/${buildInfo.id}.json`)
       /** @type {BuildInfo} */
-      const json = JSON.parse(response)
+      json = JSON.parse(response)
       if (json.status === "complete" || json.status === "error") {
-        console.log(`${json.repository}, ${json.ref}: Finish with status: ${json.status} (ID: ${id})`)
+        console.log(`${json.repository}, ${json.ref}: Finish with status: ${json.status} (ID: ${buildInfo.id})`)
         return
       } else {
         throw new Error("Not completed")
       }
     } catch (error) {
       if (attempts > maxAttempts) {
-        console.log(`${json.repository}, ${json.ref}: did not finish after ${attempts} attempts. Aborting. Error: ${error} (ID: ${id})`)
+        console.log(`${buildInfo.repository}, ${buildInfo.ref}: did not finish after ${attempts} attempts. Aborting. Error: ${error} (ID: ${buildInfo.id})`)
         return
       }
       setTimeout(() => {
@@ -132,8 +173,10 @@ const checkBuildStatus = (id) => {
 const main = async () => {
   const buildInfo = await readBuildDir()
   const sortedBuildInfo = sortBuildInfo(buildInfo)
-  const buildIds = await Promise.all(sortedBuildInfo.map((b) => sendBuildRequest(b)))
-  buildIds.forEach(id => checkBuildStatus(id))
+  const branchesExisting = await Promise.all(sortedBuildInfo.map(b => checkIfBranchExists(b.repository, b.ref)))
+  const cleanedBuildInfo = sortedBuildInfo.filter((_, i) => branchesExisting[i])
+  const newBuildInfo = await Promise.all(cleanedBuildInfo.map((b) => sendBuildRequest(b)))
+  newBuildInfo.forEach(info => checkBuildStatus(info))
 }
 
 main()

From 86834398a9f65c29ab2451d924089846ed2fe6d8 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Tue, 23 Jan 2024 10:10:36 +0100
Subject: [PATCH 06/11] Remove network host command #30

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5ca29e5..a0b6c17 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,7 @@ To rebuild all vocabularies:
 
 1. Make a backup of the dist-folder: `cp -R ./dist ./dist-backup`
 1. Make sure to have built docker image:  `docker build -t skohub-webhook .`
-1. Then mount the dist folder of the webhook container and rebuilt the vocabs: `docker run --network=host -v ./dist:/app/dist skohub-webhook:latest "npm run rebuild-vocabs"`
+1. Then mount the dist folder of the webhook container and rebuilt the vocabs: `docker run -v ./dist:/app/dist skohub-webhook:latest "npm run rebuild-vocabs"`
 
 ## Connecting to our webhook server
 

From 47656a8348687d85b36ab6b21c5c29adab9c611a Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Tue, 23 Jan 2024 10:45:22 +0100
Subject: [PATCH 07/11] Add .dockerignore to reduce image size #30

---
 .dockerignore | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..6856c70
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,5 @@
+node_modules/
+dist/
+data/
+.git/
+.env

From 554d7a313335f249113262b247703ea14f633cd9 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Tue, 23 Jan 2024 14:36:39 +0100
Subject: [PATCH 08/11] Add option to provide a github token as well as CLI
 arguments to just build a specific repo #30

---
 .env.example         |  5 +++-
 README.md            |  9 ++++++-
 src/common.js        | 16 +++++++++++--
 src/rebuildVocabs.js | 56 ++++++++++++++++++++++++++++++++------------
 4 files changed, 67 insertions(+), 19 deletions(-)

diff --git a/.env.example b/.env.example
index a15498a..4cefccb 100644
--- a/.env.example
+++ b/.env.example
@@ -3,4 +3,7 @@ SECRET=YOUR_GITHUB_WEBHOOK_SECRET
 BUILD_URL=https://test.skohub.io/build
 DOCKER_IMAGE=skohub/skohub-vocabs-docker
 DOCKER_TAG=latest
-PULL_IMAGE_SECRET=ThisIsATest
\ No newline at end of file
+PULL_IMAGE_SECRET=ThisIsATest
+GITHUB_TOKEN=
+REBUILD_MAX_ATTEMPTS=30
+
diff --git a/README.md b/README.md
index a0b6c17..9f97d30 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,8 @@ The `.env` file contains configuration details used by the static site generator
 - `DOCKER_IMAGE`: The docker image which should be used to build the vocabulary, defaults to `skohub/skohub-vocabs-docker`
 - `DOCKER_TAG`: The docker tag for the `DOCKER_IMAGE`, defaults to `latest`
 - `PULL_IMAGE_SECRET`: The secret needed for the `/image` endpoint to trigger the pull of new images via webhook.
+- `GITHUB_TOKEN`: In order to avoid api rate limits when using GitHub, provide a personal access token.
+- `REBUILD_MAX_ATTEMPTS`: Max attempts trying to check the build status of the vocabulary. The build status is checked every 2 seconds, so 30 attempts make 60 seconds of waiting for the build status.
 
 ## How does it work?
 
@@ -63,7 +65,7 @@ In order to wire this up with GitHub, this has to be available to the public. Yo
 
 To restart and rebuild the service, e.g. after a `git pull` do `docker compose up --build --force-recreate`.
 
-## Rebuilding all vocabularies
+## Rebuilding vocabularies
 
 To rebuild all vocabularies:
 
@@ -71,6 +73,11 @@ To rebuild all vocabularies:
 1. Make sure to have built docker image:  `docker build -t skohub-webhook .`
 1. Then mount the dist folder of the webhook container and rebuilt the vocabs: `docker run -v ./dist:/app/dist skohub-webhook:latest "npm run rebuild-vocabs"`
 
+
+To rebuild only a specific vocabulary you can provide the GitHub repository and the branch to build:
+
+`docker run -v ./dist:/app/dist skohub-webhook:latest npm run rebuild-vocabs -- test/test-vocabs main`
+
 ## Connecting to our webhook server
 
 Feel free to clone https://github.com/literarymachine/skos.git to poke around. Go to https://github.com/YOUR_GITHUB_USER/skos/settings/hooks/new to set up the web hook (get in touch to receive the secret). Edit https://github.com/YOUR_GITHUB_USER/skos/edit/master/hochschulfaecher.ttl and commit the changes to master. This will trigger a build and expose it at https://test.skohub.io/YOUR_GITHUB_USER/skos/w3id.org/class/hochschulfaecher/scheme.
diff --git a/src/common.js b/src/common.js
index b0a47b7..705ae66 100644
--- a/src/common.js
+++ b/src/common.js
@@ -1,6 +1,16 @@
 const crypto = require("crypto")
 const fetch = require("node-fetch")
 
+require("dotenv").config()
+
+const { GITHUB_TOKEN } = process.env
+const gitHubApiHeaders = GITHUB_TOKEN
+  ? {
+    "Authorization": `Bearer ${GITHUB_TOKEN}`,
+    "X-GitHub-Api-Version": "2022-11-28"
+  }
+  : {}
+
 const getHookGitHub = (headers, payload, SECRET) => {
   const obj = {
     type: "github",
@@ -140,7 +150,8 @@ const getRepositoryFiles = async ({ type, repository, ref, filesURL }) => {
 async function fetchTTLFilesFromGitHubRepository(repository, path = '', ref = '') {
   const response = await fetch(`https://api.github.com/repos/${repository}/contents/${path}?` + new URLSearchParams({
     ref: ref
-  }));
+  }),
+    { headers: gitHubApiHeaders });
   const contents = await response.json();
   let ttlFiles = formatGitHubFiles(contents)
   const subDirectories = contents.filter(file => file.type === 'dir');
@@ -227,5 +238,6 @@ module.exports = {
   getRepositoryFiles,
   parseHook,
   checkStdOutForError,
-  securePayload
+  securePayload,
+  gitHubApiHeaders
 }
diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
index 7b27efe..a2902f7 100644
--- a/src/rebuildVocabs.js
+++ b/src/rebuildVocabs.js
@@ -1,10 +1,11 @@
 const fs = require("fs-extra")
 const path = require('path');
-const { securePayload } = require("./common.js")
+const { securePayload, gitHubApiHeaders } = require("./common.js")
 require("dotenv").config()
 
-const { SECRET, BUILD_URL } =
-  process.env
+const { SECRET, BUILD_URL, REBUILD_MAX_ATTEMPTS } = process.env
+const args = process.argv.slice(2);
+console.log(args); // ['arg1', 'arg2', 'arg3']
 
 function protocolizeUrl(url) {
   return url.startsWith("http")
@@ -65,7 +66,9 @@ const sortBuildInfo = (buildInfo) => {
  */
 async function checkIfBranchExists(repository, ref) {
   const branchName = ref.split("/").slice(-1)[0]
-  const result = await fetch(`https://api.github.com/repos/${repository}/branches`)
+  const result = await fetch(`https://api.github.com/repos/${repository}/branches`, {
+    headers: gitHubApiHeaders
+  })
     .then(response => {
       if (!response.ok) {
         throw new Error('Network response was not ok');
@@ -75,7 +78,7 @@ async function checkIfBranchExists(repository, ref) {
     .then(data => {
       const branchExists = data.some(branch => branch.name === branchName);
       if (branchExists) {
-        console.log(`Branch "${branchName}" exists!`);
+        console.log(`${repository}/${branchName} exists!`);
         return true
       } else {
         console.log(`Branch "${branchName}" does not exist.`);
@@ -131,6 +134,11 @@ const sendBuildRequest = async (buildInfo) => {
     }
   } catch (error) {
     console.error("Error sending request", error)
+    return {
+      id: null,
+      repository: buildInfo.repository,
+      ref: buildInfo.ref
+    }
   }
 }
 
@@ -142,7 +150,7 @@ const sendBuildRequest = async (buildInfo) => {
  * }} buildInfo
  */
 const checkBuildStatus = (buildInfo) => {
-  const maxAttempts = 30
+  const maxAttempts = REBUILD_MAX_ATTEMPTS ? Number(REBUILD_MAX_ATTEMPTS) : 30
   let attempts = 0
   let json = {}
   const getData = async () => {
@@ -157,13 +165,19 @@ const checkBuildStatus = (buildInfo) => {
         throw new Error("Not completed")
       }
     } catch (error) {
-      if (attempts > maxAttempts) {
+      if (attempts > maxAttempts || !buildInfo.id) {
         console.log(`${buildInfo.repository}, ${buildInfo.ref}: did not finish after ${attempts} attempts. Aborting. Error: ${error} (ID: ${buildInfo.id})`)
         return
       }
       setTimeout(() => {
-        attempts++;
-        getData()
+        if (json?.status === "processing") {
+          // we just keep trying to get the data
+          getData()
+        } else {
+          // increase attempts as it seems to be somewhere in between defined statuses
+          attempts++;
+          getData()
+        }
       }, 2000)
     }
   }
@@ -171,12 +185,24 @@ const checkBuildStatus = (buildInfo) => {
 }
 
 const main = async () => {
-  const buildInfo = await readBuildDir()
-  const sortedBuildInfo = sortBuildInfo(buildInfo)
-  const branchesExisting = await Promise.all(sortedBuildInfo.map(b => checkIfBranchExists(b.repository, b.ref)))
-  const cleanedBuildInfo = sortedBuildInfo.filter((_, i) => branchesExisting[i])
-  const newBuildInfo = await Promise.all(cleanedBuildInfo.map((b) => sendBuildRequest(b)))
-  newBuildInfo.forEach(info => checkBuildStatus(info))
+  if (args.length) {
+    const buildInfo = {
+      repository: args[0],
+      ref: "refs/heads/" + args[1]
+    }
+    const branchesExisting = await Promise.all([buildInfo].map(b => checkIfBranchExists(b.repository, b.ref)))
+    const cleanedBuildInfo = [buildInfo].filter((_, i) => branchesExisting[i])
+    const newBuildInfo = await Promise.all(cleanedBuildInfo.map((b) => sendBuildRequest(b)))
+    newBuildInfo.forEach(info => checkBuildStatus(info))
+
+  } else {
+    const buildInfo = await readBuildDir()
+    const sortedBuildInfo = sortBuildInfo(buildInfo)
+    const branchesExisting = await Promise.all(sortedBuildInfo.map(b => checkIfBranchExists(b.repository, b.ref)))
+    const cleanedBuildInfo = sortedBuildInfo.filter((_, i) => branchesExisting[i])
+    const newBuildInfo = await Promise.all(cleanedBuildInfo.map((b) => sendBuildRequest(b)))
+    newBuildInfo.forEach(info => checkBuildStatus(info))
+  }
 }
 
 main()

From 171a0e9444f09d43a0c82570319ecf0a98eed228 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Tue, 23 Jan 2024 14:42:58 +0100
Subject: [PATCH 09/11] Delete console log

---
 src/rebuildVocabs.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/rebuildVocabs.js b/src/rebuildVocabs.js
index a2902f7..db94500 100644
--- a/src/rebuildVocabs.js
+++ b/src/rebuildVocabs.js
@@ -5,7 +5,6 @@ require("dotenv").config()
 
 const { SECRET, BUILD_URL, REBUILD_MAX_ATTEMPTS } = process.env
 const args = process.argv.slice(2);
-console.log(args); // ['arg1', 'arg2', 'arg3']
 
 function protocolizeUrl(url) {
   return url.startsWith("http")

From bb73f7ac7fbd2212f02ca562a09ca07a1163c8e1 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Tue, 23 Jan 2024 16:32:09 +0100
Subject: [PATCH 10/11] Add info about mounting .env file #30

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9f97d30..e325e40 100644
--- a/README.md
+++ b/README.md
@@ -71,12 +71,12 @@ To rebuild all vocabularies:
 
 1. Make a backup of the dist-folder: `cp -R ./dist ./dist-backup`
 1. Make sure to have built docker image:  `docker build -t skohub-webhook .`
-1. Then mount the dist folder of the webhook container and rebuilt the vocabs: `docker run -v ./dist:/app/dist skohub-webhook:latest "npm run rebuild-vocabs"`
+1. Then mount the dist folder of the webhook container and rebuilt the vocabs: `docker run -v ./.env:/app/.env -v ./dist:/app/dist skohub-webhook:latest "npm run rebuild-vocabs"`
 
 
 To rebuild only a specific vocabulary you can provide the GitHub repository and the branch to build:
 
-`docker run -v ./dist:/app/dist skohub-webhook:latest npm run rebuild-vocabs -- test/test-vocabs main`
+`docker run -v ./.env:/app/.env -v ./dist:/app/dist skohub-webhook:latest npm run rebuild-vocabs -- test/test-vocabs main`
 
 ## Connecting to our webhook server
 

From ec92b71f3d8a940a4d70ec3234c745b6ae4c60df Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Wed, 24 Jan 2024 09:46:35 +0100
Subject: [PATCH 11/11] Add notice about api rate limits and rebuild attempts
 #30

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index e325e40..2a8a004 100644
--- a/README.md
+++ b/README.md
@@ -67,6 +67,13 @@ To restart and rebuild the service, e.g. after a `git pull` do `docker compose u
 
 ## Rebuilding vocabularies
 
+**Notice:**
+
+During the rebuild of all hosted vocabularies, the service might make a lot of requests to the GitHub Api.
+Depending on the number of hosted vocabularies, you might run into API rate limits.
+To avoid this you can provide a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) in `.env` file (see above).
+You also might want to increase the number of `REBUILD_MAX_ATTEMPTS` (to something like 300 or 3000), because rebuilds might take some time, depending on your machine, network, number of vocabularies etc.
+
 To rebuild all vocabularies:
 
 1. Make a backup of the dist-folder: `cp -R ./dist ./dist-backup`