Merge pull request #1240 from bcgov/dev-rook-RQ-FOIMOD-3551

move code to dev-rook branch instead of test-rook/test-marshal

.github/workflows/openinfoservice-cd.yml

@@ -0,0 +1,113 @@
+name: OpenInfo Service CD
+
+
+on:
+  push:
+    branches:
+      - dev
+      - main
+      - dev-marshal
+      - test-marshal
+      - dev-rook
+      - test-rook
+    paths:
+      - "computingservices/OpenInfoServices/**"
+      - ".github/workflows/openinfoservice-cd.yml"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./computingservices/OpenInfoServices
+
+env:
+  APP_NAME: "reviewer-openinfoservice"
+  TOOLS_NAME: "${{secrets.OPENSHIFT4_REPOSITORY}}"
+
+jobs:
+  openinfoservice-cd-by-push:
+    runs-on: ubuntu-20.04
+
+    if: github.event_name == 'push' && github.repository == 'bcgov/foi-docreviewer'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set ENV variables for dev branch
+        if: ${{ github.ref_name == 'dev' }}
+        shell: bash
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=dev" >> $GITHUB_ENV
+          echo "BRANCH_NAME=dev" >> $GITHUB_ENV
+          echo "ENV_NAME=dev" >> $GITHUB_ENV
+
+      - name: Set ENV variables for main branch
+        if: ${{ github.ref_name == 'main' }}
+        shell: bash
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=test" >> $GITHUB_ENV
+          echo "BRANCH_NAME=main" >> $GITHUB_ENV
+          echo "ENV_NAME=test" >> $GITHUB_ENV
+
+      - name: Set ENV variables for dev-marshal branch
+        if: ${{ github.ref_name == 'dev-marshal' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=dev-marshal" >> $GITHUB_ENV
+          echo "BRANCH_NAME=dev-marshal" >> $GITHUB_ENV
+          echo "ENV_NAME=dev" >> $GITHUB_ENV
+
+      - name: Set ENV variables for test-marshal branch
+        if: ${{ github.ref_name == 'test-marshal' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=test-marshal" >> $GITHUB_ENV
+          echo "BRANCH_NAME=test-marshal" >> $GITHUB_ENV
+          echo "ENV_NAME=test" >> $GITHUB_ENV
+      
+      - name: Set ENV variables for dev-rook branch
+        if: ${{ github.ref_name == 'dev-rook' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=dev-rook" >> $GITHUB_ENV
+          echo "BRANCH_NAME=dev-rook" >> $GITHUB_ENV
+          echo "ENV_NAME=dev" >> $GITHUB_ENV
+          echo "ENV_TAG_NAME=rook" >> $GITHUB_ENV
+
+      - name: Set ENV variables for test-rook branch
+        if: ${{ github.ref_name == 'test-rook' }}
+        run: |
+          echo "For ${{ github.ref_name }} branch"
+          echo "TAG_NAME=test-rook" >> $GITHUB_ENV
+          echo "BRANCH_NAME=test-rook" >> $GITHUB_ENV
+          echo "ENV_NAME=test" >> $GITHUB_ENV
+          echo "ENV_TAG_NAME=rook" >> $GITHUB_ENV
+
+      - name: Login Openshift
+        shell: bash
+        run: |
+          oc login --server=${{secrets.OPENSHIFT4_LOGIN_REGISTRY}} --token=${{secrets.OPENSHIFT4_SA_TOKEN}}
+
+      - name: Tools project
+        shell: bash
+        run: |
+          oc project ${{ env.TOOLS_NAME }}-tools
+
+      - name: Build from ${{ env.BRANCH_NAME }} branch
+        shell: bash
+        run: |
+          oc patch bc/${{ env.APP_NAME }}-build -p '{"spec":{"source":{"contextDir":"/computingservices/OpenInfoServices","git":{"ref":"${{ env.BRANCH_NAME }}"}}}}'
+
+      - name: Start Build Openshift
+        shell: bash
+        run: |
+          oc start-build ${{ env.APP_NAME }}-build --wait
+
+      - name: Tag+Deploy for ${{ env.TAG_NAME }}
+        shell: bash
+        run: |
+          oc tag ${{ env.APP_NAME }}:latest ${{ env.APP_NAME }}:${{ env.TAG_NAME }}
+
+      # - name: Watch new rollout (trigger by image change in Openshift)
+      #   shell: bash
+      #   run: |
+      #     oc rollout status dc/${{ env.APP_NAME }}-{{ env.ENV_TAG_NAME }} -n ${{ env.TOOLS_NAME }}-${{ env.ENV_NAME }} -w

.github/workflows/openinfoservice-ci.yml

@@ -0,0 +1,46 @@
+name: Open Info Service CI
+
+
+on:
+  pull_request:
+    branches:
+      - main
+      - dev
+      - dev-marshal
+      - test-marshal
+      - dev-rook
+      - test-rook
+    paths:
+      - "computingservices/OpenInfoServices/**"
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./computingservices/OpenInfoServices
+
+jobs:
+  docker-build-check:
+    runs-on: ubuntu-latest
+    name: Build dockerfile to ensure it works
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: docker build to check strictness
+        id: docker-build
+        run: |
+          docker build -f Dockerfile.local .
+
+  go-build-check:
+    runs-on: ubuntu-latest
+    name: Build go to ensure it works
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.23'
+      - name: Build the application
+        run: go build -v .
+      - name: Test the application
+        run: go test ./...

computingservices/OpenInfoServices/.sampleenv

@@ -0,0 +1,21 @@
+FOI_DB_USER=
+FOI_DB_PASSWORD=
+FOI_DB_NAME=
+FOI_DB_HOST=
+FOI_DB_PORT=
+
+OI_REDIS_HOST=
+OI_REDIS_PORT=
+OI_REDIS_PASSWORD=
+OI_QUEUE_NAME=OpenInfoQueue
+
+OI_S3_ENV=
+OI_S3_HOST=
+OI_S3_REGION=us-east-1
+OI_S3_BUCKET=openinfopub
+OI_ACCESS_KEY=
+OI_SECRET_KEY=
+
+OI_PREFIX=packages/
+SITEMAP_PREFIX=sitemap/
+SITEMAP_PAGES_LIMIT=5000

computingservices/OpenInfoServices/Dockerfile.enqueueforpublish.local

@@ -0,0 +1,14 @@
+FROM golang:1.23-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN go build -o main .
+
+# Use a minimal base image to run the application
+FROM alpine:latest
+WORKDIR /app
+COPY --from=builder /app/main .
+COPY --from=builder /app/templates/ ./templates/.
+ENTRYPOINT ["./main"]
+CMD ["enqueueforpublish"]

computingservices/OpenInfoServices/Dockerfile.enqueueforunpublish.local

@@ -0,0 +1,14 @@
+FROM golang:1.23-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN go build -o main .
+
+# Use a minimal base image to run the application
+FROM alpine:latest
+WORKDIR /app
+COPY --from=builder /app/main .
+COPY --from=builder /app/templates/ ./templates/.
+ENTRYPOINT ["./main"]
+CMD ["enqueueforunpublish"]

computingservices/OpenInfoServices/Dockerfile.local

@@ -0,0 +1,14 @@
+FROM golang:1.23-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN go build -o main .
+
+# Use a minimal base image to run the application
+FROM alpine:latest
+WORKDIR /app
+COPY --from=builder /app/main .
+COPY --from=builder /app/templates/ ./templates/.
+ENTRYPOINT ["./main"]
+CMD ["dequeue"]

computingservices/OpenInfoServices/Dockerfile.sitemap.local

@@ -0,0 +1,14 @@
+FROM golang:1.23-alpine AS builder
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN go build -o main .
+
+# Use a minimal base image to run the application
+FROM alpine:latest
+WORKDIR /app
+COPY --from=builder /app/main .
+COPY --from=builder /app/templates/ ./templates/.
+ENTRYPOINT ["./main"]
+CMD ["sitemap"]

computingservices/OpenInfoServices/config/config.go

@@ -0,0 +1,155 @@
+package myconfig
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"sync"
+
+	"github.com/spf13/viper"
+)
+
+var (
+	//DB
+	host     string
+	port     string
+	user     string
+	password string
+	dbname   string
+
+	//Redis
+	queue         string
+	queuehost     string
+	queueport     string
+	queuepassword string
+
+	//S3
+	s3url         string
+	oibucket      string
+	oiprefix      string
+	sitemapprefix string
+	sitemaplimit  int
+	region        string
+	accessKey     string
+	secretKey     string
+	s3host        string
+
+	env string
+
+	onceDB     sync.Once
+	onceRedis  sync.Once
+	onceS3     sync.Once
+	onceS3Path sync.Once
+	onceOthers sync.Once
+)
+
+// use viper package to read .env file
+// return the value of the key
+func viperEnvVariable(key string) string {
+
+	// SetConfigFile explicitly defines the path, name and extension of the config file.
+	// Viper will use this and not check any of the config paths.
+	// .env - It will search for the .env file in given path
+	viper.SetConfigFile(getEnv("ENVFILE_PATH"))
+
+	// Find and read the config file
+	err := viper.ReadInConfig()
+
+	if err != nil {
+		log.Fatalf("Error while reading config file %s", err)
+	}
+
+	// viper.Get() returns an empty interface{}
+	// to get the underlying type of the key,
+	// we have to do the type assertion, we know the underlying value is string
+	// if we type assert to other type it will throw an error
+	value, ok := viper.Get(key).(string)
+
+	// If the type is a string then ok will be true
+	// ok will make sure the program not break
+	if !ok {
+		log.Fatalf("Invalid type assertion")
+	}
+
+	return value
+}
+
+// Lazy initialization functions
+func loadConfigDB() {
+	host = getEnv("FOI_DB_HOST")
+	port = getEnv("FOI_DB_PORT")
+	user = getEnv("FOI_DB_USER")
+	password = getEnv("FOI_DB_PASSWORD")
+	dbname = getEnv("FOI_DB_NAME")
+}
+
+func loadConfigRedis() {
+	queuehost = getEnv("OI_REDIS_HOST")
+	queueport = getEnv("OI_REDIS_PORT")
+	queuepassword = getEnv("OI_REDIS_PASSWORD")
+}
+
+func loadConfigS3() {
+	region = getEnv("OI_S3_REGION")
+	accessKey = getEnv("OI_ACCESS_KEY")
+	secretKey = getEnv("OI_SECRET_KEY")
+	s3host = getEnv("OI_S3_HOST")
+}
+
+func loadConfigS3Path() {
+	s3url = "https://" + getEnv("OI_S3_HOST") + "/"
+	oibucket = getEnv("OI_S3_BUCKET")
+	oiprefix = getEnv("OI_PREFIX")
+	sitemapprefix = getEnv("SITEMAP_PREFIX")
+
+	var strerr error
+	sitemaplimit, strerr = strconv.Atoi(getEnv("SITEMAP_PAGES_LIMIT"))
+	if strerr != nil {
+		log.Printf("Error converting string to int for SITEMAP_PAGES_LIMIT, will use default value: %v", strerr)
+		sitemaplimit = 5000
+	}
+}
+
+func loadConfigOther() {
+	env = getEnv("OI_S3_ENV")
+	queue = getEnv("OI_QUEUE_NAME")
+}
+
+// Helper function to get environment variables
+func getEnv(key string) string {
+	value, exists := os.LookupEnv(key)
+	if !exists {
+		return viperEnvVariable(key)
+	}
+	return value
+}
+
+// GetDB retrieves the database variables with lazy initialization
+func GetDB() (string, string, string, string, string) {
+	onceDB.Do(loadConfigDB) // Ensures loadConfig is called only once
+	return host, port, user, password, dbname
+}
+
+// GetRedis retrieves the redis variables with lazy initialization
+func GetRedis() (string, string, string) {
+	onceRedis.Do(loadConfigRedis) // Ensures loadConfig is called only once
+	return queuehost, queueport, queuepassword
+}
+
+// GetS3 retrieves the S3 variables with lazy initialization
+func GetS3() (string, string, string, string) {
+	onceS3.Do(loadConfigS3) // Ensures loadConfig is called only once
+	return region, accessKey, secretKey, s3host
+}
+
+// GetS3 retrieves the S3 variables with lazy initialization
+func GetS3Path() (string, string, string, string, int) {
+	onceS3Path.Do(loadConfigS3Path) // Ensures loadConfig is called only once
+	return s3url, oibucket, oiprefix, sitemapprefix, sitemaplimit
+}
+
+// GetS3 retrieves the S3 variables with lazy initialization
+func GetOthers() (string, string) {
+	onceOthers.Do(loadConfigOther) // Ensures loadConfig is called only once
+	return env, queue
+}