diff --git a/computingservices/OpenInfoServices/config/config.go b/computingservices/OpenInfoServices/config/config.go index 58b555a2..3d0d571b 100644 --- a/computingservices/OpenInfoServices/config/config.go +++ b/computingservices/OpenInfoServices/config/config.go @@ -34,13 +34,24 @@ var ( secretKey string s3host string - env string - - onceDB sync.Once - onceRedis sync.Once - onceS3 sync.Once - onceS3Path sync.Once - onceOthers sync.Once + //Keycloak + keycloakurl string + keycloakrealm string + keycloakclientid string + keycloakclientsecret string + keycloakuser string + keycloakpassword string + + //Other + env string + foiflowapi string + + onceDB sync.Once + onceRedis sync.Once + onceS3 sync.Once + onceS3Path sync.Once + onceKeycloak sync.Once + onceOthers sync.Once ) // use viper package to read .env file @@ -110,9 +121,19 @@ func loadConfigS3Path() { } } +func loadConfigKeycloak() { + keycloakurl = getEnv("KEYCLOAK_URL") + keycloakrealm = getEnv("KEYCLOAK_URL_REALM") + keycloakclientid = getEnv("KEYCLOAK_CLIENT_ID") + keycloakclientsecret = getEnv("KEYCLOAK_CLIENT_SECRET") + keycloakuser = getEnv("KEYCLOAK_USER") + keycloakpassword = getEnv("KEYCLOAK_PASS") +} + func loadConfigOther() { env = getEnv("OI_S3_ENV") queue = getEnv("OI_QUEUE_NAME") + foiflowapi = getEnv("FOIFLOW_BASE_API_URL") } // Helper function to get environment variables @@ -148,8 +169,14 @@ func GetS3Path() (string, string, string, string, int) { return s3url, oibucket, oiprefix, sitemapprefix, sitemaplimit } +// GetKeycloak retrieves the Keycloak variables with lazy initialization +func GetKeycloak() (string, string, string, string, string, string) { + onceKeycloak.Do(loadConfigKeycloak) // Ensures loadConfig is called only once + return keycloakurl, keycloakrealm, keycloakclientid, keycloakclientsecret, keycloakuser, keycloakpassword +} + // GetS3 retrieves the S3 variables with lazy initialization -func GetOthers() (string, string) { +func GetOthers() (string, string, string) { onceOthers.Do(loadConfigOther) // Ensures loadConfig is called only once - return env, queue + return env, queue, foiflowapi } diff --git a/computingservices/OpenInfoServices/lib/awslib/s3.go b/computingservices/OpenInfoServices/lib/awslib/s3.go index bd653ec9..c66a6473 100644 --- a/computingservices/OpenInfoServices/lib/awslib/s3.go +++ b/computingservices/OpenInfoServices/lib/awslib/s3.go @@ -190,7 +190,7 @@ func ScanS3(openInfoBucket string, openInfoPrefix string, urlPrefix string, file func CopyS3(sourceBucket string, sourcePrefix string, filemappings []AdditionalFile) { s3url, oibucket, oiprefix, sitemapprefix, sitemaplimit = myconfig.GetS3Path() - env, _ := myconfig.GetOthers() + env, _, _ := myconfig.GetOthers() // bucket := "dev-openinfopub" bucket := sourceBucket diff --git a/computingservices/OpenInfoServices/lib/db/dbservices.go b/computingservices/OpenInfoServices/lib/db/dbservices.go index c3237fba..14504432 100644 --- a/computingservices/OpenInfoServices/lib/db/dbservices.go +++ b/computingservices/OpenInfoServices/lib/db/dbservices.go @@ -1,6 +1,7 @@ package dbservice import ( + httpservice "OpenInfoServices/lib/http" "database/sql" "fmt" "log" @@ -40,6 +41,7 @@ type OpenInfoRecord struct { Sitemap_pages string Type string Additionalfiles []AdditionalFile + Foirequestid int } func Conn(dsn string) (*sql.DB, error) { @@ -109,6 +111,7 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) { SELECT oi.foiopeninforequestid, mr.foiministryrequestid, + r.foirequestid, mr.axisrequestid, mr.description, oi.publicationdate, @@ -148,7 +151,7 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) { oiRecordsMap := make(map[int]*OpenInfoRecord) for rows.Next() { - var openinfoid, foiministryrequestid, additionalfileid sql.NullInt64 + var openinfoid, foiministryrequestid, foirequestid, additionalfileid sql.NullInt64 var axisrequestid, description, published_date, contributor, applicant_type, bcgovcode, sitemap_pages, queuetype, filename, s3uripath sql.NullString var fees sql.NullFloat64 var isactive bool @@ -169,6 +172,7 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) { err := rows.Scan( &openinfoid, &foiministryrequestid, + &foirequestid, &axisrequestid, &description, &published_date, @@ -187,11 +191,12 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) { return records, fmt.Errorf("failed to retrieve query result for prepublish: %w", err) } - if openinfoid.Valid && foiministryrequestid.Valid && axisrequestid.Valid && description.Valid && published_date.Valid && contributor.Valid && applicant_type.Valid && fees.Valid && bcgovcode.Valid && sitemap_pages.Valid && queuetype.Valid { + if openinfoid.Valid && foiministryrequestid.Valid && foirequestid.Valid && axisrequestid.Valid && description.Valid && published_date.Valid && contributor.Valid && applicant_type.Valid && fees.Valid && bcgovcode.Valid && sitemap_pages.Valid && queuetype.Valid { if _, ok := oiRecordsMap[int(foiministryrequestid.Int64)]; !ok { oiRecordsMap[int(foiministryrequestid.Int64)] = &OpenInfoRecord{ Openinfoid: int(openinfoid.Int64), Foiministryrequestid: int(foiministryrequestid.Int64), + Foirequestid: int(foirequestid.Int64), Axisrequestid: axisrequestid.String, Description: description.String, Published_date: published_date.String, @@ -238,6 +243,7 @@ func GetOIRecordsForPublishing(db *sql.DB) ([]OpenInfoRecord, error) { SELECT oi.foiopeninforequestid, mr.foiministryrequestid, + r.foirequestid, mr.axisrequestid, mr.description, oi.publicationdate, @@ -270,6 +276,7 @@ func GetOIRecordsForPublishing(db *sql.DB) ([]OpenInfoRecord, error) { err := rows.Scan( &record.Openinfoid, &record.Foiministryrequestid, + &record.Foirequestid, &record.Axisrequestid, &record.Description, &record.Published_date, @@ -337,7 +344,7 @@ func GetOIRecordsForUnpublishing(db *sql.DB) ([]OpenInfoRecord, error) { return records, nil } -func UpdateOIRecordState(db *sql.DB, foiministryrequestid int, publishingstatus string, message string, sitemap_pages string) error { +func UpdateOIRecordState(db *sql.DB, foiflowapi string, foiministryrequestid int, foirequestid int, publishingstatus string, message string, sitemap_pages string, oistatusid int) error { // Begin a transaction tx, err := db.Begin() @@ -380,9 +387,39 @@ func UpdateOIRecordState(db *sql.DB, foiministryrequestid int, publishingstatus log.Fatalf("Error committing transaction: %v", err) } + // Update request state in FOIMinistryRequests + // endpoint + section := "oistatusid" + endpoint := fmt.Sprintf("%s/foirequests/%d/ministryrequest/%d/section/%s", foiflowapi, foirequestid, foiministryrequestid, section) + + // payload + payload := map[string]int{"oistatusid": oistatusid} + _, err = httpservice.HttpPost(endpoint, payload) + return err } +func GetFoirequestID(db *sql.DB, foiministryrequestid int) (int, error) { + + // Query to get foirequestid + var foirequestid int + query := ` + SELECT foirequest_id + FROM public."FOIMinistryRequests" + WHERE foiministryrequestid=$1 + ORDER BY version DESC + LIMIT 1 + ` + + // Execute the query + err := db.QueryRow(query, foiministryrequestid).Scan(&foirequestid) + if err != nil { + return 0, fmt.Errorf("failed to retrieve foirequestid: %w", err) + } + + return foirequestid, err +} + func LogError(db *sql.DB, foiministryrequestid int, publishingstatus string, message string) error { // Begin a transaction diff --git a/computingservices/OpenInfoServices/lib/http/http.go b/computingservices/OpenInfoServices/lib/http/http.go new file mode 100644 index 00000000..d2db3f79 --- /dev/null +++ b/computingservices/OpenInfoServices/lib/http/http.go @@ -0,0 +1,118 @@ +package httpservice + +import ( + myconfig "OpenInfoServices/config" + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" +) + +type KeycloakTokenResponse struct { + AccessToken string `json:"access_token"` +} + +func getBearerToken() (string, error) { + keycloakurl, keycloakrealm, keycloakclientid, keycloakclientsecret, keycloakuser, keycloakpassword := myconfig.GetKeycloak() + + url := keycloakurl + "/auth/realms/" + keycloakrealm + "/protocol/openid-connect/token" + // data := "client_id=" + keycloakclientid + "&client_secret=" + keycloakclientsecret + "&grant_type=client_credentials" + data := fmt.Sprintf("client_id=%s&client_secret=%s&grant_type=password&username=%s&password=%s", keycloakclientid, keycloakclientsecret, keycloakuser, keycloakpassword) + + req, err := http.NewRequest("POST", url, bytes.NewBufferString(data)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var tokenResponse KeycloakTokenResponse + err = json.Unmarshal(body, &tokenResponse) + if err != nil { + return "", err + } + + return tokenResponse.AccessToken, nil +} + +func HttpPost(endpoint string, payload map[string]int) ([]byte, error) { + bearerToken, err0 := getBearerToken() + if err0 != nil { + log.Fatalf("Failed to get Bearer token: %v", err0) + } + + jsonPayload, err1 := json.Marshal(payload) + if err1 != nil { + return nil, err1 + } + + req, err2 := http.NewRequest("POST", endpoint, bytes.NewBuffer(jsonPayload)) + if err2 != nil { + return nil, err2 + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", bearerToken)) + + client := &http.Client{} + resp, err3 := client.Do(req) + if err3 != nil { + return nil, err3 + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return body, fmt.Errorf("received non-200 response code: %d", resp.StatusCode) + } + + return body, nil +} + +func HttpGet(endpoint string) ([]byte, error) { + bearerToken, err0 := getBearerToken() + if err0 != nil { + log.Fatalf("Failed to get Bearer token: %v", err0) + } + fmt.Printf("token: %s", bearerToken) + + req, err1 := http.NewRequest("GET", endpoint, nil) + if err1 != nil { + return nil, err1 + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", bearerToken)) + + client := &http.Client{} + resp, err2 := client.Do(req) + if err2 != nil { + return nil, err2 + } + defer resp.Body.Close() + + body, err3 := io.ReadAll(resp.Body) + if err3 != nil { + return nil, err3 + } + + if resp.StatusCode != http.StatusOK { + return body, fmt.Errorf("received non-200 response code: %d", resp.StatusCode) + } + + return body, nil +} diff --git a/computingservices/OpenInfoServices/main.go b/computingservices/OpenInfoServices/main.go index c84d2d20..f2009727 100644 --- a/computingservices/OpenInfoServices/main.go +++ b/computingservices/OpenInfoServices/main.go @@ -5,6 +5,7 @@ import ( myconfig "OpenInfoServices/config" "OpenInfoServices/lib/awslib" dbservice "OpenInfoServices/lib/db" + httpservice "OpenInfoServices/lib/http" redislib "OpenInfoServices/lib/queue" oiservices "OpenInfoServices/services" "encoding/json" @@ -23,6 +24,7 @@ const ( dateformat = "2006-01-02" openstatus_sitemap = "ready for crawling" openstatus_sitemap_message = "sitemap ready" + oistatus_published = "Published" ) var ( @@ -44,7 +46,8 @@ var ( sitemaplimit int //Others - env string + env string + foiflowapi string ) func main() { @@ -59,7 +62,7 @@ func main() { host, port, user, password, dbname = myconfig.GetDB() s3url, oibucket, oiprefix, sitemapprefix, sitemaplimit = myconfig.GetS3Path() - env, queue = myconfig.GetOthers() + env, queue, foiflowapi = myconfig.GetOthers() dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) @@ -268,9 +271,16 @@ func main() { } } + // Retrieve oipublicationstatus_id based on oistatus + var oistatusid int + err = db.QueryRow(`SELECT oistatusid FROM public."OpenInformationStatuses" WHERE name = $1 and isactive = TRUE`, oistatus_published).Scan(&oistatusid) + if err != nil { + log.Fatalf("Error retrieving oistatusid: %v", err) + } + // Update openinfo table status & sitemap_pages file name to DB for _, item := range records { - err = dbservice.UpdateOIRecordState(db, item.Foiministryrequestid, openstatus_sitemap, openstatus_sitemap_message, item.Sitemap_pages) + err = dbservice.UpdateOIRecordState(db, foiflowapi, item.Foiministryrequestid, item.Foirequestid, openstatus_sitemap, openstatus_sitemap_message, item.Sitemap_pages, oistatusid) if err != nil { log.Fatalf("%v", err) return @@ -280,42 +290,73 @@ func main() { case "test": //----- put testing script here for manual test ----- - // test unpublish - // Connect DB - db, err1 := dbservice.Conn(dsn) + // ======== test unpublish + // // Connect DB + // db, err1 := dbservice.Conn(dsn) + // if err1 != nil { + // log.Fatalf("%v", err1) + // return + // } + // defer db.Close() + + // // Create a Redis client + // rdb := redislib.CreateRedisClient() + // // Define the queue name + // queueName := queue + + // // Subscribe to the queue and read messages + // message, err := redislib.ReadMessage(rdb, queueName) + // if err != nil { + // log.Fatalf("%v", err) + // return + // } + + // fmt.Printf("Message read from queue: %s\n", message) + + // var msg oiservices.OpenInfoMessage + // err = json.Unmarshal([]byte(message), &msg) + // if err != nil { + // log.Fatalf("could not parse json string: %v", err) + // return + // } + + // fmt.Printf("openinfoid: %d\n", msg.Openinfoid) + // fmt.Printf("foiministryrequestid: %d\n", msg.Foiministryrequestid) + // fmt.Printf("published_date: %s\n", msg.Published_date) + // fmt.Printf("ID: %s, Description: %s, Published Date: %s, Contributor: %s, Applicant Type: %s, Fees: %v\n", msg.Axisrequestid, msg.Description, msg.Published_date, msg.Contributor, msg.Applicant_type, msg.Fees) + + // oiservices.Unpublish(msg, db) + + // ========== test http + // // Connect DB + // db, err0 := dbservice.Conn(dsn) + // if err0 != nil { + // log.Fatalf("%v", err0) + // return + // } + // defer db.Close() + + // fid, err := dbservice.GetFoirequestID(db, 1) + // if err != nil { + // log.Fatalf("Error: %v", err) + // } + // fmt.Printf("FOIRequestID: %d\n", fid) + + endpoint1 := fmt.Sprintf("%s/api/foiflow/programareas", foiflowapi) + getResponse, err1 := httpservice.HttpGet(endpoint1) if err1 != nil { - log.Fatalf("%v", err1) - return - } - defer db.Close() - - // Create a Redis client - rdb := redislib.CreateRedisClient() - // Define the queue name - queueName := queue - - // Subscribe to the queue and read messages - message, err := redislib.ReadMessage(rdb, queueName) - if err != nil { - log.Fatalf("%v", err) - return + log.Fatalf("Failed to make GET call: %v", err1) } - - fmt.Printf("Message read from queue: %s\n", message) - - var msg oiservices.OpenInfoMessage - err = json.Unmarshal([]byte(message), &msg) - if err != nil { - log.Fatalf("could not parse json string: %v", err) - return - } - - fmt.Printf("openinfoid: %d\n", msg.Openinfoid) - fmt.Printf("foiministryrequestid: %d\n", msg.Foiministryrequestid) - fmt.Printf("published_date: %s\n", msg.Published_date) - fmt.Printf("ID: %s, Description: %s, Published Date: %s, Contributor: %s, Applicant Type: %s, Fees: %v\n", msg.Axisrequestid, msg.Description, msg.Published_date, msg.Contributor, msg.Applicant_type, msg.Fees) - - oiservices.Unpublish(msg, db) + fmt.Printf("GET Response: %s\n", getResponse) + + // section := "oistatusid" + // endpoint2 := fmt.Sprintf("%s/foirequests/%d/ministryrequest/%d/section/%s", foiflowapi, 1, 1, section) + // postPayload := map[string]int{"oistatusid": 4} + // postResponse, err2 := httpservice.HttpPost(endpoint2, postPayload) + // if err2 != nil { + // log.Fatalf("Failed to make POST call: %v", err2) + // } + // fmt.Printf("POST Response: %s\n", postResponse) //----- test script end ----- diff --git a/computingservices/OpenInfoServices/services/messagehandler.go b/computingservices/OpenInfoServices/services/messagehandler.go index 5a25427e..19b18ade 100644 --- a/computingservices/OpenInfoServices/services/messagehandler.go +++ b/computingservices/OpenInfoServices/services/messagehandler.go @@ -26,6 +26,7 @@ const ( type OpenInfoMessage struct { Openinfoid int `json:"openinfoid"` Foiministryrequestid int `json:"foiministryrequestid"` + Foirequestid int `json:"foirequestid"` Axisrequestid string `json:"axisrequestid"` Description string `json:"description"` Published_date string `json:"published_date"` @@ -51,7 +52,7 @@ var ( func Publish(msg OpenInfoMessage, db *sql.DB) { s3url, oibucket, oiprefix, sitemapprefix, _ = myconfig.GetS3Path() - env, _ = myconfig.GetOthers() + env, _, _ = myconfig.GetOthers() oibucket := env + "-" + oibucket @@ -122,7 +123,7 @@ func Unpublish(msg OpenInfoMessage, db *sql.DB) { // Remove folder from s3 s3url, oibucket, oiprefix, sitemapprefix, _ = myconfig.GetS3Path() - env, _ = myconfig.GetOthers() + env, _, _ = myconfig.GetOthers() destBucket := env + "-" + oibucket destPrefix := oiprefix diff --git a/docker-compose.yml b/docker-compose.yml index 992f1dc4..85e45a63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -349,6 +349,13 @@ services: - OI_PREFIX=${OI_PREFIX} - SITEMAP_PREFIX=${SITEMAP_PREFIX} - SITEMAP_PAGES_LIMIT=${SITEMAP_PAGES_LIMIT} + - KEYCLOAK_URL=${KEYCLOAK_URL} + - KEYCLOAK_URL_REALM=${KEYCLOAK_URL_REALM} + - KEYCLOAK_CLIENT_ID=${KEYCLOAK_CLIENT_ID} + - KEYCLOAK_CLIENT_SECRET=${KEYCLOAK_CLIENT_SECRET} + - KEYCLOAK_USER=${KEYCLOAK_USER} + - KEYCLOAK_PASS=${KEYCLOAK_PASS} + - FOIFLOW_BASE_API_URL=${FOIFLOW_BASE_API_URL} volumes: dbdata: networks: diff --git a/openshift/templates/openinfoservice/openinfoservice-deploy.yaml b/openshift/templates/openinfoservice/openinfoservice-deploy.yaml index 846e1d51..4d24d4a8 100644 --- a/openshift/templates/openinfoservice/openinfoservice-deploy.yaml +++ b/openshift/templates/openinfoservice/openinfoservice-deploy.yaml @@ -144,6 +144,41 @@ objects: secretKeyRef: name: "${SECRETS}" key: SITEMAP_PAGES_LIMIT + - name: KEYCLOAK_URL + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: KEYCLOAK_URL + - name: KEYCLOAK_URL_REALM + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: KEYCLOAK_URL_REALM + - name: KEYCLOAK_CLIENT_ID + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: KEYCLOAK_CLIENT_ID + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: KEYCLOAK_CLIENT_SECRET + - name: KEYCLOAK_USER + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: KEYCLOAK_USER + - name: KEYCLOAK_PASS + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: KEYCLOAK_PASS + - name: FOIFLOW_BASE_API_URL + valueFrom: + secretKeyRef: + name: "${SECRETS}" + key: FOIFLOW_BASE_API_URL resources: requests: cpu: "50m"