From 3bd05280f86d48490953d1f1db380d411bd5e3be Mon Sep 17 00:00:00 2001 From: abin-aot Date: Wed, 25 Dec 2024 00:47:56 +0000 Subject: [PATCH] #FOIMOD-3636 PUSH TO SOLR updates --- .../src/.vscode/launch.json | 15 ++++++ .../src/messageprocessor.go | 27 +++++++++- .../solrsearchservices/solrsearchservice.go | 54 +++++++++++++++++++ .../src/types/FOISolrSearchStruct.go | 13 +++++ 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 computingservices/documentextractservices/azuredocextractservice/src/.vscode/launch.json create mode 100644 computingservices/documentextractservices/azuredocextractservice/src/solrsearchservices/solrsearchservice.go create mode 100644 computingservices/documentextractservices/azuredocextractservice/src/types/FOISolrSearchStruct.go diff --git a/computingservices/documentextractservices/azuredocextractservice/src/.vscode/launch.json b/computingservices/documentextractservices/azuredocextractservice/src/.vscode/launch.json new file mode 100644 index 0000000..8655150 --- /dev/null +++ b/computingservices/documentextractservices/azuredocextractservice/src/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${fileDirname}" + } + ] +} \ No newline at end of file diff --git a/computingservices/documentextractservices/azuredocextractservice/src/messageprocessor.go b/computingservices/documentextractservices/azuredocextractservice/src/messageprocessor.go index 6540a02..f17a482 100644 --- a/computingservices/documentextractservices/azuredocextractservice/src/messageprocessor.go +++ b/computingservices/documentextractservices/azuredocextractservice/src/messageprocessor.go @@ -4,10 +4,12 @@ import ( "azuredocextractservice/azureservices" "azuredocextractservice/httpservices" "azuredocextractservice/s3services" + "azuredocextractservice/solrsearchservices" "azuredocextractservice/types" "fmt" "log" "net/url" + "strconv" "strings" "time" ) @@ -36,9 +38,30 @@ func main() { var parsedURL = document.DocumentS3URL var jsonStrbytes []byte = getBytesfromDocumentPath(parsedURL) analysisResults, _analyzeerr := azureservices.CallAzureDocument(jsonStrbytes) - if _analyzeerr != nil { + if _analyzeerr == nil && analysisResults.Status == "succeeded" { + + searchdocumentpagelines := []types.SOLRSearchDocument{} //pUSH to solr. - //analysisResults.AnalyzeResult.Pages + for _, page := range analysisResults.AnalyzeResult.Pages { + for _, line := range page.Lines { + _solrsearchdocuemnt := types.SOLRSearchDocument{ + FoiDocumentID: strconv.Itoa(int(document.DocumentID)), + FoiRequestNumber: request.RequestNumber, + FoiMinistryRequestID: request.MinistryRequestID, + FoiMinistryCode: request.MinistryCode, + FoiDocumentFileName: document.DocumentName, + FoiDocumentPageNumber: page.PageNumber, + FoiDocumentSentence: line.Content, + FoiRequestMiscInfo: document.DocumentS3URL, + } + searchdocumentpagelines = append(searchdocumentpagelines, _solrsearchdocuemnt) + fmt.Println(_solrsearchdocuemnt.FoiDocumentFileName) + } + + } + + solrsearchservices.PushtoSolr(searchdocumentpagelines) + } } } diff --git a/computingservices/documentextractservices/azuredocextractservice/src/solrsearchservices/solrsearchservice.go b/computingservices/documentextractservices/azuredocextractservice/src/solrsearchservices/solrsearchservice.go new file mode 100644 index 0000000..b03627c --- /dev/null +++ b/computingservices/documentextractservices/azuredocextractservice/src/solrsearchservices/solrsearchservice.go @@ -0,0 +1,54 @@ +package solrsearchservices + +import ( + "azuredocextractservice/types" + "azuredocextractservice/utils" + "bytes" + "encoding/json" + "fmt" + "log" + "net/http" +) + +func PushtoSolr(searchdocs []types.SOLRSearchDocument) bool { + + // Convert the struct to JSON + jsonData, err := json.Marshal(searchdocs) + fmt.Println("SOLR Search Data starts here") + fmt.Println(string(jsonData)) + fmt.Println("SOLR Search Data ends here") + if err != nil { + log.Fatal("Error marshaling JSON:", err) + } + + // Solr endpoint URL (Replace with your Solr endpoint) + url := utils.ViperEnvVariable("foisearchsolrpostendpoint") + // Create a POST request with JSON data + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + log.Fatal("Error creating request:", err) + } + + // Set the appropriate headers for JSON content + req.Header.Set("Content-Type", "application/json") + username := utils.ViperEnvVariable("solradmin") + password := utils.ViperEnvVariable("solrpassword") + req.SetBasicAuth(username, password) + // Send the request using the http client + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Fatal("Error sending request:", err) + } + defer resp.Body.Close() + + // Handle the response + if resp.StatusCode == http.StatusOK { + fmt.Println("Successfully posted to Solr") + return true + } else { + fmt.Printf("Failed to post to Solr. Status: %s\n", resp.Status) + return false + } + +} diff --git a/computingservices/documentextractservices/azuredocextractservice/src/types/FOISolrSearchStruct.go b/computingservices/documentextractservices/azuredocextractservice/src/types/FOISolrSearchStruct.go new file mode 100644 index 0000000..4f779e2 --- /dev/null +++ b/computingservices/documentextractservices/azuredocextractservice/src/types/FOISolrSearchStruct.go @@ -0,0 +1,13 @@ +package types + +type SOLRSearchDocument struct { + FoiDocumentID string `json:"foidocumentid"` + FoiRequestNumber string `json:"foirequestnumber"` + FoiMinistryRequestID string `json:"foiministryrequestid"` + FoiMinistryCode string `json:"foiministrycode"` + FoiDocumentFileName string `json:"foidocumentfilename"` + FoiDocumentPageNumber int `json:"foidocumentpagenumber"` + FoiDocumentSentence string `json:"foidocumentsentence"` + FoiDocumentSentenceID int `json:"foidocumentsentenceId"` + FoiRequestMiscInfo string `json:"foirequestmiscinfo"` +}