Merge pull request #1244 from bcgov/dev-rook-RQ-FOIMOD-3551

skip deleted additional files

computingservices/OpenInfoServices/lib/awslib/s3.go

@@ -67,6 +67,7 @@ type AdditionalFile struct {
 	Additionalfileid int    `json:"additionalfileid"`
 	Filename         string `json:"filename"`
 	S3uripath        string `json:"s3uripath"`
+	Isactive         bool   `json:"isactive"`
 }
 
 type resolverV2 struct{}
@@ -139,8 +140,11 @@ func ScanS3(openInfoBucket string, openInfoPrefix string, urlPrefix string, file
 
 		// Get the file name
 		base := path.Base(filePath)
-		originalFileName, found := getOriginalName(filemappings, base)
+		originalFileName, found, isactive := getOriginalName(filemappings, base)
 		if found {
+			if !isactive { // Skip over the deleted files (isactive = false)
+				continue
+			}
 			base = originalFileName
 		}
 		// fmt.Printf("Base %s\n", base)
@@ -211,8 +215,11 @@ func CopyS3(sourceBucket string, sourcePrefix string, filemappings []AdditionalF
 
 		// Get the file name
 		base := path.Base(sourceKey)
-		originalFileName, found := getOriginalName(filemappings, base)
+		originalFileName, found, isactive := getOriginalName(filemappings, base)
 		if found {
+			if !isactive { // Skip over the deleted files (isactive = false)
+				continue
+			}
 			base = originalFileName
 		}
 		// fmt.Printf("Base %s\n", base)
@@ -404,7 +411,7 @@ func contains(arr []string, str string) bool {
 }
 
 // Get Original Filename
-func getOriginalName(filemappings []AdditionalFile, key string) (string, bool) {
+func getOriginalName(filemappings []AdditionalFile, key string) (string, bool, bool) {
 	for _, item := range filemappings {
 		// Parse the URL
 		parsedURL, err := url.Parse(item.S3uripath)
@@ -420,8 +427,8 @@ func getOriginalName(filemappings []AdditionalFile, key string) (string, bool) {
 		base := path.Base(urlPath)
 
 		if base == key {
-			return item.Filename, true
+			return item.Filename, true, item.Isactive
 		}
 	}
-	return "", false
+	return "", false, true
 }

computingservices/OpenInfoServices/lib/db/dbservices.go

@@ -24,6 +24,7 @@ type AdditionalFile struct {
 	Additionalfileid int
 	Filename         string
 	S3uripath        string
+	Isactive         bool
 }
 
 type OpenInfoRecord struct {
@@ -119,7 +120,8 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) {
 			'publish' as type,
 			oifiles.additionalfileid,
 			oifiles.filename,
-			oifiles.s3uripath
+			oifiles.s3uripath,
+			oifiles.isactive
 		FROM public."FOIMinistryRequests" mr
 		INNER JOIN public."FOIRequests" r on mr.foirequest_id = r.foirequestid and mr.foirequestversion_id = r.version
 		INNER JOIN public."ProgramAreas" pa on mr.programareaid = pa.programareaid
@@ -149,6 +151,7 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) {
 		var openinfoid, foiministryrequestid, additionalfileid sql.NullInt64
 		var axisrequestid, description, published_date, contributor, applicant_type, bcgovcode, sitemap_pages, queuetype, filename, s3uripath sql.NullString
 		var fees sql.NullFloat64
+		var isactive bool
 
 		// err := rows.Scan(
 		// 	&record.Openinfoid,
@@ -178,6 +181,7 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) {
 			&additionalfileid,
 			&filename,
 			&s3uripath,
+			&isactive,
 		)
 		if err != nil {
 			return records, fmt.Errorf("failed to retrieve query result for prepublish: %w", err)
@@ -205,6 +209,7 @@ func GetOIRecordsForPrePublishing(db *sql.DB) ([]OpenInfoRecord, error) {
 					Additionalfileid: int(additionalfileid.Int64),
 					Filename:         filename.String,
 					S3uripath:        s3uripath.String,
+					Isactive:         isactive,
 				})
 			}
 		}