update k8s file with new variables + remove files after being uploaded

uug-ai · Sep 2, 2024 · 081f90d · 081f90d
1 parent 0252e02
commit 081f90d
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 39 deletions.
diff --git a/.env b/.env
@@ -1,6 +1,8 @@
-# Environment variables
+# Project parameters
+PROJECT_NAME="Your_project"
+
 # Dataset parameters
-DATASET_FORMAT="base"
+DATASET_FORMAT="flat"
 DATASET_VERSION="1"
 DATASET_UPLOAD="True"
 
@@ -22,7 +24,7 @@ STORAGE_ACCESS_KEY="xxxx"
 STORAGE_SECRET_KEY="xxx"
 
 # Integration parameters
-INTEGRATION_NAME="roboflow"
+INTEGRATION_NAME="s3"
 
 # Roboflow parameters
 RBF_API_KEY="xxx"
@@ -35,9 +37,7 @@ S3_ACCESS_KEY="xxx"
 S3_SECRET_KEY="xxx"
 S3_BUCKET="xxx"
 
-# Feature parameters
-PROJECT_NAME="Your_project"
-
+# Logging parameters
 TIME_VERBOSE="True"
 LOGGING="True"
 

diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,5 @@ data/output/*
 utils/__pycache__
 run.sh
 data/video
-.env.production
+.env.production
+__pycache__
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -11,7 +11,7 @@
             "program": "${workspaceFolder}/queue_harvesting.py",
             "console": "integratedTerminal",
             "envFile": "${workspaceFolder}/.env.production",
-            "python": "${workspaceFolder}/venv/bin/python" 
+            "python": "${workspaceFolder}/.venv/bin/python" 
         },
         {
             "name": "Python Debugger: single shot",
@@ -20,7 +20,7 @@
             "program": "${workspaceFolder}/single-shot.py",
             "console": "integratedTerminal",
             "envFile": "${workspaceFolder}/.env.production",
-            "python": "${workspaceFolder}/venv/bin/python" 
+            "python": "${workspaceFolder}/.venv/bin/python" 
         }
     ]
 }

diff --git a/integrations/s3/s3_integration.py b/integrations/s3/s3_integration.py
@@ -39,7 +39,11 @@ def upload_dataset(self, src_project_path):
 
                 # Upload the file
                 self.__upload_file__(source_path, output_path)
-                print(f'Uploaded: {source_path} to s3://{self.bucket}/{output_path}')
+                print(
+                    f'Uploaded: {source_path} to s3://{self.bucket}/{output_path}')
+
+                # Remove the file after uploading
+                os.remove(source_path)
 
     def __connect__(self):
         """
@@ -72,9 +76,11 @@ def __upload_file__(self, source_path, output_path):
         """
         try:
             self.agent.upload_file(source_path, self.bucket, output_path)
-            print(f"Successfully uploaded '{source_path}' to 's3://{self.bucket}/{output_path}'")
+            print(
+                f"Successfully uploaded '{source_path}' to 's3://{self.bucket}/{output_path}'")
         except Exception as e:
-            print(f"Failed to upload '{source_path}' to 's3://{self.bucket}/{output_path}': {e}")
+            print(
+                f"Failed to upload '{source_path}' to 's3://{self.bucket}/{output_path}': {e}")
 
     def __check_bucket_exists__(self, bucket_name):
         """
@@ -92,4 +98,5 @@ def __check_bucket_exists__(self, bucket_name):
             print(f"Bucket '{bucket_name}' found.")
 
         except:
-            raise ModuleNotFoundError(f"Bucket '{bucket_name}' does not exist.")
+            raise ModuleNotFoundError(
+                f"Bucket '{bucket_name}' does not exist.")
diff --git a/k8s-deployment.yaml b/k8s-deployment.yaml
@@ -20,23 +20,28 @@ spec:
           resources:
             limits:
               nvidia.com/gpu: 1 # requesting a single GPU
+              cpu: 2
+              memory: 2Gi
           env:
-            - name: MODEL_NAME
-              value: "yolov8n.pt"
-            - name: MODEL_NAME_2
-              value: "helmet_dectector_1k_16b_150e.pt"
-            - name: MODEL_ALLOWED_CLASSES
-              value: "0"
-            - name: MODEL_2_ALLOWED_CLASSES
-              value: "0,1,2"
+            # Project parameters
+            - name: PROJECT_NAME
+              value: "helmet"
 
+            # Dataset parameters
             - name: DATASET_FORMAT
-              value: "base"
+              value: "flat"
             - name: DATASET_VERSION
               value: "1"
             - name: DATASET_UPLOAD
               value: "True"
 
+            # Forwarding
+            - name: FORWARDING_MEDIA
+              value: "False"
+            - name: REMOVE_AFTER_PROCESSED
+              value: "False"
+
+            # Queue parameters
             - name: QUEUE_NAME
               value: "data-harvesting" # This is the topic of kafka we will read messages from.
             - name: QUEUE_HOST
@@ -47,23 +52,27 @@ spec:
             - name: QUEUE_PASSWORD
               value: yourpassword
 
+            # Kerberos Vault parameters
             - name: STORAGE_URI
               value: "http://vault-lb.kerberos-vault/api"
             - name: STORAGE_ACCESS_KEY
               value: "52gyELgxutOXUWhF"
             - name: STORAGE_SECRET_KEY
               value: "k8DrcB@hQ5XfxDENzDKcnkxBHx"
 
+            # Integration parameters
             - name: INTEGRATION_NAME
               value: "s3"
 
+            # Roboflow parameters
             - name: RBF_API_KEY
               value: "YOUR KEY"
             - name: RBF_WORKSPACE
               value: "YOUR_WS"
             - name: RBF_PROJECT
               value: "YOUR_PROJ"
-
+
+            # S3 parameters
             - name: S3_ENDPOINT
               value: "YOUR_ENDPOINT"
             - name: S3_ACCESS_KEY
@@ -73,34 +82,22 @@ spec:
             - name: S3_BUCKET
               value: "YOUR_BUCKET"
 
+            # Logging parameters
             - name: LOGGING
               value: "True"
-            - name: PLOT
-              value: "False"
-            - name: SAVE_VIDEO
-              value: "False"
-            - name: MEDIA_SAVEPATH
-              value: "/ml/data/input/video.mp4"
-            - name: OUTPUT_MEDIA_SAVEPATH
-              value: "/ml/data/input/output_video.mp4"
-
             - name: TIME_VERBOSE
               value: "True"
 
+            # Classification parameters
             - name: CLASSIFICATION_FPS
-              value: "3"
+              value: "5"
             - name: CLASSIFICATION_THRESHOLD
               value: "0.25"
             - name: MAX_NUMBER_OF_PREDICTIONS
               value: "100"
             - name: FRAMES_SKIP_AFTER_DETECT
               value: "50"
-            - name: ALLOWED_CLASSIFICATIONS
-              value: "0, 1, 2, 3, 5, 7, 14, 15, 16, 24, 26, 28"
             - name: MIN_DETECTIONS
               value: "1"
             - name: IOU
-              value: "0.85"
-
-            - name: REMOVE_AFTER_PROCESSED
-              value: "True"
+              value: "0.85"