From 081f90dc7950b2b5cfa8e39124204403e97be3a1 Mon Sep 17 00:00:00 2001 From: Cedric Verstraeten Date: Mon, 2 Sep 2024 20:30:49 +0200 Subject: [PATCH] update k8s file with new variables + remove files after being uploaded --- .env | 12 ++++---- .gitignore | 3 +- .vscode/launch.json | 4 +-- integrations/s3/s3_integration.py | 15 +++++++--- k8s-deployment.yaml | 49 +++++++++++++++---------------- 5 files changed, 44 insertions(+), 39 deletions(-) diff --git a/.env b/.env index bfe13a2..3efeafa 100644 --- a/.env +++ b/.env @@ -1,6 +1,8 @@ -# Environment variables +# Project parameters +PROJECT_NAME="Your_project" + # Dataset parameters -DATASET_FORMAT="base" +DATASET_FORMAT="flat" DATASET_VERSION="1" DATASET_UPLOAD="True" @@ -22,7 +24,7 @@ STORAGE_ACCESS_KEY="xxxx" STORAGE_SECRET_KEY="xxx" # Integration parameters -INTEGRATION_NAME="roboflow" +INTEGRATION_NAME="s3" # Roboflow parameters RBF_API_KEY="xxx" @@ -35,9 +37,7 @@ S3_ACCESS_KEY="xxx" S3_SECRET_KEY="xxx" S3_BUCKET="xxx" -# Feature parameters -PROJECT_NAME="Your_project" - +# Logging parameters TIME_VERBOSE="True" LOGGING="True" diff --git a/.gitignore b/.gitignore index 13d9303..8dbcc7f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ data/output/* utils/__pycache__ run.sh data/video -.env.production \ No newline at end of file +.env.production +__pycache__ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 0d0309a..156abdb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "program": "${workspaceFolder}/queue_harvesting.py", "console": "integratedTerminal", "envFile": "${workspaceFolder}/.env.production", - "python": "${workspaceFolder}/venv/bin/python" + "python": "${workspaceFolder}/.venv/bin/python" }, { "name": "Python Debugger: single shot", @@ -20,7 +20,7 @@ "program": "${workspaceFolder}/single-shot.py", "console": "integratedTerminal", "envFile": "${workspaceFolder}/.env.production", - "python": "${workspaceFolder}/venv/bin/python" + "python": "${workspaceFolder}/.venv/bin/python" } ] } diff --git a/integrations/s3/s3_integration.py b/integrations/s3/s3_integration.py index eb0e62d..db1bc00 100644 --- a/integrations/s3/s3_integration.py +++ b/integrations/s3/s3_integration.py @@ -39,7 +39,11 @@ def upload_dataset(self, src_project_path): # Upload the file self.__upload_file__(source_path, output_path) - print(f'Uploaded: {source_path} to s3://{self.bucket}/{output_path}') + print( + f'Uploaded: {source_path} to s3://{self.bucket}/{output_path}') + + # Remove the file after uploading + os.remove(source_path) def __connect__(self): """ @@ -72,9 +76,11 @@ def __upload_file__(self, source_path, output_path): """ try: self.agent.upload_file(source_path, self.bucket, output_path) - print(f"Successfully uploaded '{source_path}' to 's3://{self.bucket}/{output_path}'") + print( + f"Successfully uploaded '{source_path}' to 's3://{self.bucket}/{output_path}'") except Exception as e: - print(f"Failed to upload '{source_path}' to 's3://{self.bucket}/{output_path}': {e}") + print( + f"Failed to upload '{source_path}' to 's3://{self.bucket}/{output_path}': {e}") def __check_bucket_exists__(self, bucket_name): """ @@ -92,4 +98,5 @@ def __check_bucket_exists__(self, bucket_name): print(f"Bucket '{bucket_name}' found.") except: - raise ModuleNotFoundError(f"Bucket '{bucket_name}' does not exist.") + raise ModuleNotFoundError( + f"Bucket '{bucket_name}' does not exist.") diff --git a/k8s-deployment.yaml b/k8s-deployment.yaml index 8a3dbb8..4aea052 100644 --- a/k8s-deployment.yaml +++ b/k8s-deployment.yaml @@ -20,23 +20,28 @@ spec: resources: limits: nvidia.com/gpu: 1 # requesting a single GPU + cpu: 2 + memory: 2Gi env: - - name: MODEL_NAME - value: "yolov8n.pt" - - name: MODEL_NAME_2 - value: "helmet_dectector_1k_16b_150e.pt" - - name: MODEL_ALLOWED_CLASSES - value: "0" - - name: MODEL_2_ALLOWED_CLASSES - value: "0,1,2" + # Project parameters + - name: PROJECT_NAME + value: "helmet" + # Dataset parameters - name: DATASET_FORMAT - value: "base" + value: "flat" - name: DATASET_VERSION value: "1" - name: DATASET_UPLOAD value: "True" + # Forwarding + - name: FORWARDING_MEDIA + value: "False" + - name: REMOVE_AFTER_PROCESSED + value: "False" + + # Queue parameters - name: QUEUE_NAME value: "data-harvesting" # This is the topic of kafka we will read messages from. - name: QUEUE_HOST @@ -47,6 +52,7 @@ spec: - name: QUEUE_PASSWORD value: yourpassword + # Kerberos Vault parameters - name: STORAGE_URI value: "http://vault-lb.kerberos-vault/api" - name: STORAGE_ACCESS_KEY @@ -54,16 +60,19 @@ spec: - name: STORAGE_SECRET_KEY value: "k8DrcB@hQ5XfxDENzDKcnkxBHx" + # Integration parameters - name: INTEGRATION_NAME value: "s3" + # Roboflow parameters - name: RBF_API_KEY value: "YOUR KEY" - name: RBF_WORKSPACE value: "YOUR_WS" - name: RBF_PROJECT value: "YOUR_PROJ" - + + # S3 parameters - name: S3_ENDPOINT value: "YOUR_ENDPOINT" - name: S3_ACCESS_KEY @@ -73,34 +82,22 @@ spec: - name: S3_BUCKET value: "YOUR_BUCKET" + # Logging parameters - name: LOGGING value: "True" - - name: PLOT - value: "False" - - name: SAVE_VIDEO - value: "False" - - name: MEDIA_SAVEPATH - value: "/ml/data/input/video.mp4" - - name: OUTPUT_MEDIA_SAVEPATH - value: "/ml/data/input/output_video.mp4" - - name: TIME_VERBOSE value: "True" + # Classification parameters - name: CLASSIFICATION_FPS - value: "3" + value: "5" - name: CLASSIFICATION_THRESHOLD value: "0.25" - name: MAX_NUMBER_OF_PREDICTIONS value: "100" - name: FRAMES_SKIP_AFTER_DETECT value: "50" - - name: ALLOWED_CLASSIFICATIONS - value: "0, 1, 2, 3, 5, 7, 14, 15, 16, 24, 26, 28" - name: MIN_DETECTIONS value: "1" - name: IOU - value: "0.85" - - - name: REMOVE_AFTER_PROCESSED - value: "True" \ No newline at end of file + value: "0.85" \ No newline at end of file