Using github action

.github/workflows/build-and-deploy-dev.yml

@@ -1,61 +1,28 @@
-name: Build and deploy on dev
-
-
-on:
-  push:
-    branches:
-      - '**'        # matches every branch
-      - '!main'   # excludes main
+on: [push]
 
 env:
   CI_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
 
 jobs:
-  build:
+  test_job:
     runs-on: ubuntu-latest
+    name: Test job
     steps:
       - name: checkout repo content
         uses: actions/checkout@v4 # checkout the repository content
 
-      - name: setup python
-        uses: actions/setup-python@v4
+      - name: Ci with saagie tool
+        id: update
+        uses: saagie/[email protected]
         with:
-          python-version: '3.10' # install the python version needed
-
-      - name: install required python librairies
-        run: |
-          python -m pip install --upgrade pip 
-          pip install -r cicd_saagie_tool/requirements.txt
-
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v37 # check if we have file changed
-        with:
-          files_yaml: |
-            jobs:
-              - saagie/jobs/*.json
-            pipelines:
-              - saagie/pipelines/*.json
-
-      - name: Run step if any of the jobs files above change
-        if: steps.changed-files-specific.outputs.jobs_any_changed == 'true' # check if a job config file have change
-        # then it will package the job and update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.jobs_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action package_job --job_name ${file_name_without_ext} 
-            python cicd_saagie_tool/__main__.py --action update_job --job_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}}" --saagie_env dev 
-          done
-
-      - name: Run step if any of the pipelines files above change
-        if: steps.changed-files-specific.outputs.pipelines_any_changed == 'true' # check if a pipeline config file have change
-        # then it will update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.pipelines_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action update_pipeline --pipeline_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}" --saagie_env dev
-          done
+          saagie_url: ${{secrets.SAAGIE_URL}}
+          saagie_user: ${{secrets.SAAGIE_USER}}
+          saagie_pwd: ${{secrets.SAAGIE_PWD}}
+          saagie_realm: ${{secrets.SAAGIE_REALM}}
+          saagie_env: 'dev'
+          job_config_folder: './saagie/jobs/*.json'
+          pipeline_config_folder: './saagie/pipelines/*.json'
+          env_config_folder: './saagie/envs/*.json'
+          job_source_folder: './code/jobs/*/*'
+          pipeline_source_folder: './code/pipelines/*.yaml'
+          artefact_code_folder: './dist/*/*'

.github/workflows/build-and-deploy-prod.yml

@@ -1,59 +1,31 @@
-name: Build and deploy on prod
-
-
 on:
   push:
     branches:
-      - 'main'        # matches only main
+      - 'main'
+
 env:
   CI_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
 
 jobs:
-  build:
+  test_job:
     runs-on: ubuntu-latest
+    name: Test job of Saagie CI/CD tool
     steps:
       - name: checkout repo content
         uses: actions/checkout@v4 # checkout the repository content
 
-      - name: setup python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10' # install the python version needed
-
-      - name: install required python librairies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r cicd_saagie_tool/requirements.txt
-
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v37 # check if we have file changed
+      - name: Ci with saagie tool
+        id: update
+        uses: saagie/[email protected]
         with:
-          files_yaml: |
-            jobs:
-              - saagie/jobs/*.json
-            pipelines:
-              - saagie/pipelines/*.json
-
-      - name: Run step if any of the jobs files above change
-        if: steps.changed-files-specific.outputs.jobs_any_changed == 'true' # check if a job config file have change
-        # then it will package the job and update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.jobs_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action package_job --job_name ${file_name_without_ext} 
-            python cicd_saagie_tool/__main__.py --action update_job --job_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}}" --saagie_env prod 
-          done
-
-      - name: Run step if any of the pipelines files above change
-        if: steps.changed-files-specific.outputs.pipelines_any_changed == 'true' # check if a pipeline config file have change
-        # then it will update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.pipelines_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action update_pipeline --pipeline_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}" --saagie_env prod
-          done
+          saagie_url: ${{secrets.SAAGIE_URL}}
+          saagie_user: ${{secrets.SAAGIE_USER}}
+          saagie_pwd: ${{secrets.SAAGIE_PWD}}
+          saagie_realm: ${{secrets.SAAGIE_REALM}}
+          saagie_env: 'prod'
+          job_config_folder: './saagie/jobs/*.json'
+          pipeline_config_folder: './saagie/pipelines/*.json'
+          env_config_folder: './saagie/envs/*.json'
+          job_source_folder: './code/jobs/*/*'
+          pipeline_source_folder: './code/pipelines/*.yaml'
+          artefact_code_folder: './dist/*/*'

saagie/jobs/clean_data.json

@@ -1,7 +1,7 @@
 {
     "job_name": "2 Clean data",
     "file_path": "dist/clean_data/clean_data.zip",
-    "description": "new test 4",
+    "description": "new test",
     "category": "Processing",
     "technology": "python",
     "technology_catalog": "Saagie",