use action v0.0.2

saagie · Feb 21, 2024 · d9d93fa · d9d93fa
1 parent 1deabfd
commit d9d93fa
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 102 deletions.
diff --git a/.github/workflows/build-and-deploy-dev.yml b/.github/workflows/build-and-deploy-dev.yml
@@ -1,61 +1,32 @@
-name: Build and deploy on dev
-
-
-on:
-  push:
-    branches:
-      - '**'        # matches every branch
-      - '!main'   # excludes main
+on: [push]
 
 env:
   CI_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
 
 jobs:
-  build:
+  test_job:
     runs-on: ubuntu-latest
+    name: Test job
     steps:
       - name: checkout repo content
         uses: actions/checkout@v4 # checkout the repository content
 
-      - name: setup python
-        uses: actions/setup-python@v4
+      - name: Ci with saagie tool
+        id: update
+        uses: saagie/[email protected]
         with:
-          python-version: '3.10' # install the python version needed
-
-      - name: install required python librairies
-        run: |
-          python -m pip install --upgrade pip 
-          pip install -r cicd_saagie_tool/requirements.txt
-
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v37 # check if we have file changed
-        with:
-          files_yaml: |
-            jobs:
-              - saagie/jobs/*.json
-            pipelines:
-              - saagie/pipelines/*.json
-
-      - name: Run step if any of the jobs files above change
-        if: steps.changed-files-specific.outputs.jobs_any_changed == 'true' # check if a job config file have change
-        # then it will package the job and update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.jobs_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action package_job --job_name ${file_name_without_ext} 
-            python cicd_saagie_tool/__main__.py --action update_job --job_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}}" --saagie_env dev 
-          done
-
-      - name: Run step if any of the pipelines files above change
-        if: steps.changed-files-specific.outputs.pipelines_any_changed == 'true' # check if a pipeline config file have change
-        # then it will update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.pipelines_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action update_pipeline --pipeline_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}" --saagie_env dev
-          done
+          saagie_url: ${{secrets.SAAGIE_URL}}
+          saagie_user: ${{secrets.SAAGIE_USER}}
+          saagie_pwd: ${{secrets.SAAGIE_PWD}}
+          saagie_realm: ${{secrets.SAAGIE_REALM}}
+          saagie_env: 'dev'
+          job_config_folder: './saagie/jobs/*.json'
+          pipeline_config_folder: './saagie/pipelines/*.json'
+          env_config_folder: './saagie/envs/*.json'
+          job_source_folder: './code/jobs/*/*'
+          pipeline_source_folder: './code/pipelines/*.yaml'
+          artefact_code_folder: './dist/*/*'
+
+      # Print the output time
+      - name: Get the output time
+        run: echo "The time was ${{ steps.hello.outputs.time }}"
diff --git a/.github/workflows/build-and-deploy-prod.yml b/.github/workflows/build-and-deploy-prod.yml
@@ -1,59 +1,35 @@
-name: Build and deploy on prod
-
-
 on:
   push:
     branches:
-      - 'main'        # matches only main
+      - 'main'
+
 env:
   CI_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
 
 jobs:
-  build:
+  test_job:
     runs-on: ubuntu-latest
+    name: Test job of Saagie CI/CD tool
     steps:
       - name: checkout repo content
         uses: actions/checkout@v4 # checkout the repository content
 
-      - name: setup python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10' # install the python version needed
-
-      - name: install required python librairies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r cicd_saagie_tool/requirements.txt
-
-      - name: Get specific changed files
-        id: changed-files-specific
-        uses: tj-actions/changed-files@v37 # check if we have file changed
+      - name: Ci with saagie tool
+        id: update
+        uses: saagie/[email protected]
         with:
-          files_yaml: |
-            jobs:
-              - saagie/jobs/*.json
-            pipelines:
-              - saagie/pipelines/*.json
-
-      - name: Run step if any of the jobs files above change
-        if: steps.changed-files-specific.outputs.jobs_any_changed == 'true' # check if a job config file have change
-        # then it will package the job and update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.jobs_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action package_job --job_name ${file_name_without_ext} 
-            python cicd_saagie_tool/__main__.py --action update_job --job_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}}" --saagie_env prod 
-          done
-
-      - name: Run step if any of the pipelines files above change
-        if: steps.changed-files-specific.outputs.pipelines_any_changed == 'true' # check if a pipeline config file have change
-        # then it will update on Saagie
-        run: |
-          for file in ${{ steps.changed-files-specific.outputs.pipelines_all_changed_files }}; do
-            echo "$file was modified"
-            file_name=$(basename ${file})
-            file_name_without_ext=${file_name%.*}
-            python cicd_saagie_tool/__main__.py --action update_pipeline --pipeline_name ${file_name_without_ext} --saagie_url "${{secrets.SAAGIE_URL}}" --saagie_user "${{secrets.SAAGIE_USER}}" --saagie_pwd "${{secrets.SAAGIE_PWD}}" --saagie_realm "${{secrets.SAAGIE_REALM}}" --saagie_env prod
-          done
+          saagie_url: ${{secrets.SAAGIE_URL}}
+          saagie_user: ${{secrets.SAAGIE_USER}}
+          saagie_pwd: ${{secrets.SAAGIE_PWD}}
+          saagie_realm: ${{secrets.SAAGIE_REALM}}
+          saagie_env: 'prod'
+          job_config_folder: './saagie/jobs/*.json'
+          pipeline_config_folder: './saagie/pipelines/*.json'
+          env_config_folder: './saagie/envs/*.json'
+          job_source_folder: './code/jobs/*/*'
+          pipeline_source_folder: './code/pipelines/*.yaml'
+          artefact_code_folder: './dist/*/*'
+
+      # Print the output time
+      - name: Get the output time
+        run: echo "The time was ${{ steps.hello.outputs.time }}"
diff --git a/cicd_saagie_tool/requirements.txt b/cicd_saagie_tool/requirements.txt
@@ -1,3 +1,3 @@
-saagieapi==2.8.2
+saagieapi==2.11.0
 urllib3==1.26.15
 pyyaml==6.0.1
diff --git a/cicd_saagie_tool/utils.py b/cicd_saagie_tool/utils.py
@@ -71,11 +71,14 @@ def create_or_upgrade_job(client_saagie, job_config_file, env_config_file):
         return handle_log_error(f"Error when loading env config file: [{env_config_file}]", e)
 
     release_note = "WIP"
+    url_git = ""
     if "CI" in os.environ:
         if "GITHUB_SERVER_URL" in os.environ:
-            release_note = f"{os.environ['CI_COMMIT_MESSAGE']} - {os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/commit/{os.environ['GITHUB_SHA']}"
+            url_git = f"{os.environ['GITHUB_SERVER_URL']}/{os.environ['GITHUB_REPOSITORY']}/commit/{os.environ['GITHUB_SHA']}"
+            release_note = f"{os.environ['CI_COMMIT_MESSAGE']} - {url_git}"
         else:
-            release_note = f"{os.environ['CI_COMMIT_MESSAGE']} - {os.environ['CI_PROJECT_URL']}/-/commit/{os.environ['CI_COMMIT_SHA']}"
+            url_git = f"{os.environ['CI_PROJECT_URL']}/-/commit/{os.environ['CI_COMMIT_SHA']}"
+            release_note = f"{os.environ['CI_COMMIT_MESSAGE']}"
 
     res = client_saagie.jobs.create_or_upgrade(
         job_name=job_config["job_name"],
@@ -90,7 +93,8 @@ def create_or_upgrade_job(client_saagie, job_config_file, env_config_file):
         command_line=job_config["command_line"] if "command_line" in job_config and bool(job_config["command_line"]) else None,
         release_note=release_note,
         extra_technology=job_config["extra_technology"] if "extra_technology" in job_config and bool(job_config["extra_technology"]) else None,
-        extra_technology_version=job_config["extra_technology_version"] if "extra_technology_version" in job_config and bool(job_config["extra_technology_version"]) else None
+        extra_technology_version=job_config["extra_technology_version"] if "extra_technology_version" in job_config and bool(job_config["extra_technology_version"]) else None,
+        source_url=url_git
     )
     return res
 
@@ -260,4 +264,4 @@ def run_pipeline(client_saagie, pipeline_config_file, env_config_file):
     pipeline_id = client_saagie.pipelines.get_id(project_name=env_config["project_name"], pipeline_name=pipeline_info["pipeline_name"])
     logging.info("Pipeline ID: " + pipeline_id)
     logging.debug(f"Running pipeline: [{pipeline_id}]...")
-    return client_saagie.pipelines.run(pipeline_id)
+    return client_saagie.pipelines.run(pipeline_id)
diff --git a/code/jobs/clean_data/__main__.py b/code/jobs/clean_data/__main__.py
@@ -64,7 +64,6 @@ def write_data(train_df, test_df):
         with fs.open(path_test, 'w') as f:
             test_df.to_csv(f, sep=',', index=False)
 
-
     ## Cleansing
     train_df, test_df = read_data()
     train_df['text'] = train_df['text'].str.replace('[^\w\s]', '')