Adding dbt cli

technologies/job/dbt/README.md

@@ -0,0 +1,18 @@
+# DBT CLI
+
+With the dbt cli technology, you can build/run/test your dbt project just by uploading a zip containing your dbt project.
+
+## Description
+
+This technology contains the `dbt-core` library as well as the adapters for : 
+- Redshift 
+- Big Query
+- Snowflake
+- PostgreSQL
+- Spark
+
+The default behavior of this technology is to unzip the archive containing the dbt project you uploaded in Saagie and to run this project with the following command
+```
+dbt run --profiles-dir .
+```
+Feel free to update the command line with whatever dbt command you need to launch.

technologies/job/dbt/dbt-1.2/Dockerfile

@@ -0,0 +1,133 @@
+ARG build_for=linux/amd64
+
+FROM --platform=$build_for python:3.10.7-slim-bullseye as base
+
+
+ARG [email protected]
+ARG [email protected]
+ARG [email protected]
+ARG [email protected]
+ARG [email protected]
+ARG [email protected]
+# special case args
+ARG dbt_spark_version=all
+ARG dbt_third_party
+
+# Update and install system packages
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    git \
+    ssh-client \
+    software-properties-common \
+    make \
+    unzip \
+    build-essential \
+    ca-certificates \
+    libpq-dev \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
+# Install DBT
+# Env vars
+ENV PYTHONIOENCODING=utf-8
+ENV LANG=C.UTF-8
+
+# Update python
+RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir
+
+# Set docker basics
+WORKDIR /usr/app/dbt/
+VOLUME /usr/app
+ENTRYPOINT ["dbt"]
+
+##
+# dbt-core
+##
+FROM base as dbt-core
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_core_ref}#egg=dbt-core&subdirectory=core"
+
+##
+# dbt-postgres
+##
+FROM base as dbt-postgres
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_postgres_ref}#egg=dbt-postgres&subdirectory=plugins/postgres"
+
+
+##
+# dbt-redshift
+##
+FROM base as dbt-redshift
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_redshift_ref}#egg=dbt-redshift"
+
+
+##
+# dbt-bigquery
+##
+FROM base as dbt-bigquery
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_bigquery_ref}#egg=dbt-bigquery"
+
+
+##
+# dbt-snowflake
+##
+FROM base as dbt-snowflake
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake"
+
+##
+# dbt-spark
+##
+FROM base as dbt-spark
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    python-dev \
+    libsasl2-dev \
+    gcc \
+    unixodbc-dev \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_spark_ref}#egg=dbt-spark[${dbt_spark_version}]"
+
+
+##
+# dbt-third-party
+##
+FROM dbt-core as dbt-third-party
+RUN python -m pip install --no-cache-dir "${dbt_third_party}"
+
+##
+# dbt-all
+##
+FROM base as dbt-all
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    python-dev \
+    libsasl2-dev \
+    gcc \
+    unixodbc-dev \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_redshift_ref}#egg=dbt-redshift"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_bigquery_ref}#egg=dbt-bigquery"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_spark_ref}#egg=dbt-spark[${dbt_spark_version}]"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_postgres_ref}#egg=dbt-postgres&subdirectory=plugins/postgres"
+
+# Move scripts and frequently changing directive to the end of the build
+COPY resources/entrypoint.sh /entrypoint.sh
+RUN chmod 755 /entrypoint.sh
+
+WORKDIR /sandbox
+
+ENTRYPOINT  ["bash","/entrypoint.sh"]

technologies/job/dbt/dbt-1.2/resources/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if compgen -G "*.zip*" > /dev/null;
+then
+    unzip -q *.zip
+else
+    echo "ERROR : You must submit a zip file containing your dbt project"
+fi
+
+if test -f main_script;
+then sh ./main_script;
+else exec "$@"
+fi;

technologies/job/dbt/metadata.yaml

@@ -0,0 +1,30 @@
+version: "v1"
+type: JOB
+id: dbt-cli
+label: dbt
+description: dbt-cli is a transformation workflow that lets teams quickly and collaboratively deploy analytics code following software engineering best practices like modularity, portability, CI/CD, and documentation.
+available: true
+icon: dbt
+contexts:
+  - id: dbt-cli
+    label: dbt-cli 1.2
+    available: true
+    recommended: true
+    trustLevel: stable
+    job:
+      features:
+        - type: COMMAND_LINE
+          label: Command line
+          mandatory: true
+          comment: Linux shell command to launch your dbt project (dbt run, dbt test...).
+          defaultValue: dbt run --profiles-dir .
+        - type: ARTIFACT
+          label: Package containing your dbt project
+          mandatory: true
+          comment: "Compatible upload file : .zip"
+        - type: SCHEDULER
+          label: Scheduled
+          mandatory: true
+    dockerInfo:
+      image: saagie/dbt
+      version: 1.2-0.3