diff --git a/automation/jinja2/compute_deps.py b/automation/jinja2/compute_deps.py index ac8b841..2a812e5 100644 --- a/automation/jinja2/compute_deps.py +++ b/automation/jinja2/compute_deps.py @@ -30,13 +30,17 @@ def build_exec_plan(plans:list): exec_plan = [] for plan in plans: schedule(exec_plan, plan, default_exec_order) - for plan in plans: - plan_exec_order = get_current_exec_order(exec_plan, plan_name=plan.get('name')) - for plan_dependency_name in plan.get('depends_on',[]): - dependency_exec_order = get_current_exec_order(exec_plan=exec_plan,plan_name = plan_dependency_name) - if dependency_exec_order >= plan_exec_order: - schedule(exec_plan=exec_plan, plan = plan, exec_order= dependency_exec_order + 1) - plan_exec_order = dependency_exec_order + 1 + changed = True + while changed: + changed = False + for plan in plans: + plan_exec_order = get_current_exec_order(exec_plan, plan_name=plan.get('name')) + for plan_dependency_name in plan.get('depends_on',[]): + dependency_exec_order = get_current_exec_order(exec_plan=exec_plan,plan_name = plan_dependency_name) + if dependency_exec_order >= plan_exec_order: + schedule(exec_plan=exec_plan, plan = plan, exec_order= dependency_exec_order + 1) + plan_exec_order = dependency_exec_order + 1 + changed = True return exec_plan def print_exec_plan(exec_plan:list): diff --git a/automation/jinja2/render.py b/automation/jinja2/render.py index 36e157a..2e02204 100755 --- a/automation/jinja2/render.py +++ b/automation/jinja2/render.py @@ -57,6 +57,11 @@ def rend_template(self): 'name': var_data['plans'][idx]} # this is for backward compatiblity exec_plan = compute_deps.build_exec_plan(plans=var_data['plans']) var_data['exec_plan'] = exec_plan + ## reorganize plans order based on execution order + var_data['plans'] = [] + for exec_batch in exec_plan: + for plan in exec_batch: + var_data['plans'].append(plan) # build final variables data = env_data for k, v in var_data.items(): diff --git a/automation/jinja2/templates/.gitlab-ci.yml.j2 b/automation/jinja2/templates/.gitlab-ci.yml.j2 index 5c84b3f..94f3427 100644 --- a/automation/jinja2/templates/.gitlab-ci.yml.j2 +++ b/automation/jinja2/templates/.gitlab-ci.yml.j2 @@ -78,7 +78,12 @@ stages: - quality-checks - drift - init - - plan-apply-delete + {% for exec_batch in exec_plan %} + - plan-apply-{{ loop.index }} + {% endfor %} + {% for exec_batch in exec_plan %} + - delete-{{ loop.index }} + {% endfor %} - clean-cache ######################################################################################################################## @@ -254,26 +259,17 @@ stages: needs: [{% if GITLAB_JOBS["aws-creds"] %}"aws-creds", {% endif %}"terraform-prepare"] allow_failure: false -# .validate_job: &validate_job -# extends: .terraform-base -# stage: quality-checks -# needs: [{% if GITLAB_JOBS["aws-creds"] %}"aws-creds", {% endif %}"terraform-prepare"] -# allow_failure: false - .plan_job: extends: .terraform-base - stage: plan-apply-delete allow_failure: false .apply_job: extends: .terraform-base - stage: plan-apply-delete allow_failure: false when: manual .delete_job: extends: .terraform-base - stage: plan-apply-delete allow_failure: false when: manual rules: @@ -284,10 +280,8 @@ stages: ######################################################################################################################## {% if GITLAB_JOBS["aws-creds"] %} aws-creds: - # extends: .aws-cli extends: .aws_get_creds stage: prepare - # <<: *aws_get_creds {% endif %} terraform-prepare: @@ -453,24 +447,7 @@ terraform-trivy_{{ plan_slug }}: <<: *rules_{{ plan_slug }} {% endif %} - -# ######################################################################################################################## -# # VALIDATE -# ######################################################################################################################## - -# validate_{{ plan_slug }}: -# extends: .validate_job -# script: -# - make init_{{ plan_slug }} -# - make validate_{{ plan_slug }} -# cache: -# key: tf-$CI_COMMIT_REF_SLUG -# paths: -# - .terraform.d/plugin-cache/ -# - {{ plan_name }}/.terraform -# - {{ plan_name }}/.terraform.lock.hcl -# <<: *rules_{{ plan_slug }} -# {% endfor %} +{% endfor %} {# end plans #} {% if GITLAB_JOBS["sonarqube"] %} sonarqube: @@ -551,15 +528,19 @@ init_{{ plan_slug }}: {% endfor %} +{% for exec_batch in exec_plan %} +{% set stage_name = 'plan-apply-'~ loop.index %} + ######################################################################################################################## -# TERRAFORM PLAN +# TERRAFORM PLAN-APPLY BATCH {{ loop.index }} ######################################################################################################################## -{% for plan in plans %} +{% for plan in exec_batch %} {% set plan_name = plan['name'] if 'name' in plan else plan %} {% set plan_slug = plan_name | replace('/','_') %} plan_{{ plan_slug }}: extends: .plan_job + stage: {{ stage_name }} needs: {% if GITLAB_JOBS["aws-creds"] %} - aws-creds @@ -587,15 +568,12 @@ plan_{{ plan_slug }}: {% endfor %} -######################################################################################################################## -# TERRAFORM APPLY -######################################################################################################################## - -{% for plan in plans %} +{% for plan in exec_batch %} {% set plan_name = plan['name'] if 'name' in plan else plan %} {% set plan_slug = plan_name | replace('/','_') %} apply_{{ plan_slug }}: extends: .apply_job + stage: {{ stage_name }} needs: {% if GITLAB_JOBS["aws-creds"] %} - aws-creds @@ -621,21 +599,24 @@ apply_{{ plan_slug }}: {% endfor %} +{% endfor %} {# exec_plan #} ######################################################################################################################## # DELETE ######################################################################################################################## -{% set plans_delete = plans | reverse %} -{% for plan in plans_delete +%} +{% set delete_exec_plan = exec_plan | reverse %} +{% for exec_batch in delete_exec_plan %} +{% set delete_stage_name = 'delete-' ~ loop.index %} +{% for plan in exec_batch +%} {% set plan_name = plan['name'] if 'name' in plan else plan %} {% set plan_slug = plan_name | replace('/','_') %} delete_{{ plan_slug }}: extends: .delete_job + stage: {{ delete_stage_name }} needs: {% if GITLAB_JOBS["aws-creds"] %} - aws-creds {% endif %} - - plan_{{ plan_slug }} cache: key: tf-$CI_COMMIT_REF_SLUG paths: @@ -646,4 +627,5 @@ delete_{{ plan_slug }}: - make init_{{ plan_slug }} - make destroyauto_{{ plan_slug }} -{% endfor %} +{% endfor %} {# exec_batch #} +{% endfor %} {# delete_exec_plan #} \ No newline at end of file diff --git a/configure.yaml.dist b/configure.yaml.dist index bb9cdb1..e2633f8 100644 --- a/configure.yaml.dist +++ b/configure.yaml.dist @@ -181,6 +181,8 @@ plans: # - name: terraform/demo3 # # (optional) layer specific 'plan' additionals parameters # additional_var_parameters: "-var-file=../common.tfvars" +# depends_on: +# - terraform/demo2 # AWS Account ID in which your plan are deployed and containing the backend bucket # SSO Account diff --git a/docs/installation.md b/docs/installation.md index 70783a1..2095932 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -431,6 +431,30 @@ Don't forget to generate files make generate ``` +## Add dependencies between plans + +You can define explicite dependencies between your plans using the key work `depends_on` just like in the example below: + +```yaml +... +plans: +- terraform/network +- name: terraform/compute + depends_on: + - terraform/network + - terraform/storage +- name: terraform/storage + depends_on: + - terraform/network +- name: terraform/security +... +``` + +Defining explicit dependency helps organization gitlab jobs in **execution batch**. Execution batch is a stage with attached jobs which doesn't have dependencies with each other within the same stage but have dependencies with the previous stage if any. + +This dependency definition also helps generate to correct plan destruction order. + + # Update AWSTerraformStarterKit 1. Download `remove-starter-kit.sh`, make it executable and execute the shell script.