spark-cluster-setup.yml

---
- name: Create a Spark cluster
  hosts: local
  connection: local
  gather_facts: no
  vars:
    instance_tag: "{{ spark_tag }}"
    instance_count: "{{ spark_worker_per_az_count }}"
    instance_function: spark
    az_list: "{{ cluster_az_list }}"
    instance_type: "{{ spark_instance_type }}"
  tasks:
  - name: Import configuration properties
    include_vars: 
      dir: vars

  - name: "stat {{ feature_key_path }}"
    local_action: "stat path={{ feature_key_path }}"
    become: no
    register: feature_key_meta

  - name: "Check feature key {{ feature_key_path }} exists"
    fail:
      msg: "Specified feature key {{ feature_key_path }} not found - required to use Spark connector"
    when: not feature_key_meta.stat.exists    
    
  - name: "Setup {{ instance_function }} instance vars"
    include_vars: modules/instance-setup-vars.yml

  - name: "Setup {{ instance_function }} instances"
    import_tasks: modules/instance-setup.yml

  - meta: refresh_inventory

- name: Set up Spark
  hosts: "{{ spark_tag }}"
  gather_facts: false
  become: no
  remote_user: "{{ os_config['remote_user'] }}"
  vars_files:
  - vars/cluster-config.yml
  - vars/constants.yml
  - vars/os-level-config.yml  
  tasks:
  - name: Import configuration properties
    include_vars: 
      dir: vars

  - block:
    - name: If Ubuntu, update package cache
      shell: |
          sudo apt update
      when: operating_system == ubuntu_os

  - name: Install Java
    package:
      name:
        - "{{ os_config['java_package_name'] }}"
      state: present
    become: yes

  - name: Install Scala
    shell: |
      set -e    
      if [ -z $(which scala 2>/dev/null ) ]
      then
        cd /tmp
        wget https://downloads.typesafe.com/scala/{{ scala_version }}/scala-{{ scala_version }}.tgz
        tar xzf "scala-{{ scala_version }}.tgz"
        mkdir "{{ scala_home }}"
        rm "/tmp/scala-{{ scala_version }}/bin/"*.bat
        mv "/tmp/scala-{{ scala_version }}/bin" "/tmp/scala-{{ scala_version }}/lib" "{{ scala_home }}"
        ln -s "{{ scala_home }}/bin/"* "/usr/bin/"
        rm "/tmp/scala-{{ scala_version }}.tgz"
        rm -rf "/tmp/scala-{{ scala_version }}"
      fi
    become: yes

  - name: Install Spark
    shell: |
      set -e    
      mkdir -p /etc/aerospike    
      if [ ! -d /spark ]
      then
        cd /tmp
        wget https://downloads.apache.org/spark/spark-{{ spark_version }}/spark-{{ spark_version }}-bin-hadoop{{ hadoop_version}}.tgz
        tar xvfz spark-{{ spark_version }}-bin-hadoop{{ hadoop_version}}.tgz
        mv spark-{{ spark_version }}-bin-hadoop{{ hadoop_version}} /spark
        rm spark-{{ spark_version }}-bin-hadoop{{ hadoop_version}}.tgz
      fi
    become: yes

  - name: Install Aerospike Spark Connect
    shell: |
      set -e
      if [ ! -f /spark/jars/ aerospike-spark-{{ aerospike_spark_connect_version }}_spark_{{ spark_version }}_{{ hadoop_version }}_allshaded.jar ]
      then
        cd /tmp
        wget https://www.aerospike.com/enterprise/download/connectors/aerospike-spark/{{ aerospike_spark_connect_version }}/artifact/jar -O aerospike-spark-{{ aerospike_spark_connect_version }}_spark_{{ spark_version }}_{{ hadoop_version }}_allshaded.jar
        mv aerospike-spark-{{ aerospike_spark_connect_version }}_spark_{{ spark_version }}_{{ hadoop_version }}_allshaded.jar /spark/jars/
      fi
    become: yes

  - name: Copy feature key
    copy:
      src: "{{ feature_key_path }}"
      dest: /etc/aerospike/features.conf
      owner: root
      group: root
      mode: '0644'        
    become: yes

- name: Start Master
  hosts: "{{ spark_tag }}[0]"
  gather_facts: false
  become: no
  remote_user: "{{ os_config['remote_user'] }}"
  vars_files:
  - vars/cluster-config.yml
  - vars/constants.yml
  - vars/os-level-config.yml  
  tasks:
  - name: Start Master
    become: yes
    shell: |
      set -e
      /spark/sbin/stop-master.sh
      /spark/sbin/start-master.sh

- name: Start Spark worker processes
  hosts: "{{ spark_tag }}"
  gather_facts: false
  become: no
  remote_user: "{{ os_config['remote_user'] }}"
  vars_files:
  - vars/cluster-config.yml
  - vars/constants.yml
  - vars/os-level-config.yml  
  tasks:
  - name: Import configuration properties
    include_vars: 
      dir: vars

  - name: Get Spark master
    delegate_to: localhost
    ec2_instance_info:
      region: "{{ aws_region }}"
      filters:
        instance-state-name: [ "pending", "running" ]  
        dns-name: "{{ groups.aerospike_spark[0] }}"
        "tag:group": "{{ spark_tag }}"   
    register: spark_info

  - name: Start agents
    become: yes
    shell: |
      set -e    
      /spark/sbin/stop-slave.sh 
      /spark/sbin/start-slave.sh spark://{{ spark_info.instances[0].private_ip_address }}:{{ spark_master_port }}

- name: Spark Info
  hosts: local
  connection: local
  gather_facts: no
  tasks:
  - name: Import configuration properties
    include_vars: 
      dir: vars

  - name: Get Spark master
    ec2_instance_info:
      region: "{{ aws_region }}"
      filters:
        instance-state-name: [ "pending", "running" ]  
        dns-name: "{{ groups.aerospike_spark[0] }}"
        "tag:group": "{{ spark_tag }}"   
    register: spark_info

  - name: Spark master IP & master internal url
    debug: 
      msg: "Spark master is {{ groups.aerospike_spark[0] }}. Spark master internal url is spark://{{ spark_info.instances[0].private_ip_address }}:{{ spark_master_port }}."

  - name: Spark web URL
    debug: 
        msg: "Spark web url is  http://{{ groups.aerospike_spark[0] }}:{{ spark_master_web_port }}"