docker-compose.yml

services:
  minio:
    container_name: minio
    image: minio/minio
    networks:
      - my_network
    ports:
      - "9000:9000"
      - "9090:9090"
    volumes:
      - ./mount/minio/data:/data
      - ./mount/minio-config:/root/.minio
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
    command: server /data --console-address ":9090"
  postgres:
    container_name: postgres
    image: postgres:14.2-alpine
    ports:
      - 5432:5432
    environment:
      POSTGRES_USER: hive
      POSTGRES_PASSWORD: hive123
      POSTGRES_DB: metastore_db
    depends_on: 
      - minio
    networks:
      - my_network
    volumes:
      - ./mount/postgres/data:/var/lib/postgresql/data
  hive-metastore:
    container_name: hive-metastore
    build: ./images/hive-metastore
    ports:
      - 9083:9083
    depends_on:
      - postgres
    networks:
      - my_network
    restart: always
  spark-master:
    container_name: spark-master
    build: ./images/spark
    ports:
      - "5050:5050"
      - "7077:7077"
    depends_on:
      - postgres
    networks:
      - my_network
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - SPARK_LOCAL_IP=spark-master
    command: usr/local/spark/bin/spark-class org.apache.spark.deploy.master.Master --ip spark-master --port 7077 --webui-port 5050
  spark-worker-a:
    container_name: spark-worker-a
    build: ./images/spark
    depends_on:
      - spark-master
    networks:
      - my_network
    ports:
      - "5051:5051"
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - SPARK_LOCAL_IP=spark-worker-a
    command: usr/local/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port 5051 spark://spark-master:7077
  spark-worker-b:
    container_name: spark-worker-b
    build: ./images/spark
    depends_on:
      - spark-master
    networks:
      - my_network
    ports:
      - "5052:5052"
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - SPARK_LOCAL_IP=spark-worker-b
    command: usr/local/spark/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port 5052 spark://spark-master:7077
  workspace:
    container_name: workspace
    build: ./images/workspace
    depends_on:
      - spark-master
    networks:
      - my_network
    ports:
      - "8888:8888"
      - "4040:4040"
    volumes:
      - ./work:/usr/local/spark_dev/work
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
    command: jupyter lab --port 8888 --LabApp.token=''
  trino:
    container_name: trino
    build: ./images/trino
    depends_on:
      - hive-metastore
    networks:
      - my_network
    ports:
      - "8080:8080"
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
networks:
  my_network:
    driver: bridge
    name: my_network