diff --git a/.github/workflows/sdk-e2e-tests.yaml b/.github/workflows/sdk-e2e-tests.yaml index 5f5d7ad6ee..3362c6353d 100644 --- a/.github/workflows/sdk-e2e-tests.yaml +++ b/.github/workflows/sdk-e2e-tests.yaml @@ -16,11 +16,7 @@ on: jobs: run-e2e: name: SDK E2E Tests ${{matrix.python_version}} - runs-on: ubuntu-latest - defaults: - run: - working-directory: sdks/python - + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -34,41 +30,49 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{matrix.python_version}} + + - name: Run latest Opik server + run: | + cd deployment/docker-compose + docker compose up -d + + - name: Check Opik server avialability + shell: bash + run: | + chmod +x ${{ github.workspace }}/tests_end_to_end/installer/*.sh + cd ${{ github.workspace }}/deployment/docker-compose + echo "Check Docker pods are up" + ${{ github.workspace }}/tests_end_to_end/installer/check_docker_compose_pods.sh + echo "Check backend health" + ${{ github.workspace }}/tests_end_to_end/installer/check_backend.sh - name: Install opik SDK run: | + cd ${{ github.workspace }}/sdks/python pip install -r tests/test_requirements.txt pip install . - - name: Run latest opik server - run: | - cd ../../deployment/docker-compose - docker compose up -d - cd - - - name: Run tests run: | - echo "Waiting for server to come up..." - sleep 30 + cd ${{ github.workspace }}/sdks/python export OPIK_URL_OVERRIDE=http://localhost:5173/api pytest tests/e2e -vv - name: Keep BE log in case of failure if: failure() run: | - docker logs opik-backend-1 > opik-backend_p${{matrix.python_version}}.log + docker logs opik-backend-1 > ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log - name: Attach BE log if: failure() uses: actions/upload-artifact@v4 with: name: opik-backend-log-p${{matrix.python_version}} - path: sdks/python/opik-backend_p${{matrix.python_version}}.log + path: ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log - name: Stop opik server if: always() run: | - cd ../../deployment/docker-compose + cd ${{ github.workspace }}/deployment/docker-compose docker compose down - cd - diff --git a/deployment/docker-compose/README.md b/deployment/docker-compose/README.md index f6162ea497..26c3ce5c09 100644 --- a/deployment/docker-compose/README.md +++ b/deployment/docker-compose/README.md @@ -31,6 +31,27 @@ cd deployment/docker-compose docker compose up -d --build ``` +## Exposing Database and Backend Ports for Local Development + +If you're a developer and need to expose the database and backend ports to your host machine for local testing or debugging, you can use the provided Docker Compose override file. + +### Steps to Expose Ports + +Run the following command to start the services and expose the ports: + +```bash +docker compose -f docker-compose.yml -f docker-compose.override.yml up +``` + +This will expose the following services to the host machine + +- Redis: Available on port 6379 +- ClickHouse: Available on ports 8123 (HTTP) and 9000 (Native Protocol) +- MySQL: Available on port 3306 +- Backend: Available on ports 8080 and 3003 + + + ## Stop opik ```bash diff --git a/deployment/docker-compose/docker-compose.override.yaml b/deployment/docker-compose/docker-compose.override.yaml new file mode 100644 index 0000000000..c75dd09d9d --- /dev/null +++ b/deployment/docker-compose/docker-compose.override.yaml @@ -0,0 +1,23 @@ +version: '3' +services: + mysql: + ports: + - "3306:3306" # Exposing MySQL port to host + + redis: + ports: + - "6379:6379" # Exposing Redis port to host + + clickhouse: + ports: + - "8123:8123" # Exposing ClickHouse HTTP port to host + - "9000:9000" # Exposing ClickHouse Native Protocol port to host + + backend: + ports: + - "8080:8080" # Exposing backend HTTP port to host + - "3003:3003" # Exposing additional backend port to host + + frontend: + ports: + - "5173:5173" # Exposing frontend dev server port to host diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index a065fe136e..6449653f47 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -10,13 +10,14 @@ services: MYSQL_USER: opik MYSQL_PASSWORD: opik healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "--silent"] + test: [ "CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "--silent" ] timeout: 1s interval: 1s retries: 300 ports: - - "3306:3306" - volumes: # Mounted on your $HOME folder + - "3306" + volumes: + # Mounted on your $HOME folder - ~/opik/mysql:/var/lib/mysql/ redis: @@ -24,7 +25,7 @@ services: hostname: redis command: redis-server --requirepass opik ports: - - '6379:6379' + - '6379' healthcheck: test: [ "CMD", "nc", "-z", "localhost", "6379" ] interval: 2s @@ -33,7 +34,6 @@ services: start_period: 30s restart: always - clickhouse: image: clickhouse/clickhouse-server:23.8.15.35-alpine hostname: clickhouse @@ -45,9 +45,10 @@ services: # https://clickhouse.com/docs/en/operations/access-rights#enabling-access-control CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 ports: - - "8123:8123" # HTTP default port - - "9000:9000" # Native Protocol port - volumes: # Mounted on your $HOME folder + - "8123" # HTTP default port + - "9000" # Native Protocol port + volumes: + # Mounted on your $HOME folder - ~/opik/clickhouse/data:/var/lib/clickhouse/ - ~/opik/clickhouse/logs:/var/log/clickhouse-server/ healthcheck: @@ -63,27 +64,27 @@ services: dockerfile: Dockerfile platform: linux/amd64 hostname: backend - command: ["bash", "-c", "./run_db_migrations.sh && ./entrypoint.sh"] + command: [ "bash", "-c", "./run_db_migrations.sh && ./entrypoint.sh" ] environment: - DOCKER_BUILDKIT: 1 - STATE_DB_URL: "jdbc:mysql://mysql:3306/opik?createDatabaseIfNotExist=true&rewriteBatchedStatements=true" - STATE_DB_DATABASE_NAME: opik - STATE_DB_USER: opik - STATE_DB_PASS: opik - ANALYTICS_DB_MIGRATIONS_URL: "jdbc:clickhouse://clickhouse:8123" - ANALYTICS_DB_MIGRATIONS_USER: opik - ANALYTICS_DB_MIGRATIONS_PASS: opik - ANALYTICS_DB_PROTOCOL: "HTTP" - ANALYTICS_DB_HOST: "clickhouse" - ANALYTICS_DB_PORT: 8123 - ANALYTICS_DB_USERNAME: opik - ANALYTICS_DB_DATABASE_NAME: opik - JAVA_OPTS: "-Dliquibase.propertySubstitutionEnabled=true" - REDIS_URL: redis://:opik@redis:6379/ - ANALYTICS_DB_PASS: opik + DOCKER_BUILDKIT: 1 + STATE_DB_URL: "jdbc:mysql://mysql:3306/opik?createDatabaseIfNotExist=true&rewriteBatchedStatements=true" + STATE_DB_DATABASE_NAME: opik + STATE_DB_USER: opik + STATE_DB_PASS: opik + ANALYTICS_DB_MIGRATIONS_URL: "jdbc:clickhouse://clickhouse:8123" + ANALYTICS_DB_MIGRATIONS_USER: opik + ANALYTICS_DB_MIGRATIONS_PASS: opik + ANALYTICS_DB_PROTOCOL: "HTTP" + ANALYTICS_DB_HOST: "clickhouse" + ANALYTICS_DB_PORT: 8123 + ANALYTICS_DB_USERNAME: opik + ANALYTICS_DB_DATABASE_NAME: opik + JAVA_OPTS: "-Dliquibase.propertySubstitutionEnabled=true" + REDIS_URL: redis://:opik@redis:6379/ + ANALYTICS_DB_PASS: opik ports: - - "8080:8080" - - "3003:3003" + - "8080" + - "3003" depends_on: mysql: condition: service_healthy @@ -101,11 +102,11 @@ services: - "5173:5173" extra_hosts: - "apihost:host-gateway" - volumes: + volumes: - ./nginx_default_local.conf:/etc/nginx/conf.d/default.conf depends_on: backend: condition: service_started networks: - default: \ No newline at end of file + default: diff --git a/deployment/docker-compose/nginx_default_local.conf b/deployment/docker-compose/nginx_default_local.conf index 46175df8da..c91f2391e0 100644 --- a/deployment/docker-compose/nginx_default_local.conf +++ b/deployment/docker-compose/nginx_default_local.conf @@ -6,7 +6,7 @@ server { location /api/ { rewrite /api/(.*) /$1 break; - proxy_pass http://apihost:8080; + proxy_pass http://backend:8080; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;