Merge branch 'main' into thiagohora/OPIK-68_add_rate_limit

comet-ml · Sep 20, 2024 · de8b430 · de8b430
2 parents f17343a + cacd942
commit de8b430
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 47 deletions.
diff --git a/.github/workflows/sdk-e2e-tests.yaml b/.github/workflows/sdk-e2e-tests.yaml
@@ -16,11 +16,7 @@ on:
 jobs:
     run-e2e:
         name: SDK E2E Tests ${{matrix.python_version}}
-        runs-on: ubuntu-latest
-        defaults:
-            run:
-                working-directory: sdks/python
-
+        runs-on: ubuntu-latest   
         strategy:
             fail-fast: false
             matrix:
@@ -34,41 +30,49 @@ jobs:
           uses: actions/setup-python@v5
           with:
             python-version: ${{matrix.python_version}}
+
+        - name: Run latest Opik server
+          run: |
+            cd deployment/docker-compose
+            docker compose up -d
+        
+        - name: Check Opik server avialability
+          shell: bash
+          run: |
+            chmod +x ${{ github.workspace }}/tests_end_to_end/installer/*.sh
+            cd ${{ github.workspace }}/deployment/docker-compose
+            echo "Check Docker pods are up"
+            ${{ github.workspace }}/tests_end_to_end/installer/check_docker_compose_pods.sh
+            echo "Check backend health"
+            ${{ github.workspace }}/tests_end_to_end/installer/check_backend.sh
 
         - name: Install opik SDK
           run: |
+            cd ${{ github.workspace }}/sdks/python
             pip install -r tests/test_requirements.txt
             pip install .
 
-        - name: Run latest opik server
-          run: |
-            cd ../../deployment/docker-compose
-            docker compose up -d
-            cd -
-
         - name: Run tests
           run: |
-            echo "Waiting for server to come up..."
-            sleep 30
+            cd ${{ github.workspace }}/sdks/python
             export OPIK_URL_OVERRIDE=http://localhost:5173/api
             pytest tests/e2e -vv 
 
         - name: Keep BE log in case of failure
           if: failure()
           run: |
-            docker logs opik-backend-1 > opik-backend_p${{matrix.python_version}}.log
+            docker logs opik-backend-1 > ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log
 
         - name: Attach BE log
           if: failure()
           uses: actions/upload-artifact@v4
           with:
               name: opik-backend-log-p${{matrix.python_version}}
-              path: sdks/python/opik-backend_p${{matrix.python_version}}.log
+              path: ${{ github.workspace }}/opik-backend_p${{matrix.python_version}}.log
 
         - name: Stop opik server
           if: always()
           run: |
-            cd ../../deployment/docker-compose
+            cd ${{ github.workspace }}/deployment/docker-compose
             docker compose down
-            cd -
 
diff --git a/deployment/docker-compose/README.md b/deployment/docker-compose/README.md
@@ -31,6 +31,27 @@ cd deployment/docker-compose
 docker compose up -d --build
 ```
 
+## Exposing Database and Backend Ports for Local Development
+
+If you're a developer and need to expose the database and backend ports to your host machine for local testing or debugging, you can use the provided Docker Compose override file.
+
+### Steps to Expose Ports
+
+Run the following command to start the services and expose the ports:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.override.yml up
+```
+
+This will expose the following services to the host machine
+
+- Redis: Available on port 6379
+- ClickHouse: Available on ports 8123 (HTTP) and 9000 (Native Protocol)
+- MySQL: Available on port 3306
+- Backend: Available on ports 8080 and 3003
+
+
+
 ## Stop opik
 
 ```bash

diff --git a/deployment/docker-compose/docker-compose.override.yaml b/deployment/docker-compose/docker-compose.override.yaml
@@ -0,0 +1,23 @@
+version: '3'
+services:
+  mysql:
+    ports:
+      - "3306:3306" # Exposing MySQL port to host
+
+  redis:
+    ports:
+      - "6379:6379" # Exposing Redis port to host
+
+  clickhouse:
+    ports:
+      - "8123:8123" # Exposing ClickHouse HTTP port to host
+      - "9000:9000" # Exposing ClickHouse Native Protocol port to host
+
+  backend:
+    ports:
+      - "8080:8080" # Exposing backend HTTP port to host
+      - "3003:3003" # Exposing additional backend port to host
+
+  frontend:
+    ports:
+      - "5173:5173" # Exposing frontend dev server port to host
diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml
@@ -10,21 +10,22 @@ services:
       MYSQL_USER: opik
       MYSQL_PASSWORD: opik
     healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "--silent"]
+      test: [ "CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "--silent" ]
       timeout: 1s
       interval: 1s
       retries: 300
     ports:
-      - "3306:3306"
-    volumes: # Mounted on your $HOME folder
+      - "3306"
+    volumes:
+      # Mounted on your $HOME folder
       - ~/opik/mysql:/var/lib/mysql/
 
   redis:
     image: redis:7.2.4-alpine3.19
     hostname: redis
     command: redis-server --requirepass opik
     ports:
-      - '6379:6379'
+      - '6379'
     healthcheck:
       test: [ "CMD", "nc", "-z", "localhost", "6379" ]
       interval: 2s
@@ -33,7 +34,6 @@ services:
       start_period: 30s
     restart: always
 
-
   clickhouse:
     image: clickhouse/clickhouse-server:23.8.15.35-alpine
     hostname: clickhouse
@@ -45,9 +45,10 @@ services:
       # https://clickhouse.com/docs/en/operations/access-rights#enabling-access-control
       CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
     ports:
-      - "8123:8123" # HTTP default port
-      - "9000:9000" # Native Protocol port
-    volumes: # Mounted on your $HOME folder
+      - "8123" # HTTP default port
+      - "9000" # Native Protocol port
+    volumes:
+      # Mounted on your $HOME folder
       - ~/opik/clickhouse/data:/var/lib/clickhouse/
       - ~/opik/clickhouse/logs:/var/log/clickhouse-server/
     healthcheck:
@@ -63,27 +64,27 @@ services:
       dockerfile: Dockerfile
     platform: linux/amd64
     hostname: backend
-    command: ["bash", "-c", "./run_db_migrations.sh && ./entrypoint.sh"]
+    command: [ "bash", "-c", "./run_db_migrations.sh && ./entrypoint.sh" ]
     environment:
-        DOCKER_BUILDKIT: 1
-        STATE_DB_URL: "jdbc:mysql://mysql:3306/opik?createDatabaseIfNotExist=true&rewriteBatchedStatements=true"
-        STATE_DB_DATABASE_NAME: opik
-        STATE_DB_USER: opik
-        STATE_DB_PASS: opik
-        ANALYTICS_DB_MIGRATIONS_URL: "jdbc:clickhouse://clickhouse:8123"
-        ANALYTICS_DB_MIGRATIONS_USER: opik
-        ANALYTICS_DB_MIGRATIONS_PASS: opik
-        ANALYTICS_DB_PROTOCOL: "HTTP"
-        ANALYTICS_DB_HOST: "clickhouse"
-        ANALYTICS_DB_PORT: 8123
-        ANALYTICS_DB_USERNAME: opik
-        ANALYTICS_DB_DATABASE_NAME: opik
-        JAVA_OPTS: "-Dliquibase.propertySubstitutionEnabled=true"
-        REDIS_URL: redis://:opik@redis:6379/
-        ANALYTICS_DB_PASS: opik
+      DOCKER_BUILDKIT: 1
+      STATE_DB_URL: "jdbc:mysql://mysql:3306/opik?createDatabaseIfNotExist=true&rewriteBatchedStatements=true"
+      STATE_DB_DATABASE_NAME: opik
+      STATE_DB_USER: opik
+      STATE_DB_PASS: opik
+      ANALYTICS_DB_MIGRATIONS_URL: "jdbc:clickhouse://clickhouse:8123"
+      ANALYTICS_DB_MIGRATIONS_USER: opik
+      ANALYTICS_DB_MIGRATIONS_PASS: opik
+      ANALYTICS_DB_PROTOCOL: "HTTP"
+      ANALYTICS_DB_HOST: "clickhouse"
+      ANALYTICS_DB_PORT: 8123
+      ANALYTICS_DB_USERNAME: opik
+      ANALYTICS_DB_DATABASE_NAME: opik
+      JAVA_OPTS: "-Dliquibase.propertySubstitutionEnabled=true"
+      REDIS_URL: redis://:opik@redis:6379/
+      ANALYTICS_DB_PASS: opik
     ports:
-      - "8080:8080"
-      - "3003:3003"
+      - "8080"
+      - "3003"
     depends_on:
       mysql:
         condition: service_healthy
@@ -101,11 +102,11 @@ services:
       - "5173:5173"
     extra_hosts:
       - "apihost:host-gateway"
-    volumes: 
+    volumes:
       - ./nginx_default_local.conf:/etc/nginx/conf.d/default.conf
     depends_on:
       backend:
         condition: service_started
 
 networks:
-  default:
+  default:
diff --git a/deployment/docker-compose/nginx_default_local.conf b/deployment/docker-compose/nginx_default_local.conf
@@ -6,7 +6,7 @@ server {
 
     location /api/ {
         rewrite /api/(.*) /$1  break;
-        proxy_pass http://apihost:8080;
+        proxy_pass http://backend:8080;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;