test: add rbac backup restore check in ci (#455) #1389
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-backup-restore-cross-version: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| deploy_tools: [docker-compose] | |
| milvus_mode: [standalone] | |
| another_milvus_mode: [standalone] | |
| source_image_tag: [2.3-latest, 2.4-latest] | |
| target_image_tag: [master-latest, 2.4-latest] | |
| exclude: | |
| - source_image_tag: 2.4-latest | |
| target_image_tag: 2.4-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: pip | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.18.0' | |
| cache: true | |
| - name: Build | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| go get | |
| go build | |
| - name: Install dependency | |
| timeout-minutes: 5 | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt --trusted-host https://test.pypi.org | |
| sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Milvus deploy | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.source_image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Prepare data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/prepare_data.py | |
| - name: Backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup check | |
| ./milvus-backup list | |
| ./milvus-backup create -n my_backup | |
| ./milvus-backup list | |
| - name: Restore backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup restore -n my_backup -s _recover | |
| - name: Verify data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/verify_data.py | |
| - name: Save Backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup | |
| - name: delete backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup delete -n my_backup | |
| ./milvus-backup list | |
| - name: Uninstall Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| docker-compose down | |
| sudo rm -rf volumes | |
| - name: Deploy Another Milvus | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.another_milvus_mode }} | |
| run: | | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.target_image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Copy Backup to Another Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup | |
| sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup | |
| - name: List backup from another Milvus | |
| timeout-minutes: 5 | |
| working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio | |
| shell: bash | |
| run: | | |
| pwd | |
| ls -l | |
| tree | |
| - name: Restore backup from another Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup restore -n my_backup -s _recover | |
| - name: Verify data from another Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/verify_data.py | |
| test-backup-restore-after-upgrade: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| deploy_tools: [docker-compose] | |
| milvus_mode: [standalone] | |
| another_milvus_mode: [standalone] | |
| source_image_tag: [v2.4.17, 2.4-latest] | |
| target_image_tag: [master-latest, 2.4-latest] | |
| exclude: | |
| - source_image_tag: 2.4-latest | |
| target_image_tag: 2.4-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: pip | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.18.0' | |
| cache: true | |
| - name: Build | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| go get | |
| go build | |
| - name: Install dependency | |
| timeout-minutes: 5 | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt --trusted-host https://test.pypi.org | |
| sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Milvus deploy | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.source_image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Prepare data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/prepare_data.py | |
| - name: Upgrade Milvus | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.another_milvus_mode }} | |
| run: | | |
| docker-compose down | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.target_image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup check | |
| ./milvus-backup list | |
| ./milvus-backup create -n my_backup | |
| ./milvus-backup list | |
| - name: Restore backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup restore -n my_backup -s _recover | |
| - name: Verify data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/verify_data.py | |
| test-backup-restore-cli: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| deploy_tools: [docker-compose] | |
| milvus_mode: [standalone] | |
| another_milvus_mode: [standalone] | |
| image_tag: [master-latest] | |
| # mq_type: [pulsar, kafka] # TODO: add pulsar and kafka | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: pip | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.18.0' | |
| cache: true | |
| - name: Build | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| go get | |
| go build | |
| - name: Install dependency | |
| timeout-minutes: 5 | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt --trusted-host https://test.pypi.org | |
| sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Milvus deploy | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Prepare data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/prepare_data.py | |
| - name: Backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup check | |
| ./milvus-backup list | |
| ./milvus-backup create -n my_backup | |
| ./milvus-backup list | |
| - name: Restore backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup restore -n my_backup -s _recover | |
| - name: Verify data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/verify_data.py | |
| - name: Save Backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup | |
| - name: delete backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup delete -n my_backup | |
| ./milvus-backup list | |
| - name: Uninstall Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| docker-compose down | |
| sudo rm -rf volumes | |
| - name: Deploy Another Milvus | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.another_milvus_mode }} | |
| run: | | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Copy Backup to Another Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup | |
| sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup | |
| - name: List backup from another Milvus | |
| timeout-minutes: 5 | |
| working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio | |
| shell: bash | |
| run: | | |
| pwd | |
| ls -l | |
| tree | |
| - name: Restore backup from another Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup restore -n my_backup -s _recover | |
| - name: Verify data from another Milvus | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/verify_data.py | |
| test-backup-restore-with-rbac-config: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| deploy_tools: [helm] | |
| milvus_mode: [standalone] | |
| image_tag: [master-latest] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: pip | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.18.0' | |
| cache: "true" | |
| - name: Creating kind cluster | |
| if: ${{ matrix.deploy_tools == 'helm' }} | |
| uses: helm/[email protected] | |
| - name: Build | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| yq -i '.log.level = "debug"' configs/backup.yaml | |
| yq -i '.milvus.authorizationEnabled = true' configs/backup.yaml | |
| yq -i '.minio.bucketName = "milvus-bucket"' configs/backup.yaml | |
| yq -i '.minio.rootPath = "file"' configs/backup.yaml | |
| yq -i '.minio.backupPort = 9010' configs/backup.yaml | |
| yq -i '.minio.backupBucketName = "milvus-bucket"' configs/backup.yaml | |
| yq -i '.minio.backupRootPath = "backup"' configs/backup.yaml | |
| yq -i '.minio.crossStorage = true' configs/backup.yaml | |
| cat configs/backup.yaml || true | |
| go get | |
| go build | |
| - name: Install dependency | |
| timeout-minutes: 5 | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt --trusted-host https://test.pypi.org | |
| sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Milvus deploy | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| if [ ${{ matrix.deploy_tools}} == "helm" ]; then | |
| helm repo add milvus https://zilliztech.github.io/milvus-helm | |
| helm repo update | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag | |
| yq -i ".image.all.tag=\"${tag}\"" rbac-values.yaml | |
| helm install --wait --debug --timeout 600s milvus-backup milvus/milvus -f rbac-values.yaml | |
| helm install --wait --debug --timeout 600s milvus-restore milvus/milvus -f rbac-values.yaml | |
| kubectl get pods | |
| kubectl port-forward service/milvus-backup 19530 >/dev/null 2>&1 & | |
| kubectl port-forward service/milvus-backup-minio 9000 >/dev/null 2>&1 & | |
| kubectl port-forward service/milvus-restore 19531:19530 >/dev/null 2>&1 & | |
| kubectl port-forward service/milvus-restore-minio 9010:9000 >/dev/null 2>&1 & | |
| sleep 10 | |
| nc -vz 127.0.0.1 19530 | |
| nc -vz 127.0.0.1 9000 | |
| nc -vz 127.0.0.1 19531 | |
| nc -vz 127.0.0.1 9010 | |
| sleep 10 | |
| kubectl get pods -n default | grep milvus-backup | |
| kubectl get pods -n default | grep milvus-restore | |
| fi | |
| if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| fi | |
| - name: Prepare data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/prepare_data.py --uri http://127.0.0.1:19530 | |
| python example/prepare_rbac.py --uri http://127.0.0.1:19530 | |
| - name: Backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup check | |
| ./milvus-backup list | |
| ./milvus-backup create -n my_backup --rbac | |
| ./milvus-backup list | |
| - name: Modify Restore Config | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| yq -i '.milvus.port = 19531' configs/backup.yaml | |
| yq -i '.minio.port = 9010' configs/backup.yaml | |
| - name: Restore backup | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| ./milvus-backup restore -n my_backup -s _recover --rbac | |
| - name: Verify data | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| python example/verify_data.py --uri http://127.0.0.1:19531 | |
| python example/verify_rbac.py --uri http://127.0.0.1:19531 | |
| - name: Export logs | |
| if: ${{ always() }} | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| if [ ${{ matrix.deploy_tools}} == "helm" ]; then | |
| kubectl get pods | |
| bash ../../scripts/export_log_k8s.sh default milvus-backup logs | |
| bash ../../scripts/export_log_k8s.sh default milvus-restore logs | |
| fi | |
| if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then | |
| bash ../../scripts/export_log_docker.sh logs | |
| fi | |
| - name: Upload logs | |
| if: ${{ ! success() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: custom-config-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }} | |
| path: | | |
| ./logs | |
| ./server.log | |
| /tmp/ci_logs | |
| deployment/${{ matrix.milvus_mode }}/logs | |
| test-backup-restore-api: | |
| runs-on: ubuntu-latest | |
| needs: test-backup-restore-cli | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| deploy_tools: [docker-compose] | |
| milvus_mode: [standalone] | |
| image_tag: [master-latest, 2.4-latest] | |
| case_tag: [L0, L1, L2, MASTER] | |
| exclude: | |
| - image_tag: 2.4-latest | |
| case_tag: MASTER | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: pip | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.18.0' | |
| cache: true | |
| - name: Build | |
| timeout-minutes: 5 | |
| shell: bash | |
| run: | | |
| yq -i '.log.level = "debug"' configs/backup.yaml | |
| cat configs/backup.yaml || true | |
| go get | |
| go build | |
| bash build_image.sh | |
| - name: Install dependency | |
| timeout-minutes: 5 | |
| working-directory: tests | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt --trusted-host https://test.pypi.org | |
| sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Milvus deploy | |
| timeout-minutes: 15 | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag | |
| yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml | |
| docker-compose up -d | |
| bash ../../scripts/check_healthy.sh | |
| docker-compose ps -a | |
| - name: Start backup server | |
| timeout-minutes: 5 | |
| shell: bash | |
| working-directory: build/deployment | |
| run: | | |
| docker-compose up -d | |
| sleep 10 | |
| docker-compose ps -a | |
| - name: Run test | |
| timeout-minutes: 120 | |
| shell: bash | |
| working-directory: tests | |
| run: | | |
| pytest -s -v --tags ${{ matrix.case_tag }} -n 4 | |
| - name: Get Milvus status | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| docker-compose ps -a || true | |
| - name: Export Milvus logs | |
| if: ${{ always() }} | |
| shell: bash | |
| working-directory: deployment/${{ matrix.milvus_mode }} | |
| run: | | |
| if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then | |
| bash ../../scripts/export_log_docker.sh logs | |
| fi | |
| - name: Export Backup Server logs | |
| if: ${{ always() }} | |
| shell: bash | |
| working-directory: build/deployment | |
| run: | | |
| docker-compose logs > /tmp/ci_logs/backup_server.log 2>&1 || true | |
| - name: Upload logs | |
| if: ${{ ! success() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: api-test-logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}-${{ matrix.image_tag }}-${{ matrix.case_tag }} | |
| path: | | |
| /tmp/ci_logs | |
| deployment/${{ matrix.milvus_mode }}/logs |