Skip to content
gateway-test

fix: skip dirs with no permissions for release-1.0 #841

Sign in to view logs

fix: skip dirs with no permissions for release-1.0

fix: skip dirs with no permissions for release-1.0 #841

Workflow file for this run

.github/workflows/gateway.yml at 657ff16
name: "gateway-test"
on:
push:
branches:
- release-**
paths-ignore:
- 'docs/**'
- '**.md'
pull_request:
#The branches below must be a subset of the branches above
branches:
- release-**
paths-ignore:
- 'docs/**'
- '**.md'
workflow_dispatch:
jobs:
upload-big-file-with-redis:
runs-on: ubuntu-20.04
services:
redis:
# Docker Hub image
image: redis
# Set health checks to wait until redis has started
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps port 6379 on service container to the host
- 6379:6379
steps:
- uses: actions/checkout@v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "redis://localhost/1"
file_size: "100M"
file_count: 5
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-redis:
runs-on: ubuntu-20.04
services:
redis:
# Docker Hub image
image: redis
# Set health checks to wait until redis has started
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps port 6379 on service container to the host
- 6379:6379
steps:
- uses: actions/checkout@v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "redis://localhost/1"
file_size: "8K"
file_count: 2500
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-mysql-read-uncommitted:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "8K"
file_count: 2500
isolation_level: "read uncommitted"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-mysql-read-committed:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "8K"
file_count: 2500
isolation_level: "read committed"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-mysql-repeatable-read:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "8K"
file_count: 2500
isolation_level: "repeatable read"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-mysql-serializable:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "8K"
file_count: 2500
isolation_level: "serializable"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-mysql-read-uncommitted:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "read uncommitted"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-mysql-read-committed:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "read committed"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-mysql-repeatable-read:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "repeatable read"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-mysql-serializable:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "serializable"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-mariadb-read-committed:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: install
run: |
docker run -p 127.0.0.1:3306:3306 --name mdb -e MARIADB_ROOT_PASSWORD=root -d mariadb:latest
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:root@\\(127.0.0.1\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "read committed"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-badgerdb:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo go get github.com/dgraph-io/badger/v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "badger://badger-data"
file_size: "100M"
file_count: 1
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-badgerdb:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo go get github.com/dgraph-io/badger/v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "badger://badger-data"
file_size: "8K"
file_count: 2500
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-sqlite:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "sqlite3://test.db"
file_size: "8K"
file_count: 2500
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-sqlite:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo /etc/init.d/mysql start
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "sqlite3://test.db"
file_size: "100M"
file_count: 5
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-pg:
runs-on: ubuntu-20.04
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
steps:
- uses: actions/checkout@v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "postgres://postgres:[email protected]:5432/test?sslmode=disable"
file_size: "8K"
file_count: 2500
threads: 10
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-pg-read-committed:
runs-on: ubuntu-20.04
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
steps:
- uses: actions/checkout@v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "postgres://postgres:[email protected]:5432/test?sslmode=disable"
file_size: "100M"
file_count: 3
threads: 10
isolation_level: "read committed"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-pg-repeatable-read:
runs-on: ubuntu-20.04
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
steps:
- uses: actions/checkout@v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "postgres://postgres:[email protected]:5432/test?sslmode=disable"
file_size: "100M"
file_count: 5
threads: 10
isolation_level: "repeatable read"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-pg-serializable:
runs-on: ubuntu-20.04
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
steps:
- uses: actions/checkout@v3
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "postgres://postgres:[email protected]:5432/test?sslmode=disable"
file_size: "100M"
file_count: 5
threads: 10
isolation_level: "serializable"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-tidb-read-committed:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Install TiDB
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
source /home/runner/.profile
tiup playground 5.4.0 &
sleep 120
mysql -h127.0.0.1 -P4000 -uroot -e "set global tidb_enable_noop_functions=1;"
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:@\\(127.0.0.1:4000\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "read committed"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-tidb-repeatable-read:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Install TiDB
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
source /home/runner/.profile
tiup playground 5.4.0 &
sleep 120
mysql -h127.0.0.1 -P4000 -uroot -e "set global tidb_enable_noop_functions=1;"
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "mysql://root:@\\(127.0.0.1:4000\\)/test"
file_size: "100M"
file_count: 5
isolation_level: "repeatable read"
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-small-files-with-etcd:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo apt install etcd
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "etcd://localhost:2379/jfs"
file_size: "8K"
file_count: 2500
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-etcd:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- run: sudo apt install etcd
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "etcd://localhost:2379/jfs"
file_size: "200M"
file_count: 5
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
# upload-big-file-with-oceanbase-read-committed:
# runs-on: [self-hosted, daily-build, bench-02]
# steps:
# - uses: actions/checkout@v3
# - name: install ob
# run: |
# docker rm obstandalone --force || echo "remove obstandalone failed"
# docker run -p 2881:2881 --name obstandalone -e MINI_MODE=1 -d oceanbase/oceanbase-ce
# sleep 60
# mysql -h127.0.0.1 -P2881 -uroot -e "ALTER SYSTEM SET _ob_enable_prepared_statement=TRUE;"
# - name: Test
# uses: ./.github/actions/gateway
# with:
# meta_url: "mysql://root:@\\(127.0.0.1:2881\\)/test"
# file_size: "200M"
# file_count: 5
# isolation_level: "read committed"
# upload-big-file-with-oceanbase-repeatable-read:
# runs-on: [self-hosted, daily-build, bench-03]
# steps:
# - uses: actions/checkout@v3
# - name: install ob
# run: |
# docker rm obstandalone --force || echo "remove obstandalone failed"
# docker run -p 2881:2881 --name obstandalone -e MINI_MODE=1 -d oceanbase/oceanbase-ce
# sleep 60
# mysql -h127.0.0.1 -P2881 -uroot -e "ALTER SYSTEM SET _ob_enable_prepared_statement=TRUE;"
# - name: Test
# uses: ./.github/actions/gateway
# with:
# meta_url: "mysql://root:@\\(localhost:2881\\)/test"
# file_size: "200M"
# file_count: 5
# isolation_level: "repeatable read"
# upload-big-file-with-oceanbase-serializable:
# runs-on: [self-hosted, daily-build, bench-03]
# steps:
# - uses: actions/checkout@v3
# - name: install ob
# run: |
# docker rm obstandalone --force || echo "remove obstandalone failed"
# docker run -p 2881:2881 --name obstandalone -e MINI_MODE=1 -d oceanbase/oceanbase-ce
# sleep 60
# mysql -h127.0.0.1 -P2881 -uroot -e "ALTER SYSTEM SET _ob_enable_prepared_statement=TRUE;"
# - name: Test
# uses: ./.github/actions/gateway
# with:
# meta_url: "mysql://root:@\\(localhost:2881\\)/test"
# file_size: "200M"
# file_count: 5
# isolation_level: "serializable"
upload-small-files-with-tikv:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Install TiKV
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
source /home/runner/.bash_profile
source /home/runner/.profile
tiup playground --mode tikv-slim &
sleep 5
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "tikv://127.0.0.1:2379"
file_size: "8K"
file_count: 2500
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
upload-big-files-with-tikv:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v3
- name: Install TiKV
run: |
curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
source /home/runner/.bash_profile
source /home/runner/.profile
tiup playground --mode tikv-slim &
sleep 5
- name: Test
uses: ./.github/actions/gateway
with:
meta_url: "tikv://127.0.0.1:2379"
file_size: "100M"
file_count: 5
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1
sync-to-gateway-with-multithread:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 1
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: '1.18.x'
- name: Build linux target
run: |
make juicefs
#wget -q https://github.com/juicedata/juicefs/releases/download/v1.0.0-beta3/juicefs-1.0.0-beta3-linux-amd64.tar.gz
#tar -xzf juicefs-1.0.0-beta3-linux-amd64.tar.gz
- name: Test sync to gateway with multithread
run: |
#test for PR: https://github.com/juicedata/juicefs/pull/2024
wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc
chmod +x mc
sudo apt install md5deep
sudo /etc/init.d/mysql start
meta_url="mysql://root:root@(127.0.0.1)/test"
volume=myjfs
mp=/tmp/myjfs
db_name=$(basename $meta_url)
mysql -uroot -proot -e "set global transaction isolation level repeatable read;"
mysql -uroot -proot -e "drop database if exists $db_name; create database $db_name;"
export MINIO_ROOT_USER=minioadmin
export MINIO_ROOT_PASSWORD=minioadmin
./juicefs format $meta_url $volume
./juicefs gateway $meta_url localhost:9000 &
start=`date +%s`
./juicefs sync --dirs ./ s3://minioadmin:minioadmin@localhost:9000/$volume/source_dir/ --no-https -p 50
end=`date +%s`
time=$((end-start))
echo time is: $time
./juicefs mount -d $meta_url $mp
rm -rf $mp/source_dir
- name: Setup upterm session
if: ${{ failure() }}
uses: lhotari/action-upterm@v1