Skip to content

Commit

Permalink
[hotfix][ci] Optimize CI performance by implementing parallel execution
Browse files Browse the repository at this point in the history
  • Loading branch information
@GOODBOY008
GOODBOY008 committed Oct 15, 2024
1 parent daf27fa commit e825131
Show file tree
Hide file tree
Showing 3 changed files with 382 additions and 317 deletions.
349 changes: 32 additions & 317 deletions .github/workflows/flink_cdc.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,78 +30,13 @@ on:
- 'docs/**'
- 'README.md'

# Concurrency strategy:
# github.workflow: distinguish this workflow from others
# github.event_name: distinguish `push` event from `pull_request` event
# github.event.number: set to the number of the pull request if `pull_request` event
# github.run_id: otherwise, it's a `push` or `schedule` event, only cancel if we rerun the workflow
#
# Reference:
# https://docs.github.com/en/actions/using-jobs/using-concurrency
# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true

env:
MODULES_CORE: "\
flink-cdc-cli,\
flink-cdc-common,\
flink-cdc-composer,\
flink-cdc-runtime"

MODULES_PIPELINE_CONNECTORS: "\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"

MODULES_MYSQL: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"

MODULES_POSTGRES: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"

MODULES_ORACLE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"

MODULES_MONGODB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"

MODULES_SQLSERVER: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"

MODULES_TIDB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"

MODULES_OCEANBASE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"

MODULES_DB2: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"

MODULES_VITESS: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"

MODULES_PIPELINE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"

MODULES_SOURCE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"

jobs:
license_check:
name: License Check
runs-on: ubuntu-latest
steps:
- name: Check out repository code
Expand All @@ -116,260 +51,40 @@ jobs:
run: mvn --no-snapshot-updates -B package -DskipTests
- name: Run license check
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb

compile_and_test:
needs: license_check
# Only run the CI pipeline for the flink-cdc-connectors repository
# if: github.repository == 'apache/flink-cdc-connectors'
runs-on: ubuntu-latest
ut:
strategy:
matrix:
jdk: [ 8 ]
module: [ "core",
"pipeline_connectors",
"mysql",
"postgres",
"oracle",
"mongodb",
"sqlserver",
"tidb",
"oceanbase",
"db2",
"vitess",
"pipeline_e2e",
"source_e2e"
]
timeout-minutes: 120
env:
MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
-Dorg.slf4j.simpleLogger.showDateTime=true \
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
--no-snapshot-updates -B \
--settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \
-Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
-Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
steps:
- run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"

- name: Clean up disk space
run: |
set -euo pipefail
echo "Disk space before cleanup"
df -h
echo "Cleaning up disk space"
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
echo "Disk space after cleanup"
df -h
- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true

- name: Set JDK
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'maven'

- name: Set Maven 3.8.6
uses: stCarolas/setup-maven@v5
with:
maven-version: 3.8.6

- name: Compile and test ${{ matrix.module }}
timeout-minutes: 90
run: |
set -o pipefail
case ${{ matrix.module }} in
("core")
modules=${{ env.MODULES_CORE }}
;;
("pipeline_connectors")
modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
;;
("mysql")
modules=${{ env.MODULES_MYSQL }}
;;
("postgres")
modules=${{ env.MODULES_POSTGRES }}
;;
("oracle")
modules=${{ env.MODULES_ORACLE }}
;;
("mongodb")
modules=${{ env.MODULES_MONGODB }}
;;
("sqlserver")
modules=${{ env.MODULES_SQLSERVER }}
;;
("tidb")
modules=${{ env.MODULES_TIDB }}
;;
("oceanbase")
modules=${{ env.MODULES_OCEANBASE }}
;;
("db2")
modules=${{ env.MODULES_DB2 }}
;;
("vitess")
modules=${{ env.MODULES_VITESS }}
;;
("pipeline_e2e")
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
modules=${{ env.MODULES_PIPELINE_E2E }}
;;
("source_e2e")
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
modules=${{ env.MODULES_SOURCE_E2E }}
;;
esac
if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
compile_modules=$modules
fi
mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify
- name: Print JVM thread dumps when cancelled
if: ${{ failure() }}
run: |
# ----------------------------------------------------------------------------
# Copyright 2023 The Netty Project
#
# ----------------------------------------------------------------------------
# Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
echo "$OSTYPE"
if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
echo "Setting up JVM thread dumps"
# use jattach so that Java processes in docker containers are also covered
# download jattach
curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
if command -v sha256sum &> /dev/null; then
# verify hash of jattach binary
sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1
fi
chmod +x /tmp/jattach
for java_pid in $(sudo pgrep java); do
echo "----------------------- pid $java_pid -----------------------"
echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)"
sudo /tmp/jattach $java_pid jcmd VM.command_line || true
sudo /tmp/jattach $java_pid jcmd "Thread.print -l"
sudo /tmp/jattach $java_pid jcmd GC.heap_info || true
done
else
for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
echo "----------------------- pid $java_pid -----------------------"
jcmd $java_pid VM.command_line || true
jcmd $java_pid Thread.print -l
jcmd $java_pid GC.heap_info || true
done
fi
exit 0
migration_test_ut:
needs: license_check
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true
- name: Compile snapshot CDC version
run: mvn --no-snapshot-updates -B install -DskipTests
- name: Run migration tests
run: cd flink-cdc-migration-tests && mvn clean verify

pipeline_migration_test:
needs: migration_test_ut
runs-on: ubuntu-latest
java-version: [ '8' ]
module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
name: Unit Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
java-version: ${{ matrix.java-version }}
module: ${{ matrix.module }}
pipeline_e2e:
strategy:
matrix:
java-version: [ '8', '11' ]

steps:
- uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 3.0
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: temurin
cache: maven
- name: Install dependencies
run: gem install terminal-table
- name: Prepare CDC versions
run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- name: Prepare Flink distro
run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
working-directory: ./tools/mig-test
- name: Patch Flink configs
run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
working-directory: ./tools/mig-test
- name: Start containers
run: cd conf && docker compose up -d
working-directory: ./tools/mig-test
- name: Run migration tests
run: FLINK_HOME=./flink-1.18.1/ ruby run_migration_test.rb
working-directory: ./tools/mig-test
- name: Stop containers
if: always()
run: cd conf && docker compose down
working-directory: ./tools/mig-test

data_stream_migration_test:
needs: migration_test_ut
runs-on: ubuntu-latest
java-version: [ '8' ]
flink-version: ['1.17.2', '1.18.1', '1.19.1', '1.20.0']
module: [ 'pipeline_e2e' ]
name: Pipeline E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
java-version: ${{ matrix.java-version }}
flink-version: ${{ matrix.flink-version }}
module: ${{ matrix.module }}
source_e2e:
strategy:
matrix:
java-version: [ '8', '11' ]

steps:
- uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 3.0
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: temurin
cache: maven
- name: Install dependencies
run: gem install terminal-table
- name: Prepare CDC versions
run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- name: Prepare Flink distro
run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
working-directory: ./tools/mig-test
- name: Patch Flink configs
run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
working-directory: ./tools/mig-test
- name: Compile Dummy DataStream Jobs
run: cd datastream && ruby compile_jobs.rb
working-directory: ./tools/mig-test
- name: Start containers
run: cd conf && docker compose up -d
working-directory: ./tools/mig-test
- name: Run migration tests
run: cd datastream && FLINK_HOME=../flink-1.18.1/ ruby run_migration_test.rb
working-directory: ./tools/mig-test
- name: Stop containers
if: always()
run: cd conf && docker compose down
working-directory: ./tools/mig-test
java-version: [ '8' ]
flink-version: ['1.16.3', '1.17.2', '1.18.1', '1.19.1', '1.20.0']
module: [ 'source_e2e' ]
name: Source E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
java-version: ${{ matrix.java-version }}
flink-version: ${{ matrix.flink-version }}
module: ${{ matrix.module }}
migration_test:
name: Migration Tests
uses: ./.github/workflows/flink_cdc_migration_test.yml
Loading

0 comments on commit e825131

Please sign in to comment.