Skip to content

Commit

Permalink
Merge pull request #289 from cashapp/mysql84
Browse files Browse the repository at this point in the history
Add support for MySQL 8.4
  • Loading branch information
kolbe authored May 23, 2024
2 parents 0722021 + 68aff41 commit 41ad6c8
Show file tree
Hide file tree
Showing 10 changed files with 209 additions and 27 deletions.
21 changes: 0 additions & 21 deletions .github/workflows/mysql82.yml

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: MySQL 8.1 innovation release
name: MySQL 8.4 GA
on:
push:
branches: [ main ]
Expand All @@ -15,7 +15,7 @@ jobs:
uses: actions/checkout@v4

- name: Deploy MySQL
run: scripts/dbdeployer_install_81.sh
run: scripts/manual_install_84.sh && mysql -u root -h 127.0.0.1 -P 3306 -e "CREATE DATABASE test;"

- name: Test
run: MYSQL_DSN="msandbox:msandbox@tcp(127.0.0.1:8100)/test" go test -race -v ./...
run: MYSQL_DSN="root:@tcp(127.0.0.1:3306)/test" REPLICA_DSN="root:@tcp(127.0.0.1:3307)/test" go test -race -v ./...
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
Spirit is a _reimplementation_ of the schema change tool [gh-ost](https://github.com/github/gh-ost).

It works very similar to gh-ost except:
- It only supports MySQL 8.0
- It only supports MySQL 8.0 and higher
- It is multi-threaded in both the row-copying and the binlog applying phase
- It supports resume-from-checkpoint

Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.21

require (
github.com/alecthomas/kong v0.7.1
github.com/go-mysql-org/go-mysql v1.7.1-0.20231227030153-e817d9813fe3
github.com/go-mysql-org/go-mysql v1.8.0
github.com/go-sql-driver/mysql v1.7.1
github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32
github.com/pingcap/tidb/pkg/parser v0.0.0-20231103042308-035ad5ccbe67
Expand All @@ -21,6 +21,7 @@ require (
github.com/Masterminds/semver v1.5.0 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/google/uuid v1.4.0 // indirect
github.com/klauspost/compress v1.17.1 // indirect
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-mysql-org/go-mysql v1.7.1-0.20231227030153-e817d9813fe3 h1:3LDcDJa6U5onHlQ6Z0fjEF5up/ZI4mlTRpx6zHjq9hg=
github.com/go-mysql-org/go-mysql v1.7.1-0.20231227030153-e817d9813fe3/go.mod h1:AGmdMIbBbcuUDRjzvHskI7+1SmPKhgKrqf3g8lknIc4=
github.com/go-mysql-org/go-mysql v1.8.0 h1:bN+/Q5yyQXQOAabXPkI3GZX43w4Tsj2DIthjC9i6CkQ=
github.com/go-mysql-org/go-mysql v1.8.0/go.mod h1:kwbF156Z9Sy8amP3E1SZp7/s/0PuJj/xKaOWToQiq0Y=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
Expand Down
9 changes: 9 additions & 0 deletions pkg/dbconn/dbconn.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,3 +200,12 @@ func BeginStandardTrx(ctx context.Context, db *sql.DB, opts *sql.TxOptions) (*sq
}
return trx, connectionID, nil
}

// IsMySQL84 returns true if the MySQL version can positively be identified as 8.4
func IsMySQL84(db *sql.DB) bool {
var version string
if err := db.QueryRow("select substr(version(), 1, 3)").Scan(&version); err != nil {
return false // can't tell
}
return version == "8.4"
}
11 changes: 10 additions & 1 deletion pkg/repl/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ type Client struct {
timingHistory []time.Duration
concurrency int

isMySQL84 bool

// The periodic flush lock is just used for ensuring only one periodic flush runs at a time,
// and when we disable it, no more periodic flushes will run. The actual flushing is protected
// by a lower level lock (sync.Mutex on Client)
Expand Down Expand Up @@ -274,7 +276,11 @@ func (c *Client) pksToRowValueConstructor(d []string) string {
func (c *Client) getCurrentBinlogPosition() (mysql.Position, error) {
var binlogFile, fake string
var binlogPos uint32
err := c.db.QueryRow("SHOW MASTER STATUS").Scan(&binlogFile, &binlogPos, &fake, &fake, &fake) //nolint: execinquery
var binlogPosStmt = "SHOW MASTER STATUS"
if c.isMySQL84 {
binlogPosStmt = "SHOW BINARY LOG STATUS"
}
err := c.db.QueryRow(binlogPosStmt).Scan(&binlogFile, &binlogPos, &fake, &fake, &fake) //nolint: execinquery
if err != nil {
return mysql.Position{}, err
}
Expand Down Expand Up @@ -305,6 +311,9 @@ func (c *Client) Run() (err error) {
cfg.TLSConfig = dbconn.NewTLSConfig()
cfg.TLSConfig.ServerName = utils.StripPort(cfg.Addr)
}
if dbconn.IsMySQL84(c.db) { // handle MySQL 8.4
c.isMySQL84 = true
}
c.canal, err = canal.NewCanal(cfg)
if err != nil {
return err
Expand Down
3 changes: 3 additions & 0 deletions pkg/repl/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ func TestReplClientResumeFromPoint(t *testing.T) {
Concurrency: 4,
TargetBatchTime: time.Second,
})
if dbconn.IsMySQL84(db) { // handle MySQL 8.4
client.isMySQL84 = true
}
pos, err := client.getCurrentBinlogPosition()
assert.NoError(t, err)
pos.Pos = 4
Expand Down
164 changes: 164 additions & 0 deletions scripts/manual_install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
#!/bin/bash
#
# source this file to get the following functions:
# - setup: install required packages
# - activate <version>: download and extract the specified version of MySQL
# - initialize_mysql <name>: initialize a MySQL data directory
# - start_mysql <name> <port> <server_id>: start a MySQL server
# - check_mysql <name>: check if a MySQL server is running
# - deploy_replication <num_nodes> <base_port>: deploy a replication setup
# - exec_mysql_sock <name> <query>: execute a query on a MySQL server

set -xe

setup() {
local cmd=()
if (( UID != 0 ))
then
cmd=(sudo)
fi
cmd+=(apt)

"${cmd[@]}" update

packages=(
ca-certificates
libaio1
libnuma1
wget
sudo
xz-utils
)

"${cmd[@]}" install -y "${packages[@]}"
}

mysql_cmd=(
./bin/mysqld --no-defaults
--socket="mysql.sock"
--log-error="mysql.err"
--pid-file="mysql.pid"
--user="$(id -u -n)"
--gtid-mode=ON
--enforce-gtid-consistency=ON
)
if (( UID == 0 ))
then
mysql_cmd+=(--user=root)
fi

activate() {
if [[ $1 ]]; then
version=$1
else
echo "Usage: activate <version>" >&2
return 1
fi

minor_version="${version%.*}"

printf -v filename "mysql-%s-linux-glibc2.28-x86_64.tar.xz" "$version"
printf -v url "https://dev.mysql.com/get/Downloads/MySQL-%s/%s" "$minor_version" "$filename"
dirname="${filename%.tar.xz}"

if [[ -d $dirname ]]; then
pushd "$dirname"
return
fi
if [[ ! -f $filename ]]; then
wget "$url"
fi
tar -xf "$filename"
pushd "$dirname"
}

initialize_mysql() {
local datadir=data
if [[ $1 ]]; then
datadir="data-$1"
fi
"${mysql_cmd[@]}" --initialize-insecure --datadir="$datadir"
}

start_mysql() {
local datadir=data
local name
local port=3306
local server_id=1

if [[ $1 ]]; then
name=$1
datadir="data-$1"
fi
if [[ $2 ]]; then
port=$2
fi
if [[ $3 ]]; then
server_id=$3
fi

"${mysql_cmd[@]}" --datadir="$datadir" --port="$port" --server-id="$server_id" &

pid=$!

sleep 1
if ! kill -0 "$pid"; then
cat "$datadir/mysql.err"
return 1
fi

check_mysql "$name"
}

check_mysql() {
local datadir=$1
local i
for ((i=0;i<5;i++))
do
sleep 1
exec_mysql_sock "$datadir" 'SELECT 1' > /dev/null && return
done
printf %s\\n "MySQL check of $datadir failed"
return 1
}


deploy_replication() {
num_nodes=2
base_port=3306

if [[ $1 ]]; then
num_nodes=$1
fi
if [[ $2 ]]; then
base_port=$2
fi

initialize_mysql primary
start_mysql primary "$base_port"

exec_mysql_sock primary "CREATE USER 'repl'@'%' IDENTIFIED BY 'replica'"
exec_mysql_sock primary "GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'"

# SOURCE_AUTO_POSITION=1"

local i
for ((i=1;i<num_nodes;i++))
do
name="replica$i"
port=$((base_port + i))
initialize_mysql "$name"
start_mysql "$name" "$port" "$(( 100 + i ))"
exec_mysql_sock "$name" "CHANGE REPLICATION SOURCE TO SOURCE_HOST='127.0.0.1', SOURCE_PORT=$base_port, GET_SOURCE_PUBLIC_KEY=1"
exec_mysql_sock "$name" "START REPLICA USER='repl' PASSWORD='replica'"
done
}

exec_mysql_sock() {
local datadir=data
if [[ $1 ]]; then
datadir="data-$1"
fi
./bin/mysql -u root -S "$datadir/mysql.sock" -e "$2"
}

13 changes: 13 additions & 0 deletions scripts/manual_install_84.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

set -xe

source "${0%/*}"/manual_install.sh

version=8.4.0

pushd /tmp

setup
activate "$version"
deploy_replication

0 comments on commit 41ad6c8

Please sign in to comment.