9 changed files with 682 additions and 0 deletions
@ -0,0 +1,5 @@ |
|||
rem 编译为 Linux 64 位可执行文件 |
|||
set GOOS=linux |
|||
set GOARCH=amd64 |
|||
set CGO_ENABLED=0 |
|||
go build -o xttbinlog main.go |
|||
@ -0,0 +1,17 @@ |
|||
mysql: |
|||
addr: "127.0.0.1:3306" |
|||
user: "canal" |
|||
password: "canal_password" |
|||
database: "test" |
|||
|
|||
sync: |
|||
tables: |
|||
- "user" |
|||
- "order" |
|||
- "product" |
|||
batch_size: 100 |
|||
workers: 4 |
|||
|
|||
# storage: |
|||
# type: "redis" |
|||
# redis_url: "redis://localhost:6379/0" |
|||
@ -0,0 +1,39 @@ |
|||
package config |
|||
|
|||
import "github.com/go-mysql-org/go-mysql/canal" |
|||
|
|||
type Config struct { |
|||
MySQL struct { |
|||
Addr string `yaml:"addr"` |
|||
User string `yaml:"user"` |
|||
Password string `yaml:"password"` |
|||
Database string `yaml:"database"` |
|||
} `yaml:"mysql"` |
|||
|
|||
Sync struct { |
|||
Tables []string `yaml:"tables"` |
|||
BatchSize int `yaml:"batch_size"` |
|||
Workers int `yaml:"workers"` |
|||
} `yaml:"sync"` |
|||
|
|||
Storage struct { |
|||
Type string `yaml:"type"` // memory, redis, file
|
|||
FilePath string `yaml:"file_path"` |
|||
RedisURL string `yaml:"redis_url"` |
|||
} `yaml:"storage"` |
|||
} |
|||
|
|||
func (c *Config) ToCanalConfig() *canal.Config { |
|||
cfg := canal.NewDefaultConfig() |
|||
cfg.Addr = c.MySQL.Addr |
|||
cfg.User = c.MySQL.User |
|||
cfg.Password = c.MySQL.Password |
|||
cfg.Dump.ExecutionPath = "" |
|||
|
|||
// 只监听指定数据库
|
|||
if c.MySQL.Database != "" { |
|||
cfg.IncludeTableRegex = []string{c.MySQL.Database + "\\..*"} |
|||
} |
|||
|
|||
return cfg |
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
module xttbinlog |
|||
|
|||
go 1.24.6 |
|||
|
|||
require ( |
|||
github.com/go-mysql-org/go-mysql v1.13.0 |
|||
github.com/go-redis/redis/v8 v8.11.0 |
|||
gopkg.in/yaml.v2 v2.4.0 |
|||
) |
|||
|
|||
require ( |
|||
filippo.io/edwards25519 v1.1.0 // indirect |
|||
github.com/BurntSushi/toml v1.3.2 // indirect |
|||
github.com/cespare/xxhash/v2 v2.1.1 // indirect |
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect |
|||
github.com/goccy/go-json v0.10.2 // indirect |
|||
github.com/google/uuid v1.3.0 // indirect |
|||
github.com/klauspost/compress v1.17.8 // indirect |
|||
github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec // indirect |
|||
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect |
|||
github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a // indirect |
|||
github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d // indirect |
|||
github.com/shopspring/decimal v1.2.0 // indirect |
|||
go.uber.org/atomic v1.11.0 // indirect |
|||
go.uber.org/multierr v1.11.0 // indirect |
|||
go.uber.org/zap v1.27.0 // indirect |
|||
golang.org/x/text v0.24.0 // indirect |
|||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect |
|||
) |
|||
@ -0,0 +1,158 @@ |
|||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= |
|||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= |
|||
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= |
|||
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= |
|||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= |
|||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= |
|||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= |
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= |
|||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= |
|||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= |
|||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= |
|||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= |
|||
github.com/go-mysql-org/go-mysql v1.13.0 h1:Hlsa5x1bX/wBFtMbdIOmb6YzyaVNBWnwrb8gSIEPMDc= |
|||
github.com/go-mysql-org/go-mysql v1.13.0/go.mod h1:FQxw17uRbFvMZFK+dPtIPufbU46nBdrGaxOw0ac9MFs= |
|||
github.com/go-redis/redis/v8 v8.11.0 h1:O1Td0mQ8UFChQ3N9zFQqo6kTU2cJ+/it88gDB+zg0wo= |
|||
github.com/go-redis/redis/v8 v8.11.0/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M= |
|||
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= |
|||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= |
|||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= |
|||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= |
|||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
|||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= |
|||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= |
|||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= |
|||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= |
|||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= |
|||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= |
|||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= |
|||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= |
|||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= |
|||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= |
|||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= |
|||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= |
|||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= |
|||
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= |
|||
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= |
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= |
|||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= |
|||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= |
|||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= |
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= |
|||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= |
|||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= |
|||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= |
|||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= |
|||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
|||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= |
|||
github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= |
|||
github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= |
|||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= |
|||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= |
|||
github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= |
|||
github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= |
|||
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= |
|||
github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec h1:3EiGmeJWoNixU+EwllIn26x6s4njiWRXewdx2zlYa84= |
|||
github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= |
|||
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= |
|||
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= |
|||
github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a h1:WIhmJBlNGmnCWH6TLMdZfNEDaiU8cFpZe3iaqDbQ0M8= |
|||
github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= |
|||
github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d h1:3Ej6eTuLZp25p3aH/EXdReRHY12hjZYs3RrGp7iLdag= |
|||
github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d/go.mod h1:+8feuexTKcXHZF/dkDfvCwEyBAmgb4paFc3/WeYV2eE= |
|||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
|||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
|||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= |
|||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= |
|||
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= |
|||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= |
|||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
|||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= |
|||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= |
|||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= |
|||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= |
|||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= |
|||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= |
|||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= |
|||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= |
|||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= |
|||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= |
|||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= |
|||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= |
|||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= |
|||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= |
|||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= |
|||
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= |
|||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= |
|||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= |
|||
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= |
|||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= |
|||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= |
|||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= |
|||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= |
|||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= |
|||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= |
|||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= |
|||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
|||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
|||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= |
|||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= |
|||
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= |
|||
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= |
|||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= |
|||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
|||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
|||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= |
|||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= |
|||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
|||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
|||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= |
|||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= |
|||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= |
|||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= |
|||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= |
|||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= |
|||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= |
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= |
|||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= |
|||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= |
|||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= |
|||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= |
|||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= |
|||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= |
|||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= |
|||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
|||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
|||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= |
|||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
|||
@ -0,0 +1,167 @@ |
|||
package handler |
|||
|
|||
import ( |
|||
"context" |
|||
"log" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/go-mysql-org/go-mysql/canal" |
|||
"github.com/go-mysql-org/go-mysql/mysql" |
|||
) |
|||
|
|||
type DataEvent struct { |
|||
Action string `json:"action"` |
|||
Schema string `json:"schema"` |
|||
Table string `json:"table"` |
|||
Timestamp time.Time `json:"timestamp"` |
|||
Data []interface{} `json:"data"` |
|||
OldData []interface{} `json:"old_data,omitempty"` |
|||
} |
|||
|
|||
type EventHandler struct { |
|||
canal.DummyEventHandler |
|||
eventChan chan *DataEvent |
|||
batchSize int |
|||
workers int |
|||
wg sync.WaitGroup |
|||
ctx context.Context |
|||
cancel context.CancelFunc |
|||
} |
|||
|
|||
func NewEventHandler(batchSize, workers int) *EventHandler { |
|||
ctx, cancel := context.WithCancel(context.Background()) |
|||
handler := &EventHandler{ |
|||
eventChan: make(chan *DataEvent, 1000), |
|||
batchSize: batchSize, |
|||
workers: workers, |
|||
ctx: ctx, |
|||
cancel: cancel, |
|||
} |
|||
|
|||
handler.startWorkers() |
|||
return handler |
|||
} |
|||
|
|||
func (h *EventHandler) startWorkers() { |
|||
for i := 0; i < h.workers; i++ { |
|||
h.wg.Add(1) |
|||
go h.worker(i) |
|||
} |
|||
} |
|||
|
|||
func (h *EventHandler) worker(id int) { |
|||
defer h.wg.Done() |
|||
|
|||
var batch []*DataEvent |
|||
batchTimer := time.NewTimer(100 * time.Millisecond) |
|||
defer batchTimer.Stop() |
|||
|
|||
for { |
|||
select { |
|||
case event, ok := <-h.eventChan: |
|||
if !ok { |
|||
// 通道关闭，处理剩余数据
|
|||
if len(batch) > 0 { |
|||
h.processBatch(batch) |
|||
} |
|||
return |
|||
} |
|||
|
|||
batch = append(batch, event) |
|||
|
|||
if len(batch) >= h.batchSize { |
|||
h.processBatch(batch) |
|||
batch = nil |
|||
batchTimer.Reset(100 * time.Millisecond) |
|||
} |
|||
|
|||
case <-batchTimer.C: |
|||
if len(batch) > 0 { |
|||
h.processBatch(batch) |
|||
batch = nil |
|||
} |
|||
batchTimer.Reset(100 * time.Millisecond) |
|||
|
|||
case <-h.ctx.Done(): |
|||
if len(batch) > 0 { |
|||
h.processBatch(batch) |
|||
} |
|||
return |
|||
} |
|||
} |
|||
} |
|||
|
|||
func (h *EventHandler) processBatch(events []*DataEvent) { |
|||
// 这里实现数据同步逻辑
|
|||
// 可以同步到其他数据库、消息队列、ES等
|
|||
|
|||
log.Printf("处理批次: %d 个事件", len(events)) |
|||
for _, event := range events { |
|||
// 示例：打印事件信息
|
|||
log.Printf("同步事件: %s.%s %s", event.Schema, event.Table, event.Action) |
|||
|
|||
// 实际应用中，这里可以同步到：
|
|||
// 1. 其他MySQL数据库
|
|||
// 2. Elasticsearch
|
|||
// 3. Redis缓存
|
|||
// 4. Kafka消息队列
|
|||
// 5. 其他数据存储
|
|||
} |
|||
} |
|||
|
|||
func (h *EventHandler) OnRow(e *canal.RowsEvent) error { |
|||
var event *DataEvent |
|||
|
|||
switch e.Action { |
|||
case "insert": |
|||
for _, row := range e.Rows { |
|||
event = &DataEvent{ |
|||
Action: "insert", |
|||
Schema: e.Table.Schema, |
|||
Table: e.Table.Name, |
|||
Timestamp: time.Now(), |
|||
Data: row, |
|||
} |
|||
h.eventChan <- event |
|||
} |
|||
|
|||
case "update": |
|||
for i := 0; i < len(e.Rows); i += 2 { |
|||
event = &DataEvent{ |
|||
Action: "update", |
|||
Schema: e.Table.Schema, |
|||
Table: e.Table.Name, |
|||
Timestamp: time.Now(), |
|||
OldData: e.Rows[i], |
|||
Data: e.Rows[i+1], |
|||
} |
|||
h.eventChan <- event |
|||
} |
|||
|
|||
case "delete": |
|||
for _, row := range e.Rows { |
|||
event = &DataEvent{ |
|||
Action: "delete", |
|||
Schema: e.Table.Schema, |
|||
Table: e.Table.Name, |
|||
Timestamp: time.Now(), |
|||
Data: row, |
|||
} |
|||
h.eventChan <- event |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (h *EventHandler) OnPosSynced(pos mysql.Position, force bool) error { |
|||
log.Printf("位置已同步: %s, force: %t", pos, force) |
|||
return nil |
|||
} |
|||
|
|||
func (h *EventHandler) Close() { |
|||
h.cancel() |
|||
close(h.eventChan) |
|||
h.wg.Wait() |
|||
} |
|||
@ -0,0 +1,62 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"log" |
|||
"os" |
|||
"os/signal" |
|||
"syscall" |
|||
|
|||
"xttbinlog/config" |
|||
"xttbinlog/syncer" |
|||
|
|||
"gopkg.in/yaml.v2" |
|||
) |
|||
|
|||
func main() { |
|||
// 读取配置
|
|||
cfg, err := loadConfig("config.yaml") |
|||
if err != nil { |
|||
log.Fatalf("加载配置失败: %v", err) |
|||
} |
|||
|
|||
// 创建数据同步器
|
|||
syncer, err := syncer.NewDataSyncer(cfg) |
|||
if err != nil { |
|||
log.Fatalf("创建同步器失败: %v", err) |
|||
} |
|||
|
|||
// 处理信号
|
|||
setupSignalHandler(syncer) |
|||
|
|||
// 启动同步
|
|||
if err := syncer.Start(); err != nil { |
|||
log.Fatalf("启动同步失败: %v", err) |
|||
} |
|||
} |
|||
|
|||
func loadConfig(filename string) (*config.Config, error) { |
|||
data, err := os.ReadFile(filename) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
var cfg config.Config |
|||
err = yaml.Unmarshal(data, &cfg) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return &cfg, nil |
|||
} |
|||
|
|||
func setupSignalHandler(syncer *syncer.DataSyncer) { |
|||
sigChan := make(chan os.Signal, 1) |
|||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) |
|||
|
|||
go func() { |
|||
sig := <-sigChan |
|||
log.Printf("接收到信号: %v", sig) |
|||
syncer.Stop() |
|||
os.Exit(0) |
|||
}() |
|||
} |
|||
@ -0,0 +1,93 @@ |
|||
package storage |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"os" |
|||
"sync" |
|||
|
|||
"github.com/go-mysql-org/go-mysql/mysql" |
|||
"github.com/go-redis/redis/v8" |
|||
) |
|||
|
|||
type PositionStorage interface { |
|||
Save(position mysql.Position) error |
|||
Load() (mysql.Position, error) |
|||
} |
|||
|
|||
// 文件存储
|
|||
type FileStorage struct { |
|||
filePath string |
|||
mutex sync.RWMutex |
|||
} |
|||
|
|||
func NewFileStorage(filePath string) *FileStorage { |
|||
return &FileStorage{filePath: filePath} |
|||
} |
|||
|
|||
func (fs *FileStorage) Save(pos mysql.Position) error { |
|||
fs.mutex.Lock() |
|||
defer fs.mutex.Unlock() |
|||
|
|||
data, err := json.Marshal(pos) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return os.WriteFile(fs.filePath, data, 0644) |
|||
} |
|||
|
|||
func (fs *FileStorage) Load() (mysql.Position, error) { |
|||
fs.mutex.RLock() |
|||
defer fs.mutex.RUnlock() |
|||
|
|||
data, err := os.ReadFile(fs.filePath) |
|||
if err != nil { |
|||
if os.IsNotExist(err) { |
|||
return mysql.Position{}, nil |
|||
} |
|||
return mysql.Position{}, err |
|||
} |
|||
|
|||
var pos mysql.Position |
|||
err = json.Unmarshal(data, &pos) |
|||
return pos, err |
|||
} |
|||
|
|||
// Redis存储
|
|||
type RedisStorage struct { |
|||
client *redis.Client |
|||
key string |
|||
} |
|||
|
|||
func NewRedisStorage(redisURL, key string) (*RedisStorage, error) { |
|||
opt, err := redis.ParseURL(redisURL) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
client := redis.NewClient(opt) |
|||
return &RedisStorage{client: client, key: key}, nil |
|||
} |
|||
|
|||
func (rs *RedisStorage) Save(pos mysql.Position) error { |
|||
data, err := json.Marshal(pos) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return rs.client.Set(rs.client.Context(), rs.key, data, 0).Err() |
|||
} |
|||
|
|||
func (rs *RedisStorage) Load() (mysql.Position, error) { |
|||
data, err := rs.client.Get(rs.client.Context(), rs.key).Bytes() |
|||
if err == redis.Nil { |
|||
return mysql.Position{}, nil |
|||
} |
|||
if err != nil { |
|||
return mysql.Position{}, err |
|||
} |
|||
|
|||
var pos mysql.Position |
|||
err = json.Unmarshal(data, &pos) |
|||
return pos, err |
|||
} |
|||
@ -0,0 +1,112 @@ |
|||
package syncer |
|||
|
|||
import ( |
|||
"log" |
|||
"time" |
|||
|
|||
"github.com/go-mysql-org/go-mysql/canal" |
|||
"github.com/go-mysql-org/go-mysql/mysql" |
|||
|
|||
"xttbinlog/config" |
|||
"xttbinlog/handler" |
|||
"xttbinlog/storage" |
|||
) |
|||
|
|||
type DataSyncer struct { |
|||
canal *canal.Canal |
|||
handler *handler.EventHandler |
|||
storage storage.PositionStorage |
|||
config *config.Config |
|||
} |
|||
|
|||
func NewDataSyncer(cfg *config.Config) (*DataSyncer, error) { |
|||
// 创建canal实例
|
|||
c, err := canal.NewCanal(cfg.ToCanalConfig()) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
// 创建位置存储器
|
|||
var posStorage storage.PositionStorage |
|||
switch cfg.Storage.Type { |
|||
case "redis": |
|||
posStorage, err = storage.NewRedisStorage(cfg.Storage.RedisURL, "binlog_position") |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
case "file": |
|||
posStorage = storage.NewFileStorage(cfg.Storage.FilePath) |
|||
default: |
|||
posStorage = storage.NewFileStorage("./binlog.position") |
|||
} |
|||
|
|||
// 创建事件处理器
|
|||
eventHandler := handler.NewEventHandler(cfg.Sync.BatchSize, cfg.Sync.Workers) |
|||
|
|||
syncer := &DataSyncer{ |
|||
canal: c, |
|||
handler: eventHandler, |
|||
storage: posStorage, |
|||
config: cfg, |
|||
} |
|||
|
|||
c.SetEventHandler(eventHandler) |
|||
return syncer, nil |
|||
} |
|||
|
|||
func (ds *DataSyncer) Start() error { |
|||
// 加载保存的位置
|
|||
savedPos, err := ds.storage.Load() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
var startPos mysql.Position |
|||
|
|||
if savedPos.Name != "" { |
|||
log.Printf("从保存的位置开始: %s", savedPos) |
|||
startPos = savedPos |
|||
} else { |
|||
// 第一次启动，从当前位置开始
|
|||
pos, err := ds.canal.GetMasterPos() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
startPos = pos |
|||
log.Printf("第一次启动，从当前位置开始: %s", startPos) |
|||
} |
|||
|
|||
// 启动同步
|
|||
go ds.positionSaver() |
|||
|
|||
log.Printf("开始数据同步...") |
|||
return ds.canal.RunFrom(startPos) |
|||
} |
|||
|
|||
func (ds *DataSyncer) positionSaver() { |
|||
ticker := time.NewTicker(5 * time.Second) |
|||
defer ticker.Stop() |
|||
|
|||
for { |
|||
select { |
|||
case <-ticker.C: |
|||
pos := ds.canal.SyncedPosition() |
|||
if err := ds.storage.Save(pos); err != nil { |
|||
log.Printf("保存位置失败: %v", err) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
func (ds *DataSyncer) Stop() { |
|||
log.Printf("停止数据同步...") |
|||
|
|||
// 保存最后的位置
|
|||
pos := ds.canal.SyncedPosition() |
|||
if err := ds.storage.Save(pos); err != nil { |
|||
log.Printf("保存最后位置失败: %v", err) |
|||
} |
|||
|
|||
ds.handler.Close() |
|||
ds.canal.Close() |
|||
} |
|||
Loading…
Reference in new issue