| 特色 | +Canal | +mysql_stream | +go-mysql-transfer | +
|---|---|---|---|
| 开发语言 | +Java | +Python | +Golang | +
| 高可用 | +支持 | +支持 | +支持 | +
| 接收端 | +编码定制 | +Kafka等(MQ) | +Redis、MongoDB、Elasticsearch、RabbitMQ、Kafka、RocketMQ、HTTP API 后续支持更多 |
+
| 全量数据初始化 | +不支持 | +支持 | +支持 | +
| 数据格式 | +编码定制 | +Json（固定格式） | +Json（规则配置) 模板语法 Lua脚本 |
+
+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package global + +import ( + "fmt" + "io/ioutil" + "log" + "path/filepath" + "runtime" + "strings" + + "github.com/juju/errors" + "gopkg.in/yaml.v2" + + "go-mysql-transfer/util/files" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/nets" + "go-mysql-transfer/util/sys" +) + +const ( + _targetRedis = "REDIS" + _targetMongodb = "MONGODB" + _targetRocketmq = "ROCKETMQ" + _targetRabbitmq = "RABBITMQ" + _targetKafka = "KAFKA" + _targetElasticsearch = "ELASTICSEARCH" + _targetScript = "SCRIPT" + + RedisGroupTypeSentinel = "sentinel" + RedisGroupTypeCluster = "cluster" + + _dataDir = "store" + + _zkRootDir = "/transfer" // ZooKeeper and Etcd root + + _flushBulkInterval = 200 + _flushBulkSize = 100 + + // update or insert + UpsertAction = "upsert" +) + +var _config *Config + +type Config struct { + Target string `yaml:"target"` // 目标类型，支持redis、mongodb + + Addr string `yaml:"addr"` + User string `yaml:"user"` + Password string `yaml:"pass"` + Charset string `yaml:"charset"` + + SlaveID uint32 `yaml:"slave_id"` + Flavor string `yaml:"flavor"` + DataDir string `yaml:"data_dir"` + + DumpExec string `yaml:"mysqldump"` + SkipMasterData bool `yaml:"skip_master_data"` + + Maxprocs int `yaml:"maxprocs"` // 最大协程数，默认CPU核心数*2 + BulkSize int64 `yaml:"bulk_size"` + + FlushBulkInterval int `yaml:"flush_bulk_interval"` + + SkipNoPkTable bool `yaml:"skip_no_pk_table"` + + RuleConfigs []*Rule `yaml:"rule"` + + LoggerConfig *logs.Config `yaml:"logger"` // 日志配置 + + EnableExporter bool `yaml:"enable_exporter"` // 启用prometheus exporter，默认false + ExporterPort int `yaml:"exporter_addr"` // prometheus exporter端口 + + EnableWebAdmin bool `yaml:"enable_web_admin"` // 启用Web监控，默认false + WebAdminPort int `yaml:"web_admin_port"` // web监控端口,默认8060 + + Cluster *Cluster `yaml:"cluster"` // 集群配置 + // ------------------- REDIS ----------------- + RedisAddr string `yaml:"redis_addrs"` //redis地址 + RedisGroupType string `yaml:"redis_group_type"` //集群类型 sentinel或者cluster + RedisMasterName string `yaml:"redis_master_name"` //Master节点名称 + RedisPass string `yaml:"redis_pass"` //redis密码 + RedisDatabase int `yaml:"redis_database"` //redis数据库 + + // ------------------- ROCKETMQ ----------------- + RocketmqNameServers string `yaml:"rocketmq_name_servers"` //rocketmq命名服务地址，多个用逗号分隔 + RocketmqGroupName string `yaml:"rocketmq_group_name"` //rocketmq group name,默认为空 + RocketmqInstanceName string `yaml:"rocketmq_instance_name"` //rocketmq instance name,默认为空 + RocketmqAccessKey string `yaml:"rocketmq_access_key"` //访问控制 accessKey,默认为空 + RocketmqSecretKey string `yaml:"rocketmq_secret_key"` //访问控制 secretKey,默认为空 + + // ------------------- MONGODB ----------------- + MongodbAddr string `yaml:"mongodb_addrs"` //mongodb地址，多个用逗号分隔 + MongodbUsername string `yaml:"mongodb_username"` //mongodb用户名，默认为空 + MongodbPassword string `yaml:"mongodb_password"` //mongodb密码，默认为空 + + // ------------------- RABBITMQ ----------------- + RabbitmqAddr string `yaml:"rabbitmq_addr"` //连接字符串,如: amqp://guest:guest@localhost:5672/ + + // ------------------- KAFKA ----------------- + KafkaAddr string `yaml:"kafka_addrs"` //kafka连接地址，多个用逗号分隔 + KafkaSASLUser string `yaml:"kafka_sasl_user"` //kafka SASL_PLAINTEXT认证模式 用户名 + KafkaSASLPassword string `yaml:"kafka_sasl_password"` //kafka SASL_PLAINTEXT认证模式 密码 + + // ------------------- ES ----------------- + ElsAddr string `yaml:"es_addrs"` //Elasticsearch连接地址，多个用逗号分隔 + ElsUser string `yaml:"es_user"` //Elasticsearch用户名 + ElsPassword string `yaml:"es_password"` //Elasticsearch密码 + ElsVersion int `yaml:"es_version"` //Elasticsearch版本，支持6和7、默认为7 + + isReserveRawData bool //保留原始数据 + isMQ bool //是否消息队列 +} + +type Cluster struct { + Name string `yaml:"name"` + BindIp string `yaml:"bind_ip"` //绑定IP + ZkAddrs string `yaml:"zk_addrs"` + ZkAuthentication string `yaml:"zk_authentication"` + EtcdAddrs string `yaml:"etcd_addrs"` + EtcdUser string `yaml:"etcd_user"` + EtcdPassword string `yaml:"etcd_password"` +} + +func initConfig(fileName string) error { + data, err := ioutil.ReadFile(fileName) + if err != nil { + return errors.Trace(err) + } + + var c Config + + if err := yaml.Unmarshal(data, &c); err != nil { + return errors.Trace(err) + } + + if err := checkConfig(&c); err != nil { + return errors.Trace(err) + } + + if err := checkClusterConfig(&c); err != nil { + return errors.Trace(err) + } + + switch strings.ToUpper(c.Target) { + case _targetRedis: + if err := checkRedisConfig(&c); err != nil { + return errors.Trace(err) + } + case _targetRocketmq: + if err := checkRocketmqConfig(&c); err != nil { + return errors.Trace(err) + } + case _targetMongodb: + if err := checkMongodbConfig(&c); err != nil { + return errors.Trace(err) + } + case _targetRabbitmq: + if err := checkRabbitmqConfig(&c); err != nil { + return errors.Trace(err) + } + case _targetKafka: + if err := checkKafkaConfig(&c); err != nil { + return errors.Trace(err) + } + case _targetElasticsearch: + if err := checkElsConfig(&c); err != nil { + return errors.Trace(err) + } + case _targetScript: + + default: + return errors.Errorf("unsupported target: %s", c.Target) + } + + _config = &c + + return nil +} + +func checkConfig(c *Config) error { + if c.Target == "" { + return errors.Errorf("empty target not allowed") + } + + if c.Addr == "" { + return errors.Errorf("empty addr not allowed") + } + + if c.User == "" { + return errors.Errorf("empty user not allowed") + } + + if c.Password == "" { + return errors.Errorf("empty pass not allowed") + } + + if c.Charset == "" { + return errors.Errorf("empty charset not allowed") + } + + if c.SlaveID == 0 { + return errors.Errorf("empty slave_id not allowed") + } + + if c.Flavor == "" { + c.Flavor = "mysql" + } + + if c.FlushBulkInterval == 0 { + c.FlushBulkInterval = _flushBulkInterval + } + + if c.BulkSize == 0 { + c.BulkSize = _flushBulkSize + } + + if c.DataDir == "" { + c.DataDir = filepath.Join(sys.CurrentDirectory(), _dataDir) + } + + if err := files.MkdirIfNecessary(c.DataDir); err != nil { + return err + } + + if c.LoggerConfig == nil { + c.LoggerConfig = &logs.Config{ + Store: filepath.Join(c.DataDir, "log"), + } + } + if c.LoggerConfig.Store == "" { + c.LoggerConfig.Store = filepath.Join(c.DataDir, "log") + } + + if err := files.MkdirIfNecessary(c.LoggerConfig.Store); err != nil { + return err + } + + if c.ExporterPort == 0 { + c.ExporterPort = 9595 + } + + if c.WebAdminPort == 0 { + c.WebAdminPort = 8060 + } + + if c.Maxprocs <= 0 { + c.Maxprocs = runtime.NumCPU() * 2 + } + + if c.RuleConfigs == nil { + return errors.Errorf("empty rules not allowed") + } + + return nil +} + +func Cfg() *Config { + return _config +} + +func checkClusterConfig(c *Config) error { + if c.Cluster == nil { + return nil + } + + if c.Cluster.ZkAddrs == "" && c.Cluster.EtcdAddrs == "" { + return nil + } + + if c.Cluster.Name == "" { + return errors.Errorf("empty name not allowed in cluster") + } + + if c.Cluster.BindIp != "" && !nets.CheckIp(c.Cluster.BindIp) { + return errors.New("配置文件错误：配置项'bind_ip' 应为一个IP地址，可以为空") + } + if c.Cluster.BindIp == "" { + ips, err := nets.GetIpList() + if err != nil { + return err + } + if len(ips) > 1 { + return errors.New(fmt.Sprintf( + "检测到机器上存在多个IP地址：%v，无法确定向其他集群节点暴露那个IP。请在配置文件'bind_ip'配置项中指定", ips)) + } + c.Cluster.BindIp = ips[0] + } + + if c.IsZk() { + log.Println("cluster by Zookeeper") + } + if c.IsEtcd() { + log.Println("cluster by Etcd") + } + + return nil +} + +func checkRedisConfig(c *Config) error { + if len(c.RedisAddr) == 0 { + return errors.Errorf("empty redis_addrs not allowed") + } + + addrList := strings.Split(c.RedisAddr, ",") + if len(addrList) > 1 { + if c.RedisGroupType == "" { + return errors.Errorf("empty group_type not allowed") + } + if c.RedisGroupType == RedisGroupTypeSentinel && c.RedisMasterName == "" { + return errors.Errorf("empty master_name not allowed") + } + } + + c.isReserveRawData = true + return nil +} + +func checkRocketmqConfig(c *Config) error { + if len(c.RocketmqNameServers) == 0 { + return errors.Errorf("empty rocketmq_name_servers not allowed") + } + + c.isReserveRawData = true + c.isMQ = true + return nil +} + +func checkMongodbConfig(c *Config) error { + if len(c.MongodbAddr) == 0 { + return errors.Errorf("empty mongodb_addrs not allowed") + } + + return nil +} + +func checkRabbitmqConfig(c *Config) error { + if len(c.RabbitmqAddr) == 0 { + return errors.Errorf("empty rabbitmq_addr not allowed") + } + + c.isReserveRawData = true + c.isMQ = true + return nil +} + +func checkKafkaConfig(c *Config) error { + if len(c.KafkaAddr) == 0 { + return errors.Errorf("empty kafka_addrs not allowed") + } + + c.isReserveRawData = true + c.isMQ = true + return nil +} + +func checkElsConfig(c *Config) error { + if len(c.ElsAddr) == 0 { + return errors.Errorf("empty es_addrs not allowed") + } + + if !strings.HasPrefix(c.ElsAddr, "http") { + c.ElsAddr = "http://" + c.ElsAddr + } + + if c.ElsVersion == 0 { + c.ElsVersion = 7 + } + + if !(c.ElsVersion == 6 || c.ElsVersion == 7) { + return errors.Errorf("elasticsearch version must 6 or 7") + } + + return nil +} + +func (c *Config) IsCluster() bool { + if !c.IsZk() && !c.IsEtcd() { + return false + } + + return true +} + +func (c *Config) IsZk() bool { + if c.Cluster == nil { + return false + } + if c.Cluster.ZkAddrs == "" { + return false + } + + return true +} + +func (c *Config) IsEtcd() bool { + if c.Cluster == nil { + return false + } + if c.Cluster.EtcdAddrs == "" { + return false + } + + return true +} + +func (c *Config) IsRedis() bool { + return strings.ToUpper(c.Target) == _targetRedis +} + +func (c *Config) IsRocketmq() bool { + return strings.ToUpper(c.Target) == _targetRocketmq +} + +func (c *Config) IsMongodb() bool { + return strings.ToUpper(c.Target) == _targetMongodb +} + +func (c *Config) IsRabbitmq() bool { + return strings.ToUpper(c.Target) == _targetRabbitmq +} + +func (c *Config) IsKafka() bool { + return strings.ToUpper(c.Target) == _targetKafka +} + +func (c *Config) IsEls() bool { + return strings.ToUpper(c.Target) == _targetElasticsearch +} + +func (c *Config) IsScript() bool { + return strings.ToUpper(c.Target) == _targetScript +} + +func (c *Config) IsExporterEnable() bool { + return c.EnableExporter +} + +func (c *Config) IsReserveRawData() bool { + return c.isReserveRawData +} + +func (c *Config) IsMQ() bool { + return c.isMQ +} + +func (c *Config) Destination() string { + var des string + switch strings.ToUpper(c.Target) { + case _targetRedis: + des += "redis(" + des += c.RedisAddr + des += ")" + case _targetRocketmq: + des += "rocketmq(" + des += c.RocketmqNameServers + des += ")" + case _targetMongodb: + des += "mongodb(" + des += c.MongodbAddr + des += ")" + case _targetRabbitmq: + des += "rabbitmq(" + des += c.RabbitmqAddr + des += ")" + case _targetKafka: + des += "kafka(" + des += c.KafkaAddr + des += ")" + case _targetElasticsearch: + des += "elasticsearch(" + des += c.ElsAddr + des += ")" + case _targetScript: + des += "Lua Script" + } + return des +} + +func (c *Config) DestStdName() string { + switch strings.ToUpper(c.Target) { + case _targetRedis: + return "Redis" + case _targetRocketmq: + return "RocketMQ" + case _targetMongodb: + return "MongoDB" + case _targetRabbitmq: + return "RabbitMQ" + case _targetKafka: + return "Kafka" + case _targetElasticsearch: + return "Elasticsearch" + } + + return "" +} + +func (c *Config) DestAddr() string { + switch strings.ToUpper(c.Target) { + case _targetRedis: + return c.RedisAddr + case _targetRocketmq: + return c.RocketmqNameServers + case _targetMongodb: + return c.MongodbAddr + case _targetRabbitmq: + return c.RabbitmqAddr + case _targetKafka: + return c.KafkaAddr + case _targetElasticsearch: + return c.ElsAddr + } + + return "" +} + +func (c *Config) ZkRootDir() string { + return _zkRootDir +} + +func (c *Config) ZkClusterDir() string { + return _zkRootDir + "/" + c.Cluster.Name +} + +func (c *Config) ZkPositionDir() string { + return _zkRootDir + "/" + c.Cluster.Name + "/position" +} + +func (c *Config) ZkElectionDir() string { + return _zkRootDir + "/" + c.Cluster.Name + "/election" +} + +func (c *Config) ZkElectedDir() string { + return _zkRootDir + "/" + c.Cluster.Name + "/elected" +} + +func (c *Config) ZkNodesDir() string { + return _zkRootDir + "/" + c.Cluster.Name + "/nodes" +} diff --git a/godb/global/context.go b/godb/global/context.go new file mode 100644 index 0000000..4bf51c1 --- /dev/null +++ b/godb/global/context.go @@ -0,0 +1,87 @@ +package global + +import ( + "fmt" + "log" + "runtime" + "strconv" + "syscall" + "time" + + sidlog "github.com/siddontang/go-log/log" + "go-mysql-transfer/util/logs" +) + +var ( + _pid int + _coordinator int + _leaderFlag bool + _leaderNode string + _currentNode string + _bootTime time.Time +) + +func SetLeaderFlag(flag bool) { + _leaderFlag = flag +} + +func IsLeader() bool { + return _leaderFlag +} + +func SetLeaderNode(leader string) { + _leaderNode = leader +} + +func LeaderNode() string { + return _leaderNode +} + +func CurrentNode() string { + return _currentNode +} + +func IsFollower() bool { + return !_leaderFlag +} + +func BootTime() time.Time { + return _bootTime +} + +func Initialize(configPath string) error { + if err := initConfig(configPath); err != nil { + return err + } + runtime.GOMAXPROCS(_config.Maxprocs) + + // 初始化global logger + if err := logs.Initialize(_config.LoggerConfig); err != nil { + return err + } + + streamHandler, err := sidlog.NewStreamHandler(logs.Writer()) + if err != nil { + return err + } + agent := sidlog.New(streamHandler, sidlog.Ltime|sidlog.Lfile|sidlog.Llevel) + sidlog.SetDefaultLogger(agent) + + _bootTime = time.Now() + _pid = syscall.Getpid() + + if _config.IsCluster(){ + if _config.EnableWebAdmin { + _currentNode = _config.Cluster.BindIp + ":" + strconv.Itoa(_config.WebAdminPort) + } else { + _currentNode = _config.Cluster.BindIp + ":" + strconv.Itoa(_pid) + } + } + + log.Println(fmt.Sprintf("process id: %d", _pid)) + log.Println(fmt.Sprintf("GOMAXPROCS :%d", _config.Maxprocs)) + log.Println(fmt.Sprintf("source %s(%s)", _config.Flavor, _config.Addr)) + log.Println(fmt.Sprintf("destination %s", _config.Destination())) + + return nil +} diff --git a/godb/global/rule.go b/godb/global/rule.go new file mode 100644 index 0000000..9847825 --- /dev/null +++ b/godb/global/rule.go @@ -0,0 +1,694 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package global + +import ( + "io/ioutil" + "path/filepath" + "strings" + "sync" + "text/template" + + "github.com/juju/errors" + "github.com/siddontang/go-mysql/schema" + "github.com/vmihailenco/msgpack" + "github.com/yuin/gopher-lua" + "github.com/yuin/gopher-lua/parse" + + "go-mysql-transfer/model" + "go-mysql-transfer/util/dates" + "go-mysql-transfer/util/files" + "go-mysql-transfer/util/stringutil" +) + +const ( + RedisStructureString = "String" + RedisStructureHash = "Hash" + RedisStructureList = "List" + RedisStructureSet = "Set" + RedisStructureSortedSet = "SortedSet" + + ValEncoderJson = "json" + ValEncoderKVCommas = "kv-commas" + ValEncoderVCommas = "v-commas" +) + +var ( + _ruleInsMap = make(map[string]*Rule) + _lockOfRuleInsMap sync.RWMutex +) + +type EsMapping struct { + Column string `yaml:"column"` // 数据库列名称 + Field string `yaml:"field"` // 映射后的ES字段名称 + Type string `yaml:"type"` // ES字段类型 + Analyzer string `yaml:"analyzer"` // ES分词器 + Format string `yaml:"format"` // 日期格式 +} + +type Rule struct { + Schema string `yaml:"schema"` + Table string `yaml:"table"` + OrderByColumn string `yaml:"order_by_column"` + ColumnLowerCase bool `yaml:"column_lower_case"` // 列名称转为小写 + ColumnUpperCase bool `yaml:"column_upper_case"` // 列名称转为大写 + ColumnUnderscoreToCamel bool `yaml:"column_underscore_to_camel"` // 列名称下划线转驼峰 + IncludeColumnConfig string `yaml:"include_columns"` // 包含的列 + ExcludeColumnConfig string `yaml:"exclude_columns"` // 排除掉的列 + ColumnMappingConfigs string `yaml:"column_mappings"` // 列名称映射 + DefaultColumnValueConfig string `yaml:"default_column_values"` // 默认的字段和值 + // #值编码，支持json、kv-commas、v-commas；默认为json；json形如：{"id":123,"name":"wangjie"} 、kv-commas形如：id=123,name="wangjie"、v-commas形如：123,wangjie + ValueEncoder string `yaml:"value_encoder"` + ValueFormatter string `yaml:"value_formatter"` //格式化定义key,{id}表示字段id的值、{name}表示字段name的值 + LuaScript string `yaml:"lua_script"` //lua 脚本 + LuaFilePath string `yaml:"lua_file_path"` //lua 文件地址 + DateFormatter string `yaml:"date_formatter"` //date类型格式化， 不填写默认2006-01-02 + DatetimeFormatter string `yaml:"datetime_formatter"` //datetime、timestamp类型格式化，不填写默认RFC3339(2006-01-02T15:04:05Z07:00) + + ReserveRawData bool `yaml:"reserve_raw_data"` // 保留update之前的数据，针对KAFKA、RABBITMQ、ROCKETMQ有效 + + // ------------------- REDIS ----------------- + //对应redis的5种数据类型 String、Hash(字典) 、List(列表) 、Set(集合)、Sorted Set(有序集合) + RedisStructure string `yaml:"redis_structure"` + RedisKeyPrefix string `yaml:"redis_key_prefix"` //key的前缀 + RedisKeyColumn string `yaml:"redis_key_column"` //使用哪个列的值作为key，不填写默认使用主键 + // 格式化定义key,如{id}-{name}；{id}表示字段id的值、{name}表示字段name的值 + RedisKeyFormatter string `yaml:"redis_key_formatter"` + RedisKeyValue string `yaml:"redis_key_value"` // key的值，固定值 + // hash的field前缀，仅redis_structure为hash时起作用 + RedisHashFieldPrefix string `yaml:"redis_hash_field_prefix"` + // 使用哪个列的值作为hash的field，仅redis_structure为hash时起作用 + RedisHashFieldColumn string `yaml:"redis_hash_field_column"` + // Sorted Set(有序集合)的Score + RedisSortedSetScoreColumn string `yaml:"redis_sorted_set_score_column"` + RedisKeyColumnIndex int + RedisKeyColumnIndexs []int + RedisHashFieldColumnIndex int + RedisHashFieldColumnIndexs []int + RedisSortedSetScoreColumnIndex int + RedisKeyTmpl *template.Template + + // ------------------- ROCKETMQ ----------------- + RocketmqTopic string `yaml:"rocketmq_topic"` //rocketmq topic名称，可以为空，为空时使用表名称 + + // ------------------- MONGODB ----------------- + MongodbDatabase string `yaml:"mongodb_database"` //mongodb database 不能为空 + MongodbCollection string `yaml:"mongodb_collection"` //mongodb collection，可以为空，默认使用表(Table)名称 + + // ------------------- RABBITMQ ----------------- + RabbitmqQueue string `yaml:"rabbitmq_queue"` //queue名称,可以为空，默认使用表(Table)名称 + + // ------------------- KAFKA ----------------- + KafkaTopic string `yaml:"kafka_topic"` //TOPIC名称,可以为空，默认使用表(Table)名称 + + // ------------------- ES ----------------- + ElsIndex string `yaml:"es_index"` //Elasticsearch Index,可以为空，默认使用表(Table)名称 + ElsType string `yaml:"es_type"` //es6.x以后一个Index只能拥有一个Type,可以为空，默认使用_doc; es7.x版本此属性无效 + EsMappings []*EsMapping `yaml:"es_mappings"` //Elasticsearch mappings映射关系,可以为空，为空时根据数据类型自己推导 + + // --------------- no config ---------------- + TableInfo *schema.Table + TableColumnSize int + IsCompositeKey bool //是否联合主键 + DefaultColumnValueMap map[string]string + PaddingMap map[string]*model.Padding + LuaProto *lua.FunctionProto + LuaFunction *lua.LFunction + ValueTmpl *template.Template +} + +func RuleDeepClone(res *Rule) (*Rule, error) { + data, err := msgpack.Marshal(res) + if err != nil { + return nil, err + } + + var r Rule + err = msgpack.Unmarshal(data, &r) + if err != nil { + return nil, err + } + + return &r, nil +} + +func RuleKey(schema string, table string) string { + return strings.ToLower(schema + ":" + table) +} + +func AddRuleIns(ruleKey string, r *Rule) { + _lockOfRuleInsMap.Lock() + defer _lockOfRuleInsMap.Unlock() + + _ruleInsMap[ruleKey] = r +} + +func RuleIns(ruleKey string) (*Rule, bool) { + _lockOfRuleInsMap.RLock() + defer _lockOfRuleInsMap.RUnlock() + + r, ok := _ruleInsMap[ruleKey] + + return r, ok +} + +func RuleInsExist(ruleKey string) bool { + _lockOfRuleInsMap.RLock() + defer _lockOfRuleInsMap.RUnlock() + + _, ok := _ruleInsMap[ruleKey] + + return ok +} + +func RuleInsTotal() int { + _lockOfRuleInsMap.RLock() + defer _lockOfRuleInsMap.RUnlock() + + return len(_ruleInsMap) +} + +func RuleInsList() []*Rule { + _lockOfRuleInsMap.RLock() + defer _lockOfRuleInsMap.RUnlock() + + list := make([]*Rule, 0, len(_ruleInsMap)) + for _, rule := range _ruleInsMap { + list = append(list, rule) + } + + return list +} + +func RuleKeyList() []string { + _lockOfRuleInsMap.RLock() + defer _lockOfRuleInsMap.RUnlock() + + list := make([]string, 0, len(_ruleInsMap)) + for k, _ := range _ruleInsMap { + list = append(list, k) + } + + return list +} + +func (s *Rule) Initialize() error { + if err := s.buildPaddingMap(); err != nil { + return err + } + + if s.ValueEncoder == "" { + s.ValueEncoder = ValEncoderJson + } + + if s.ValueFormatter != "" { + tmpl, err := template.New(s.TableInfo.Name).Parse(s.ValueFormatter) + if err != nil { + return err + } + s.ValueTmpl = tmpl + s.ValueEncoder = "" + } + + if s.DefaultColumnValueConfig != "" { + dm := make(map[string]string) + for _, t := range strings.Split(s.DefaultColumnValueConfig, ",") { + tt := strings.Split(t, "=") + if len(tt) != 2 { + return errors.Errorf("default_field_value format error in rule") + } + field := tt[0] + value := tt[1] + dm[field] = value + } + s.DefaultColumnValueMap = dm + } + + if s.DateFormatter != "" { + s.DateFormatter = dates.ConvertGoFormat(s.DateFormatter) + } + + if s.DatetimeFormatter != "" { + s.DatetimeFormatter = dates.ConvertGoFormat(s.DatetimeFormatter) + } + + if _config.IsRedis() { + if err := s.initRedisConfig(); err != nil { + return err + } + } + + if _config.IsRocketmq() { + if err := s.initRocketConfig(); err != nil { + return err + } + } + + if _config.IsMongodb() { + if err := s.initMongoConfig(); err != nil { + return err + } + } + + if _config.IsRabbitmq() { + if err := s.initRabbitmqConfig(); err != nil { + return err + } + } + + if _config.IsKafka() { + if err := s.initKafkaConfig(); err != nil { + return err + } + } + + if _config.IsEls() { + if err := s.initElsConfig(); err != nil { + return err + } + } + + if _config.IsScript() { + if s.LuaScript == "" && s.LuaFilePath == "" { + return errors.New("empty lua script not allowed") + } + } + + return nil +} + +func (s *Rule) AfterUpdateTableInfo() error { + if err := s.buildPaddingMap(); err != nil { + return err + } + + if _config.IsRedis() { + if err := s.initRedisConfig(); err != nil { + return err + } + } + + if _config.IsRocketmq() { + if err := s.initRocketConfig(); err != nil { + return err + } + } + + if _config.IsMongodb() { + if err := s.initMongoConfig(); err != nil { + return err + } + } + + if _config.IsRabbitmq() { + if err := s.initRabbitmqConfig(); err != nil { + return err + } + } + + if _config.IsKafka() { + if err := s.initKafkaConfig(); err != nil { + return err + } + } + + if _config.IsEls() { + if err := s.initElsConfig(); err != nil { + return err + } + } + + if _config.IsScript() { + if s.LuaScript == "" || s.LuaFilePath == "" { + return errors.New("empty lua script not allowed") + } + } + + return nil +} + +func (s *Rule) buildPaddingMap() error { + paddingMap := make(map[string]*model.Padding) + mappings := make(map[string]string) + + if s.ColumnMappingConfigs != "" { + ls := strings.Split(s.ColumnMappingConfigs, ",") + for _, t := range ls { + cmc := strings.Split(t, "=") + if len(cmc) != 2 { + return errors.Errorf("column_mappings format error in rule") + } + column := cmc[0] + mapped := cmc[1] + _, index := s.TableColumn(column) + if index < 0 { + return errors.Errorf("column_mappings must be table column") + } + mappings[strings.ToUpper(column)] = mapped + } + } + + if len(s.EsMappings) > 0 { + for _, mapping := range s.EsMappings { + mappings[strings.ToUpper(mapping.Column)] = mapping.Field + } + } + + var includes []string + var excludes []string + + if s.IncludeColumnConfig != "" { + includes = strings.Split(s.IncludeColumnConfig, ",") + } + if s.ExcludeColumnConfig != "" { + excludes = strings.Split(s.ExcludeColumnConfig, ",") + } + + if len(includes) > 0 { + for _, c := range includes { + _, index := s.TableColumn(c) + if index < 0 { + return errors.New("include_field must be table column") + } + paddingMap[c] = s.newPadding(mappings, c) + } + } else { + for _, column := range s.TableInfo.Columns { + include := true + for _, exclude := range excludes { + if column.Name == exclude { + include = false + } + } + if include { + paddingMap[column.Name] = s.newPadding(mappings, column.Name) + } + } + } + + s.PaddingMap = paddingMap + + return nil +} + +func (s *Rule) newPadding(mappings map[string]string, columnName string) *model.Padding { + column, index := s.TableColumn(columnName) + + wrapName := s.WrapName(column.Name) + mapped, exist := mappings[strings.ToUpper(column.Name)] + if exist { + wrapName = mapped + } + + return &model.Padding{ + WrapName: wrapName, + + ColumnIndex: index, + ColumnName: column.Name, + ColumnType: column.Type, + ColumnMetadata: column, + } +} + +func (s *Rule) TableColumn(field string) (*schema.TableColumn, int) { + for index, c := range s.TableInfo.Columns { + if strings.ToUpper(c.Name) == strings.ToUpper(field) { + return &c, index + } + } + return nil, -1 +} + +func (s *Rule) WrapName(fieldName string) string { + if s.ColumnUnderscoreToCamel { + return stringutil.Case2Camel(strings.ToLower(fieldName)) + } + if s.ColumnLowerCase { + return strings.ToLower(fieldName) + } + if s.ColumnUpperCase { + return strings.ToUpper(fieldName) + } + return fieldName +} + +func (s *Rule) LuaEnable() bool { + if s.LuaScript == "" && s.LuaFilePath == "" { + return false + } + + return true +} + +func (s *Rule) initRedisConfig() error { + if s.LuaEnable() { + return nil + } + + if s.RedisStructure == "" { + return errors.Errorf("empty redis_structure not allowed in rule") + } + + switch strings.ToUpper(s.RedisStructure) { + case "STRING": + s.RedisStructure = RedisStructureString + if s.RedisKeyColumn == "" && s.RedisKeyFormatter == "" { + if s.IsCompositeKey { + for _, v := range s.TableInfo.PKColumns { + s.RedisKeyColumnIndexs = append(s.RedisKeyColumnIndexs, v) + } + s.RedisKeyColumnIndex = -1 + } else { + s.RedisKeyColumnIndex = s.TableInfo.PKColumns[0] + } + } + case "HASH": + s.RedisStructure = RedisStructureHash + if s.RedisKeyValue == "" { + return errors.New("empty redis_key_value not allowed") + } + // init hash field + if s.RedisHashFieldColumn == "" { + if s.IsCompositeKey { + for _, v := range s.TableInfo.PKColumns { + s.RedisHashFieldColumnIndexs = append(s.RedisHashFieldColumnIndexs, v) + } + s.RedisHashFieldColumnIndex = -1 + } else { + s.RedisHashFieldColumnIndex = s.TableInfo.PKColumns[0] + } + } else { + _, index := s.TableColumn(s.RedisHashFieldColumn) + if index < 0 { + return errors.New("redis_hash_field_column must be table column") + } + s.RedisHashFieldColumnIndex = index + } + case "LIST": + s.RedisStructure = RedisStructureList + if s.RedisKeyValue == "" { + return errors.New("empty redis_key_value not allowed in rule") + } + case "SET": + s.RedisStructure = RedisStructureSet + if s.RedisKeyValue == "" { + return errors.New("empty redis_key_value not allowed in rule") + } + case "SORTEDSET": + s.RedisStructure = RedisStructureSortedSet + if s.RedisKeyValue == "" { + return errors.New("empty redis_key_value not allowed in rule") + } + if s.RedisSortedSetScoreColumn == "" { + return errors.New("empty redis_sorted_set_score_column not allowed in rule") + } + _, index := s.TableColumn(s.RedisSortedSetScoreColumn) + if index < 0 { + return errors.New("redis_sorted_set_score_column must be table column") + } + s.RedisHashFieldColumnIndex = index + default: + return errors.Errorf("redis_structure must be string or hash or list or set") + } + + if s.RedisKeyColumn != "" { + _, index := s.TableColumn(s.RedisKeyColumn) + if index < 0 { + return errors.New("redis_key_column must be table column") + } + s.RedisKeyColumnIndex = index + s.RedisKeyFormatter = "" + } + + if s.RedisKeyFormatter != "" { + tmpl, err := template.New(s.TableInfo.Name).Parse(s.RedisKeyFormatter) + if err != nil { + return err + } + s.RedisKeyTmpl = tmpl + s.RedisKeyColumnIndex = -1 + } + + return nil +} + +func (s *Rule) initRocketConfig() error { + if !s.LuaEnable() { + if s.RocketmqTopic == "" { + s.RocketmqTopic = s.Table + } + } + + return nil +} + +func (s *Rule) initMongoConfig() error { + if !s.LuaEnable() { + if s.MongodbDatabase == "" { + return errors.New("empty mongodb_database not allowed in rule") + } + + if s.MongodbCollection == "" { + s.MongodbCollection = s.Table + } + } + + return nil +} + +func (s *Rule) initRabbitmqConfig() error { + if !s.LuaEnable() { + if s.RabbitmqQueue == "" { + s.RabbitmqQueue = s.Table + } + } + + return nil +} + +func (s *Rule) initElsConfig() error { + if s.ElsIndex == "" { + s.ElsIndex = s.Table + } + + if s.ElsType == "" { + s.ElsType = "_doc" + } + + if len(s.EsMappings) > 0 { + for _, m := range s.EsMappings { + if m.Field == "" { + return errors.New("empty field not allowed in es_mappings") + } + if m.Type == "" { + return errors.New("empty type not allowed in es_mappings") + } + if m.Column == "" && !s.LuaEnable() { + return errors.New("empty column not allowed in es_mappings") + } + } + } + + return nil +} + +func (s *Rule) initKafkaConfig() error { + if !s.LuaEnable() { + if s.KafkaTopic == "" { + s.KafkaTopic = s.Table + } + } + + return nil +} + +// 编译Lua +func (s *Rule) CompileLuaScript(dataDir string) error { + script := s.LuaScript + if s.LuaFilePath != "" { + var filePath string + if files.IsExist(s.LuaFilePath) { + filePath = s.LuaFilePath + } else { + filePath = filepath.Join(dataDir, s.LuaFilePath) + } + data, err := ioutil.ReadFile(filePath) + if err != nil { + return err + } + script = string(data) + } + + if script == "" { + return errors.New("empty lua script not allowed") + } + + s.LuaScript = script + + if _config.IsRedis() { + if !strings.Contains(script, `require("redisOps")`) { + return errors.New("lua script incorrect format") + } + + if !(strings.Contains(script, `SET(`) || + strings.Contains(script, `HSET(`) || + strings.Contains(script, `RPUSH(`) || + strings.Contains(script, `SADD(`) || + strings.Contains(script, `ZADD(`) || + strings.Contains(script, `DEL(`) || + strings.Contains(script, `HDEL(`) || + strings.Contains(script, `LREM(`) || + strings.Contains(script, `ZREM(`) || + strings.Contains(script, `SREM(`)) { + + return errors.New("lua script incorrect format") + } + } + + if _config.IsRocketmq() { + if !strings.Contains(script, `require("mqOps")`) { + return errors.New("lua script incorrect format") + } + + if !(strings.Contains(script, `SEND(`)) { + return errors.New("lua script incorrect format") + } + } + + if _config.IsEls() { + if !strings.Contains(script, `require("esOps")`) { + return errors.New("lua script incorrect format") + } + } + + reader := strings.NewReader(script) + chunk, err := parse.Parse(reader, script) + if err != nil { + return err + } + + var proto *lua.FunctionProto + proto, err = lua.Compile(chunk, script) + if err != nil { + return err + } + + s.LuaProto = proto + + return nil +} diff --git a/godb/go.mod b/godb/go.mod new file mode 100644 index 0000000..1900c9f --- /dev/null +++ b/godb/go.mod @@ -0,0 +1,40 @@ +module go-mysql-transfer + +go 1.14 + +require ( + github.com/Shopify/sarama v1.27.0 + github.com/apache/rocketmq-client-go/v2 v2.0.0 + github.com/gin-gonic/gin v1.6.3 + github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df + github.com/go-redis/redis v6.15.8+incompatible + github.com/go-sql-driver/mysql v1.5.0 + github.com/jmoiron/sqlx v1.2.0 // indirect + github.com/json-iterator/go v1.1.9 + github.com/juju/errors v0.0.0-20200330140219-3fe23663418f + github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd // indirect + github.com/layeh/gopher-json v0.0.0-20190114024228-97fed8db8427 + github.com/olivere/elastic v6.2.34+incompatible + github.com/olivere/elastic/v7 v7.0.19 + github.com/onsi/ginkgo v1.14.0 // indirect + github.com/pingcap/errors v0.11.4 + github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc + github.com/pkg/errors v0.9.1 + github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 + github.com/prometheus/client_golang v1.0.0 + github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e + github.com/satori/go.uuid v1.2.0 + github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 + github.com/siddontang/go-mysql v1.1.0 + github.com/streadway/amqp v1.0.0 + github.com/vmihailenco/msgpack v4.0.4+incompatible + github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e + go.etcd.io/bbolt v1.3.3 + go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 + go.mongodb.org/mongo-driver v1.4.0 + go.uber.org/atomic v1.6.0 + go.uber.org/zap v1.15.0 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.3.0 + github.com/sony/sonyflake v1.0.0 +) diff --git a/godb/go.sum b/godb/go.sum new file mode 100644 index 0000000..38f3eec --- /dev/null +++ b/godb/go.sum @@ -0,0 +1,693 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= +github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= +github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apache/rocketmq-client-go/v2 v2.0.0 h1:D6jFj3DcNjWyjWn5N/R7Eq8v5kLqlgkFnT/DNQFnWlM= +github.com/apache/rocketmq-client-go/v2 v2.0.0/go.mod h1:oEZKFDvS7sz/RWU0839+dQBupazyBV7WX5cP6nrio0Q= +github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs= +github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= +github.com/aws/aws-sdk-go v1.33.5 h1:p2fr1ryvNTU6avUWLI+/H7FGv0TBIjzVM5WDgXBBv4U= +github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= +github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= +github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= +github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= +github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-redis/redis v6.15.8+incompatible h1:BKZuG6mCnRj5AOaWJXoCgf6rqTYnYJLe4en2hxT7r9o= +github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3 h1:3CYI9xg87xNAD+es02gZxbX/ky4KQeoFBsNOzuoAQZg= +github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf h1:Ut4tTtPNmInWiEWJRernsWm688R0RN6PFO8sZhwI0sk= +github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= +github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/errors v0.0.0-20200330140219-3fe23663418f h1:MCOvExGLpaSIzLYB4iQXEHP4jYVU6vmzLNQPdMVrxnM= +github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9 h1:Y+lzErDTURqeXqlqYi4YBYbDd7ycU74gW1ADt57/bgY= +github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/juju/retry v0.0.0-20160928201858-1998d01ba1c3/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= +github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd h1:4MRI5TGW0cRgovUipCGLF4uF+31Fo8VzkV2753OAfEE= +github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd/go.mod h1:hpGvhGHPVbNBraRLZEhoQwFLMrjK8PSlO4D3nDjKYXo= +github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= +github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/layeh/gopher-json v0.0.0-20190114024228-97fed8db8427 h1:YDEoPyEnynw+2uhIXxeWOS1vwZhYfeP92FpyLmpYmwo= +github.com/layeh/gopher-json v0.0.0-20190114024228-97fed8db8427/go.mod h1:E/q28EyUVBgBQnONAVPIdwvEsv4Ve0vaCA9JWim4+3I= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s= +github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= +github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= +github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= +github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= +github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olivere/elastic v6.2.34+incompatible h1:GdvWBAqyIyEEUd+J2sSj6EnIaBywz7zZtN+Ps4JCv0g= +github.com/olivere/elastic v6.2.34+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= +github.com/olivere/elastic/v7 v7.0.19 h1:w4F6JpqOISadhYf/n0NR1cNj73xHqh4pzPwD1Gkidts= +github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= +github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= +github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= +github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8= +github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= +github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s= +github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= +github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= +github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= +github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= +github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1 h1:J5oimSv+0emw5e/D1ZX/zh2WcMv0pOVT9QKruXfvJbg= +github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= +github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= +github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= +github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= +github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5 h1:gJUprtXZ95CeFSb3FsDvQ3cSLB9c0W/lPYdqsaRSOng= +github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8= +github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= +github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc h1:RtwQ6Kb8QWkUcInh2vzSc8HtQ2VHNNBV6datScfyF3c= +github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc/go.mod h1:8DSQhELIQr/UqWaolPciDK1Nj6ARvD0NjBmQLCRJpxY= +github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= +github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= +github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e h1:TWwzCfLrj9GH5uaT0VcvdSnrHuwEntUfoHDTYdOzNNI= +github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= +github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 h1:FUL3b97ZY2EPqg2NbXKuMHs5pXJB9hjj1fDHnF2vl28= +github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e h1:CGjiMQ0wMH4wtNWrlj6kiTbkPt2F3rbYnhGX6TWLfco= +github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= +github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= +github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= +github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= +github.com/siddontang/go-mysql v1.1.0 h1:NfkS1skrPwUd3hsUqhc6jrv24dKTNMANxKRmDsf1fMc= +github.com/siddontang/go-mysql v1.1.0/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck= +github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 h1:N8Bg45zpk/UcpNGnfJt2y/3lRWASHNTUET8owPYCgYI= +github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= +github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/struCoder/pidusage v0.1.2/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= +github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= +github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= +github.com/tidwall/gjson v1.2.1 h1:j0efZLrZUvNerEf6xqoi0NjWMK5YlLrR7Guo/dxY174= +github.com/tidwall/gjson v1.2.1/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g= +github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= +github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= +github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= +github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d h1:ggUgChAeyge4NZ4QUw6lhHsVymzwSDJOZcE0s2X8S20= +github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e h1:oIpIX9VKxSCFrfjsKpluGbNPBGq9iNnT9crH781j9wY= +github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.mongodb.org/mongo-driver v1.4.0 h1:C8rFn1VF4GVEM/rG+dSoMmlm2pyQ9cs2/oRtUATejRU= +go.mongodb.org/mongo-driver v1.4.0/go.mod h1:llVBH2pkj9HywK0Dtdt6lDikOjFLbceHVu/Rc0iMKLs= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf h1:fnPsqIDRbCSgumaMCRpoIoF2s4qxv0xSSS0BVZUE/ss= +golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191107010934-f79515f33823 h1:akkRBeitX2EZP59KdtKw310CI4WGPCNPyrLbE7WZA8Y= +golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 h1:oFSK4421fpCKRrpzIpybyBVWyht05NegY9+L/3TLAZs= +google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4 h1:hILp2hNrRnYjZpmIbx70psAHbBSEcQ1NIzDcUbJ1b6g= +gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YCZPcD3lo9AoczZpucwc= +gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 h1:VO9oZbbkvTwqLimlQt15QNdOOBArT2dw/bvzsMZBiqQ= +sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= +stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= +stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/godb/main.go b/godb/main.go new file mode 100644 index 0000000..22759d6 --- /dev/null +++ b/godb/main.go @@ -0,0 +1,198 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/signal" + "regexp" + "syscall" + + "github.com/juju/errors" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/service" + "go-mysql-transfer/storage" + "go-mysql-transfer/util/stringutil" + "go-mysql-transfer/web" +) + +var ( + helpFlag bool + cfgPath string + stockFlag bool + positionFlag bool + statusFlag bool +) + +func init() { + flag.BoolVar(&helpFlag, "help", false, "this help") + flag.StringVar(&cfgPath, "config", "app.yml", "application config file") + flag.BoolVar(&stockFlag, "stock", false, "stock data import") + flag.BoolVar(&positionFlag, "position", false, "set dump position") + flag.BoolVar(&statusFlag, "status", false, "display application status") + flag.Usage = usage +} + +func main() { + + //stockFlag = true + + //cfgPath = "D:\\transfer\\app.yml" + + //cfgPath = "D:\\transfer\\release_test_luascript.yml" + + //cfgPath = "D:\\transfer\\rabbitmq_release_test_lua.yml" + //cfgPath = "D:\\transfer\\rabbitmq_release_test_rule.yml" + + // cfgPath = "D:\\transfer\\kafka_release_test_lua.yml" + // cfgPath = "D:\\transfer\\kafka_release_test_rule.yml" + + //cfgPath = "D:\\transfer\\rocketmq_release_test_lua.yml" + //cfgPath = "D:\\transfer\\rocketmq_release_test_rule.yml" + + //cfgPath = "D:\\transfer\\es7_release_test_lua.yml" + //cfgPath = "D:\\transfer\\es7_release_test_rule.yml" + + //cfgPath = "D:\\transfer\\es6_release_test_lua.yml" + //cfgPath = "D:\\transfer\\es6_release_test_rule.yml" + + //cfgPath = "D:\\transfer\\redis_release_test_lua.yml" + //cfgPath = "D:\\transfer\\redis_release_test_rule.yml" + + //cfgPath = "D:\\transfer\\mongo_release_test_rule.yml" + //cfgPath = "D:\\transfer\\mongo_release_test_lua.yml" + + flag.Parse() + if helpFlag { + flag.Usage() + return + } + + // 初始化global + err := global.Initialize(cfgPath) + if err != nil { + println(errors.ErrorStack(err)) + return + } + + if stockFlag { + doStock() + return + } + + // 初始化Storage + err = storage.Initialize() + if err != nil { + println(errors.ErrorStack(err)) + return + } + + if statusFlag { + doStatus() + return + } + + if positionFlag { + doPosition() + return + } + + err = service.Initialize() + if err != nil { + println(errors.ErrorStack(err)) + return + } + + if err := metrics.Initialize(); err != nil { + println(errors.ErrorStack(err)) + return + } + + if err := web.Start(); err != nil { + println(errors.ErrorStack(err)) + return + } + service.StartUp() // start application + + s := make(chan os.Signal, 1) + signal.Notify(s, os.Kill, os.Interrupt, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + sin := <-s + log.Printf("application stoped，signal: %s \n", sin.String()) + + web.Close() + service.Close() + storage.Close() +} + +func doStock() { + stock := service.NewStockService() + if err := stock.Run(); err != nil { + println(errors.ErrorStack(err)) + } + stock.Close() +} + +func doStatus() { + ps := storage.NewPositionStorage() + pos, _ := ps.Get() + fmt.Printf("The current dump position is : %s %d \n", pos.Name, pos.Pos) +} + +func doPosition() { + others := flag.Args() + if len(others) != 2 { + println("error: please input the binlog's File and Position") + return + } + f := others[0] + p := others[1] + + matched, _ := regexp.MatchString(".+\\.\\d+$", f) + if !matched { + println("error: The parameter File must be like: mysql-bin.000001") + return + } + + pp, err := stringutil.ToUint32(p) + if nil != err { + println("error: The parameter Position must be number") + return + } + ps := storage.NewPositionStorage() + pos := mysql.Position{ + Name: f, + Pos: pp, + } + ps.Save(pos) + fmt.Printf("The current dump position is : %s %d \n", f, pp) +} + +func usage() { + fmt.Fprintf(os.Stderr, `version: 1.0.0 +Usage: transfer [-c filename] [-s stock] + +Options: +`) + flag.PrintDefaults() +} diff --git a/godb/metrics/metrics.go b/godb/metrics/metrics.go new file mode 100644 index 0000000..58726b1 --- /dev/null +++ b/godb/metrics/metrics.go @@ -0,0 +1,227 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package metrics + +import ( + "fmt" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/siddontang/go-mysql/canal" + "go.uber.org/atomic" + + "go-mysql-transfer/global" +) + +const ( + DestStateOK = 1 + DestStateFail = 0 + + LeaderState = 1 + FollowerState = 0 +) + +var ( + leaderState atomic.Bool + destState atomic.Bool + delay atomic.Uint32 + insertRecord map[string]*atomic.Uint64 + updateRecord map[string]*atomic.Uint64 + deleteRecord map[string]*atomic.Uint64 +) + +var ( + leaderStateGauge = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "transfer_leader_state", + Help: "The cluster leader state: 0=false, 1=true", + }, + ) + + destStateGauge = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "transfer_destination_state", + Help: "The destination running state: 0=stopped, 1=ok", + }, + ) + + delayGauge = promauto.NewGauge( + prometheus.GaugeOpts{ + Name: "transfer_delay", + Help: "The transfer slave lag", + }, + ) + + insertCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "transfer_inserted_num", + Help: "The number of data inserted to destination", + }, []string{"table"}, + ) + + updateCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "transfer_updated_num", + Help: "The number of data updated to destination", + }, []string{"table"}, + ) + + deleteCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "transfer_deleted_num", + Help: "The number of data deleted from destination", + }, []string{"table"}, + ) +) + +func Initialize() error { + if global.Cfg().EnableExporter { + go func() { + http.Handle("/", promhttp.Handler()) + http.ListenAndServe(fmt.Sprintf(":%d", global.Cfg().ExporterPort), nil) + }() + } + if global.Cfg().EnableWebAdmin { + insertRecord = make(map[string]*atomic.Uint64) + updateRecord = make(map[string]*atomic.Uint64) + deleteRecord = make(map[string]*atomic.Uint64) + for _, k := range global.RuleKeyList() { + insertRecord[k] = &atomic.Uint64{} + updateRecord[k] = &atomic.Uint64{} + deleteRecord[k] = &atomic.Uint64{} + } + } + return nil +} + +func SetLeaderState(state int) { + if global.Cfg().EnableExporter { + leaderStateGauge.Set(float64(state)) + } + if global.Cfg().EnableWebAdmin { + leaderState.Store(LeaderState == state) + } +} + +func SetDestState(state int) { + if global.Cfg().EnableExporter { + destStateGauge.Set(float64(state)) + } + if global.Cfg().EnableWebAdmin { + destState.Store(DestStateOK == state) + } +} + +func DestState() bool { + return destState.Load() +} + +func SetTransferDelay(d uint32) { + if global.Cfg().EnableExporter { + delayGauge.Set(float64(d)) + } + if global.Cfg().EnableWebAdmin { + delay.Store(d) + } +} + +func UpdateActionNum(action, lab string) { + if global.Cfg().EnableExporter { + switch action { + case canal.InsertAction: + insertCounter.WithLabelValues(lab).Inc() + case canal.UpdateAction: + updateCounter.WithLabelValues(lab).Inc() + case canal.DeleteAction: + deleteCounter.WithLabelValues(lab).Inc() + } + } + if global.Cfg().EnableWebAdmin { + switch action { + case canal.InsertAction: + if v, ok := insertRecord[lab]; ok { + v.Inc() + } + case canal.UpdateAction: + if v, ok := updateRecord[lab]; ok { + v.Inc() + } + case canal.DeleteAction: + if v, ok := deleteRecord[lab]; ok { + v.Inc() + } + } + } +} + +func InsertAmount() uint64 { + var amount uint64 + for _, v := range insertRecord { + amount += v.Load() + } + return amount +} + +func UpdateAmount() uint64 { + var amount uint64 + for _, v := range updateRecord { + amount += v.Load() + } + return amount +} + +func DeleteAmount() uint64 { + var amount uint64 + for _, v := range deleteRecord { + amount += v.Load() + } + return amount +} + +func LabInsertAmount(lab string) uint64 { + var nn uint64 + n, ok := insertRecord[lab] + if ok { + nn = n.Load() + } + return nn +} + +func LabUpdateRecord(lab string) uint64 { + var nn uint64 + n, ok := updateRecord[lab] + if ok { + nn = n.Load() + } + return nn +} + +func LabDeleteRecord(lab string) uint64 { + var nn uint64 + n, ok := deleteRecord[lab] + if ok { + nn = n.Load() + } + return nn +} + +func LeaderFlag() bool { + return leaderState.Load() +} diff --git a/godb/model/padding.go b/godb/model/padding.go new file mode 100644 index 0000000..a7be17a --- /dev/null +++ b/godb/model/padding.go @@ -0,0 +1,12 @@ +package model + +import "github.com/siddontang/go-mysql/schema" + +type Padding struct { + WrapName string + + ColumnName string + ColumnIndex int + ColumnType int + ColumnMetadata *schema.TableColumn +} diff --git a/godb/model/pipeline.go b/godb/model/pipeline.go new file mode 100644 index 0000000..6312611 --- /dev/null +++ b/godb/model/pipeline.go @@ -0,0 +1,17 @@ +package model + +const ( + PipelineInfoNormal = 1 + PipelineInfoDisable = 2 +) + +// PipelineInfo 管道 +type PipelineInfo struct { + Id uint64 + Name string + SourceId uint64 + TargetId uint64 + Status uint8 + CreateTime int64 + UpdateTime int64 +} diff --git a/godb/model/replog.go b/godb/model/replog.go new file mode 100644 index 0000000..7e78c99 --- /dev/null +++ b/godb/model/replog.go @@ -0,0 +1,11 @@ +package model + +// Replog = Replicated Log +type Replog struct { + Id uint64 + Type uint8 + Target uint8 + TargetId uint8 + timestamp int64 + Context []byte +} diff --git a/godb/model/request.go b/godb/model/request.go new file mode 100644 index 0000000..c616d2d --- /dev/null +++ b/godb/model/request.go @@ -0,0 +1,31 @@ +package model + +import "sync" + +var RowRequestPool = sync.Pool{ + New: func() interface{} { + return new(RowRequest) + }, +} + +type RowRequest struct { + RuleKey string + Action string + Timestamp uint32 + Old []interface{} + Row []interface{} +} + +type PosRequest struct { + Name string + Pos uint32 + Force bool +} + +func BuildRowRequest() *RowRequest { + return RowRequestPool.Get().(*RowRequest) +} + +func ReleaseRowRequest(t *RowRequest) { + RowRequestPool.Put(t) +} diff --git a/godb/model/respond.go b/godb/model/respond.go new file mode 100644 index 0000000..d575195 --- /dev/null +++ b/godb/model/respond.go @@ -0,0 +1,93 @@ +package model + +import "sync" + +var mqRespondPool = sync.Pool{ + New: func() interface{} { + return new(MQRespond) + }, +} + +var esRespondPool = sync.Pool{ + New: func() interface{} { + return new(ESRespond) + }, +} + +var mongoRespondPool = sync.Pool{ + New: func() interface{} { + return new(MongoRespond) + }, +} + +var redisRespondPool = sync.Pool{ + New: func() interface{} { + return new(RedisRespond) + }, +} + +type MQRespond struct { + Topic string `json:"-"` + Action string `json:"action"` + Timestamp uint32 `json:"timestamp"` + Raw interface{} `json:"raw,omitempty"` + Date interface{} `json:"date"` + ByteArray []byte `json:"-"` +} + +type ESRespond struct { + Index string + Id string + Action string + Date string +} + +type MongoRespond struct { + RuleKey string + Collection string + Action string + Id interface{} + Table map[string]interface{} +} + +type RedisRespond struct { + Action string + Structure string + Key string + Field string + Score float64 + OldVal interface{} + Val interface{} +} + +func BuildMQRespond() *MQRespond { + return mqRespondPool.Get().(*MQRespond) +} + +func ReleaseMQRespond(t *MQRespond) { + mqRespondPool.Put(t) +} + +func BuildESRespond() *ESRespond { + return esRespondPool.Get().(*ESRespond) +} + +func ReleaseESRespond(t *ESRespond) { + esRespondPool.Put(t) +} + +func BuildMongoRespond() *MongoRespond { + return mongoRespondPool.Get().(*MongoRespond) +} + +func ReleaseMongoRespond(t *MongoRespond) { + mongoRespondPool.Put(t) +} + +func BuildRedisRespond() *RedisRespond { + return redisRespondPool.Get().(*RedisRespond) +} + +func ReleaseRedisRespond(t *RedisRespond) { + redisRespondPool.Put(t) +} diff --git a/godb/model/target.go b/godb/model/target.go new file mode 100644 index 0000000..aef102a --- /dev/null +++ b/godb/model/target.go @@ -0,0 +1,46 @@ +package model + +//Type +// 1: mongodb Addr Username Password +// 2: elasticsearch Addr User Password Version +// 3: redis Addr GroupType MasterName Pass Database +// 4: rocketmq NameServers GroupName InstanceName AccessKey SecretKey +// 5: kafka Addr SASLUser SASLPassword +// 6: rabbitmq Addr +// 7: script + +const ( + TargetTypeMongodb = 1 + TargetTypeElasticsearch = 2 + TargetTypeRedis = 3 + TargetTypeRocketmq = 4 + TargetTypeKafka = 5 + TargetTypeRabbitmq = 6 + TargetTypeScript = 7 +) + +// TargetInfo 目标 +type TargetInfo struct { + Id uint64 + Type uint8 + Name string + Addr string //地址 + Username string //用户名 + Password string //密码 + + GroupType string `yaml:"group_type"` //集群类型 sentinel或者cluster + MasterName string `yaml:"master_name"` //Master节点名称 + Database int `yaml:"database"` //数据库 + + Version int //版本 + + NameServers string //命名服务地址，多个用逗号分隔 + GroupName string //group name,默认为空 + InstanceName string //instance name,默认为空 + AccessKey string //访问控制 accessKey,默认为空 + SecretKey string //访问控制 secretKey,默认为空 + + SASLUser string `yaml:"sasl_user"` //SASL_PLAINTEXT认证模式 用户名 + SASLPassword string `yaml:"sasl_password"` //SASL_PLAINTEXT认证模式 密码 + +} diff --git a/godb/proto/transport.proto b/godb/proto/transport.proto new file mode 100644 index 0000000..3d5358b --- /dev/null +++ b/godb/proto/transport.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; +// 定义包名 +option go_package ="/transport"; + +service Transport { + // Ping + rpc Ping (PingRequest) returns (PongResponse); +} + +// 定义Ping请求消息结构 +message PingRequest { + // 类型 字段 = 标识号 + string token = 1; + string context = 2; +} + +// 定义 Pong响应消息结构 +message PongResponse { + string context = 1; +} \ No newline at end of file diff --git a/godb/service/cluster_service.go b/godb/service/cluster_service.go new file mode 100644 index 0000000..4d257df --- /dev/null +++ b/godb/service/cluster_service.go @@ -0,0 +1,65 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package service + +import ( + "log" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" +) + +type ClusterService struct { + electionSignal chan bool //选举信号 +} + +func (s *ClusterService) boot() error { + log.Println("start master election") + err := _electionService.Elect() + if err != nil { + return err + } + + s.startElectListener() + + return nil +} + +func (s *ClusterService) startElectListener() { + go func() { + for { + select { + case selected := <-s.electionSignal: + global.SetLeaderNode(_electionService.Leader()) + global.SetLeaderFlag(selected) + if selected { + metrics.SetLeaderState(metrics.LeaderState) + _transferService.StartUp() + } else { + metrics.SetLeaderState(metrics.FollowerState) + _transferService.stopDump() + } + } + } + + }() +} + +func (s *ClusterService) Nodes() []string { + return _electionService.Nodes() +} diff --git a/godb/service/election/election.go b/godb/service/election/election.go new file mode 100644 index 0000000..901d070 --- /dev/null +++ b/godb/service/election/election.go @@ -0,0 +1,37 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package election + +import ( + "go-mysql-transfer/global" +) + +type Service interface { + Elect() error + IsLeader() bool + Leader() string + Nodes() []string +} + +func NewElection(_informCh chan bool) Service { + if global.Cfg().IsZk() { + return newZkElection(_informCh) + } else { + return newEtcdElection(_informCh) + } +} diff --git a/godb/service/election/etcd_election.go b/godb/service/election/etcd_election.go new file mode 100644 index 0000000..7346ba5 --- /dev/null +++ b/godb/service/election/etcd_election.go @@ -0,0 +1,170 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package election + +import ( + "context" + "fmt" + "log" + "sync" + "time" + + "github.com/juju/errors" + "go.etcd.io/etcd/clientv3/concurrency" + "go.uber.org/atomic" + + "go-mysql-transfer/global" + "go-mysql-transfer/storage" + "go-mysql-transfer/util/etcds" + "go-mysql-transfer/util/logs" +) + +const _electionNodeTTL = 2 //秒 + +type etcdElection struct { + once sync.Once + + informCh chan bool + + selected atomic.Bool + ensured atomic.Bool + leader atomic.String +} + +func newEtcdElection(_informCh chan bool) *etcdElection { + return &etcdElection{ + informCh: _informCh, + } +} + +func (s *etcdElection) Elect() error { + s.doElect() + s.ensureFollower() + return nil +} + +func (s *etcdElection) doElect() { + go func() { + + for { + session, err := concurrency.NewSession(storage.EtcdConn(), concurrency.WithTTL(_electionNodeTTL)) + if err != nil { + logs.Error(err.Error()) + return + } + elc := concurrency.NewElection(session, global.Cfg().ZkElectionDir()) + ctx := context.Background() + if err = elc.Campaign(ctx, global.CurrentNode()); err != nil { + logs.Error(errors.ErrorStack(err)) + session.Close() + s.beFollower("") + continue + } + + select { + case <-session.Done(): + s.beFollower("") + continue + default: + s.beLeader() + err = etcds.UpdateOrCreate(global.Cfg().ZkElectedDir(), elc.Key(), storage.EtcdOps()) + if err != nil { + logs.Error(errors.ErrorStack(err)) + return + } + } + + shouldBreak := false + for !shouldBreak { + select { + case <-session.Done(): + logs.Warn("etcd session has done") + shouldBreak = true + s.beFollower("") + break + case <-ctx.Done(): + ctxTmp, _ := context.WithTimeout(context.Background(), time.Second*_electionNodeTTL) + elc.Resign(ctxTmp) + session.Close() + s.beFollower("") + return + } + } + } + }() +} + +func (s *etcdElection) IsLeader() bool { + return s.selected.Load() +} + +func (s *etcdElection) Leader() string { + return s.leader.Load() +} + +func (s *etcdElection) ensureFollower() { + go func() { + for { + if s.selected.Load() { + break + } + + k, _, err := etcds.Get(global.Cfg().ZkElectedDir(), storage.EtcdOps()) + if err != nil { + logs.Error(errors.ErrorStack(err)) + continue + } + + var l []byte + l, _, err = etcds.Get(string(k), storage.EtcdOps()) + if err != nil { + logs.Error(errors.ErrorStack(err)) + continue + } + + s.ensured.Store(true) + s.beFollower(string(l)) + break + } + }() +} + +func (s *etcdElection) Nodes() []string { + var nodes []string + ls, err := etcds.List("/transfer/myTransfer/election", storage.EtcdOps()) + if err == nil { + for _, v := range ls { + nodes = append(nodes, string(v.Value)) + } + } + return nodes +} + +func (s *etcdElection) beLeader() { + s.selected.Store(true) + s.leader.Store(global.CurrentNode()) + s.informCh <- s.selected.Load() + log.Println("the current node is the master") +} + +func (s *etcdElection) beFollower(leader string) { + s.selected.Store(false) + s.informCh <- s.selected.Load() + s.leader.Store(leader) + log.Println(fmt.Sprintf("The current node is the follower, master node is : %s", s.leader.Load())) +} diff --git a/godb/service/election/zk_election.go b/godb/service/election/zk_election.go new file mode 100644 index 0000000..cc05746 --- /dev/null +++ b/godb/service/election/zk_election.go @@ -0,0 +1,166 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package election + +import ( + "fmt" + "log" + "sync" + + "github.com/samuel/go-zookeeper/zk" + "go.uber.org/atomic" + + "go-mysql-transfer/global" + "go-mysql-transfer/storage" + "go-mysql-transfer/util/logs" +) + +type zkElection struct { + once sync.Once + informCh chan bool + + selected atomic.Bool + leader atomic.String + + connectingAmount atomic.Int64 + downgraded atomic.Bool +} + +func newZkElection(_informCh chan bool) *zkElection { + return &zkElection{ + informCh: _informCh, + } +} + +func (s *zkElection) Elect() error { + data := []byte(global.CurrentNode()) + acl := zk.WorldACL(zk.PermAll) + _, err := storage.ZKConn().Create(global.Cfg().ZkElectionDir(), data, zk.FlagEphemeral, acl) + if err == nil { + s.beLeader() + } else { + v, _, err := storage.ZKConn().Get(global.Cfg().ZkElectionDir()) + if err != nil { + return err + } + leader := string(v) + if leader == global.CurrentNode() && s.downgraded.Load() { + s.beLeader() + } else { + s.beFollower(leader) + } + } + + // register + dir := global.Cfg().ZkNodesDir() + "/" + global.CurrentNode() + storage.ZKConn().Create(dir, data, zk.FlagEphemeral, acl) + + s.once.Do(func() { + s.startConnectionWatchTask() + s.startNodeWatchTask() + }) + + return nil +} + +func (s *zkElection) IsLeader() bool { + return s.selected.Load() +} + +func (s *zkElection) Leader() string { + return s.leader.Load() +} + +func (s *zkElection) Nodes() []string { + v, _, err := storage.ZKConn().Children(global.Cfg().ZkNodesDir()) + if err != nil { + return nil + } + return v +} + +func (s *zkElection) beLeader() { + s.selected.Store(true) + s.leader.Store(global.CurrentNode()) + s.informCh <- s.selected.Load() + log.Println("the current node is the master") +} + +func (s *zkElection) beFollower(leader string) { + s.selected.Store(false) + s.leader.Store(leader) + s.informCh <- s.selected.Load() + log.Println(fmt.Sprintf("The current node is the follower, master node is : %s", leader)) +} + +func (s *zkElection) startConnectionWatchTask() { + logs.Info("Start zookeeper connection Status watch task") + go func() { + for { + select { + case event := <-storage.ZKStatusSignal(): + logs.Infof("ZK ConnStatus: %v", event) + if s.selected.Load() { + if zk.StateConnecting == event.State { + s.connectingAmount.Inc() + } + if s.connectingAmount.Load() > int64(len(storage.ZKAddresses())) { + s.downgrading() + } + } + if zk.StateHasSession == event.State { + s.connectingAmount.Store(0) + if s.downgraded.Load() { + logs.Info("zookeeper HasSession restart elect") + s.Elect() + s.downgraded.Store(false) + } + } + } + } + }() +} + +func (s *zkElection) startNodeWatchTask() { + go func() { + + logs.Info("Start zookeeper election node watch task") + _, _, ch, _ := storage.ZKConn().ChildrenW(global.Cfg().ZkElectionDir()) + for { + select { + case childEvent := <-ch: + if childEvent.Type == zk.EventNodeDeleted { + logs.Info("Start elect new master ...") + err := s.Elect() + if err != nil { + logs.Errorf("elect new master error %s ", err.Error()) + } + } + } + } + + }() +} + +func (s *zkElection) downgrading() { + if !s.downgraded.Load() { + log.Println("Lost contact with zookeeper, The current node degraded to Follower") + s.downgraded.Store(true) + s.beFollower("") + } +} \ No newline at end of file diff --git a/godb/service/endpoint/elastic6.go b/godb/service/endpoint/elastic6.go new file mode 100644 index 0000000..8f5214d --- /dev/null +++ b/godb/service/endpoint/elastic6.go @@ -0,0 +1,310 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "context" + "log" + "strings" + "sync" + + "github.com/juju/errors" + "github.com/olivere/elastic" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logagent" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/stringutil" +) + +type Elastic6Endpoint struct { + first string + hosts []string + client *elastic.Client + + retryLock sync.Mutex +} + +func newElastic6Endpoint() *Elastic6Endpoint { + r := &Elastic6Endpoint{} + r.hosts = strings.Split(global.Cfg().ElsAddr, ",") + r.first = r.hosts[0] + return r +} + +func (s *Elastic6Endpoint) Connect() error { + var options []elastic.ClientOptionFunc + options = append(options, elastic.SetErrorLog(logagent.NewElsLoggerAgent())) + options = append(options, elastic.SetURL(s.hosts...)) + if global.Cfg().ElsUser != "" && global.Cfg().ElsPassword != "" { + options = append(options, elastic.SetBasicAuth(global.Cfg().ElsUser, global.Cfg().Password)) + } + + client, err := elastic.NewClient(options...) + if err != nil { + return err + } + + s.client = client + return s.indexMapping() +} + +func (s *Elastic6Endpoint) indexMapping() error { + for _, rule := range global.RuleInsList() { + exists, err := s.client.IndexExists(rule.ElsIndex).Do(context.Background()) + if err != nil { + return err + } + if exists { + err = s.updateIndexMapping(rule) + } else { + err = s.insertIndexMapping(rule) + } + if err != nil { + return err + } + } + + return nil +} + +func (s *Elastic6Endpoint) insertIndexMapping(rule *global.Rule) error { + var properties map[string]interface{} + if rule.LuaEnable() { + properties = buildPropertiesByMappings(rule) + } else { + properties = buildPropertiesByRule(rule) + } + + mapping := map[string]interface{}{ + "mappings": map[string]interface{}{ + rule.ElsType: map[string]interface{}{ + "properties": properties, + }, + }, + } + body := stringutil.ToJsonString(mapping) + + ret, err := s.client.CreateIndex(rule.ElsIndex).Body(body).Do(context.Background()) + if err != nil { + return err + } + if !ret.Acknowledged { + return errors.Errorf("create index %s err", rule.ElsIndex) + } + logs.Infof("create index succeed, index: %s", body) + + return nil +} + +func (s *Elastic6Endpoint) updateIndexMapping(rule *global.Rule) error { + ret, err := s.client.GetMapping().Index(rule.ElsIndex).Do(context.Background()) + if err != nil { + return err + } + + if ret[rule.ElsIndex]==nil{ + return nil + } + retIndex := ret[rule.ElsIndex].(map[string]interface{}) + + if retIndex["mappings"] == nil { + return nil + } + retMaps := retIndex["mappings"].(map[string]interface{}) + + if retMaps["_doc"]==nil{ + return nil + } + retDoc := retMaps["_doc"].(map[string]interface{}) + + if retDoc["properties"]==nil{ + return nil + } + retPros := retDoc["properties"].(map[string]interface{}) + + var currents map[string]interface{} + if rule.LuaEnable() { + currents = buildPropertiesByMappings(rule) + } else { + currents = buildPropertiesByRule(rule) + } + + if len(retPros) < len(currents) { + properties := make(map[string]interface{}) + mapping := map[string]interface{}{ + "properties": properties, + } + for field, current := range currents { + if _, exist := retPros[field]; !exist { + properties[field] = current + } + } + doc := stringutil.ToJsonString(mapping) + ret, err := s.client.PutMapping().Index(rule.ElsIndex).Type(rule.ElsType).BodyString(doc).Do(context.Background()) + if err != nil { + return err + } + if !ret.Acknowledged { + return errors.Errorf("update index %s err", rule.ElsIndex) + } + logs.Infof("update index succeed, index: %s", doc) + } + + return nil +} + +func (s *Elastic6Endpoint) Ping() error { + if _, _, err := s.client.Ping(s.first).Do(context.Background()); err == nil { + return nil + } + + for _, host := range s.hosts { + if _, _, err := s.client.Ping(host).Do(context.Background()); err == nil { + return nil + } + } + + return errors.New("ssx") +} + +func (s *Elastic6Endpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + bulk := s.client.Bulk() + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoESOps(kvm, row.Action, rule) + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + for _, resp := range ls { + logs.Infof("action: %s, Index: %s , Id:%s, value: %v", resp.Action, resp.Index, resp.Id, resp.Date) + s.prepareBulk(resp.Action, resp.Index, rule.ElsType, resp.Id, resp.Date, bulk) + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + body := encodeValue(rule, kvm) + logs.Infof("action: %s, Index: %s , Id:%s, value: %v", row.Action, rule.ElsIndex, id, body) + s.prepareBulk(row.Action, rule.ElsIndex, rule.ElsType, stringutil.ToString(id), body, bulk) + } + } + + if bulk.NumberOfActions()==0{ + return nil + } + + r, err := bulk.Do(context.Background()) + if err != nil { + log.Println(err.Error()) + return err + } + + if len(r.Failed()) > 0 { + for _, f := range r.Failed() { + reason := f.Index + " " + f.Type + " " + f.Result + if f.Error == nil && "not_found" == f.Result { + return nil + } + + if f.Error != nil { + reason = f.Error.Reason + } + log.Println(reason) + return errors.New(reason) + } + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *Elastic6Endpoint) Stock(rows []*model.RowRequest) int64 { + if len(rows) == 0 { + return 0 + } + + bulk := s.client.Bulk() + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoESOps(kvm, row.Action, rule) + if err != nil { + logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + break + } + for _, resp := range ls { + s.prepareBulk(resp.Action, resp.Index, rule.ElsType, resp.Id, resp.Date, bulk) + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + body := encodeValue(rule, kvm) + s.prepareBulk(row.Action, rule.ElsIndex, rule.ElsType, stringutil.ToString(id), body, bulk) + } + } + + r, err := bulk.Do(context.Background()) + if err != nil { + logs.Error(errors.ErrorStack(err)) + return 0 + } + + return int64(len(r.Succeeded())) +} + +func (s *Elastic6Endpoint) prepareBulk(action, index, _type, id, doc string, bulk *elastic.BulkService) { + switch action { + case canal.InsertAction: + req := elastic.NewBulkIndexRequest().Index(index).Type(_type).Id(id).Doc(doc) + bulk.Add(req) + case canal.UpdateAction: + req := elastic.NewBulkUpdateRequest().Index(index).Type(_type).Id(id).Doc(doc) + bulk.Add(req) + case canal.DeleteAction: + req := elastic.NewBulkDeleteRequest().Index(index).Type(_type).Id(id) + bulk.Add(req) + } + logs.Infof("index: %s, type:%s, action:%s, doc: %s", index, _type, action, doc) +} + +func (s *Elastic6Endpoint) Close() { + if s.client != nil { + s.client.Stop() + } +} diff --git a/godb/service/endpoint/elastic7.go b/godb/service/endpoint/elastic7.go new file mode 100644 index 0000000..563cadf --- /dev/null +++ b/godb/service/endpoint/elastic7.go @@ -0,0 +1,312 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "context" + "log" + "sync" + + "github.com/juju/errors" + "github.com/olivere/elastic/v7" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logagent" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/stringutil" +) + +type Elastic7Endpoint struct { + first string + hosts []string + client *elastic.Client + + retryLock sync.Mutex +} + +func newElastic7Endpoint() *Elastic7Endpoint { + hosts := elsHosts(global.Cfg().ElsAddr) + r := &Elastic7Endpoint{} + r.hosts = hosts + r.first = hosts[0] + return r +} + +func (s *Elastic7Endpoint) Connect() error { + var options []elastic.ClientOptionFunc + options = append(options, elastic.SetErrorLog(logagent.NewElsLoggerAgent())) + options = append(options, elastic.SetURL(s.hosts...)) + if global.Cfg().ElsUser != "" && global.Cfg().ElsPassword != "" { + options = append(options, elastic.SetBasicAuth(global.Cfg().ElsUser, global.Cfg().Password)) + } + + client, err := elastic.NewClient(options...) + if err != nil { + return err + } + + s.client = client + return s.indexMapping() +} + +func (s *Elastic7Endpoint) indexMapping() error { + for _, rule := range global.RuleInsList() { + exists, err := s.client.IndexExists(rule.ElsIndex).Do(context.Background()) + if err != nil { + return err + } + if exists { + err = s.updateIndexMapping(rule) + } else { + err = s.insertIndexMapping(rule) + } + if err != nil { + return err + } + } + + return nil +} + +func (s *Elastic7Endpoint) insertIndexMapping(rule *global.Rule) error { + var properties map[string]interface{} + if rule.LuaEnable() { + properties = buildPropertiesByMappings(rule) + } else { + properties = buildPropertiesByRule(rule) + } + + mapping := map[string]interface{}{ + "mappings": map[string]interface{}{ + "properties": properties, + }, + } + body := stringutil.ToJsonString(mapping) + + ret, err := s.client.CreateIndex(rule.ElsIndex).Body(body).Do(context.Background()) + if err != nil { + return err + } + if !ret.Acknowledged { + return errors.Errorf("create index %s err", rule.ElsIndex) + } + + logs.Infof("create index: %s ,mappings: %s", rule.ElsIndex, body) + + return nil +} + +func (s *Elastic7Endpoint) updateIndexMapping(rule *global.Rule) error { + ret, err := s.client.GetMapping().Index(rule.ElsIndex).Do(context.Background()) + if err != nil { + return err + } + + if ret[rule.ElsIndex]==nil{ + return nil + } + retIndex := ret[rule.ElsIndex].(map[string]interface{}) + + if retIndex["mappings"] == nil { + return nil + } + retMaps := retIndex["mappings"].(map[string]interface{}) + + if retMaps["properties"] == nil { + return nil + } + retPros := retMaps["properties"].(map[string]interface{}) + + var currents map[string]interface{} + if rule.LuaEnable() { + currents = buildPropertiesByMappings(rule) + } else { + currents = buildPropertiesByRule(rule) + } + + if len(retPros) < len(currents) { + properties := make(map[string]interface{}) + mapping := map[string]interface{}{ + "properties": properties, + } + for field, current := range currents { + if _, exist := retPros[field]; !exist { + properties[field] = current + } + } + + doc := stringutil.ToJsonString(mapping) + ret, err := s.client.PutMapping().Index(rule.ElsIndex).BodyString(doc).Do(context.Background()) + if err != nil { + return err + } + if !ret.Acknowledged { + return errors.Errorf("update index %s err", rule.ElsIndex) + } + + logs.Infof("update index: %s ,properties: %s", rule.ElsIndex, doc) + } + + return nil +} + +func (s *Elastic7Endpoint) Ping() error { + if _, _, err := s.client.Ping(s.first).Do(context.Background()); err == nil { + return nil + } + + for _, host := range s.hosts { + if _, _, err := s.client.Ping(host).Do(context.Background()); err == nil { + return nil + } + } + + return errors.New("ssx") +} + +func (s *Elastic7Endpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + bulk := s.client.Bulk() + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoESOps(kvm, row.Action, rule) + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + for _, resp := range ls { + logs.Infof("action: %s, Index: %s , Id:%s, value: %v", resp.Action, resp.Index, resp.Id, resp.Date) + s.prepareBulk(resp.Action, resp.Index, resp.Id, resp.Date, bulk) + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + body := encodeValue(rule, kvm) + logs.Infof("action: %s, Index: %s , Id:%s, value: %v", row.Action, rule.ElsIndex, id, body) + s.prepareBulk(row.Action, rule.ElsIndex, stringutil.ToString(id), body, bulk) + } + } + + if bulk.NumberOfActions() == 0 { + return nil + } + + r, err := bulk.Do(context.Background()) + if err != nil { + return err + } + + if len(r.Failed()) > 0 { + for _, f := range r.Failed() { + reason := f.Index + " " + f.Type + " " + f.Result + if f.Error == nil && "not_found" == f.Result { + return nil + } + + if f.Error != nil { + reason = f.Error.Reason + } + log.Println(reason) + return errors.New(reason) + } + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *Elastic7Endpoint) Stock(rows []*model.RowRequest) int64 { + if len(rows) == 0 { + return 0 + } + + bulk := s.client.Bulk() + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoESOps(kvm, row.Action, rule) + if err != nil { + logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + break + } + for _, resp := range ls { + s.prepareBulk(resp.Action, resp.Index, resp.Id, resp.Date, bulk) + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + body := encodeValue(rule, kvm) + s.prepareBulk(row.Action, rule.ElsIndex, stringutil.ToString(id), body, bulk) + } + } + + r, err := bulk.Do(context.Background()) + if err != nil { + logs.Error(errors.ErrorStack(err)) + return 0 + } + + if len(r.Failed()) > 0 { + for _, f := range r.Failed() { + logs.Error(f.Error.Reason) + } + } + + return int64(len(r.Succeeded())) +} + +func (s *Elastic7Endpoint) prepareBulk(action, index, id, doc string, bulk *elastic.BulkService) { + switch action { + case canal.InsertAction: + req := elastic.NewBulkIndexRequest().Index(index).Id(id).Doc(doc) + bulk.Add(req) + case canal.UpdateAction: + req := elastic.NewBulkUpdateRequest().Index(index).Id(id).Doc(doc) + bulk.Add(req) + case canal.DeleteAction: + req := elastic.NewBulkDeleteRequest().Index(index).Id(id) + bulk.Add(req) + } + + logs.Infof("index: %s, doc: %s", index, doc) +} + +func (s *Elastic7Endpoint) Close() { + if s.client != nil { + s.client.Stop() + } +} diff --git a/godb/service/endpoint/endpoint.go b/godb/service/endpoint/endpoint.go new file mode 100644 index 0000000..20b1d53 --- /dev/null +++ b/godb/service/endpoint/endpoint.go @@ -0,0 +1,400 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "bytes" + "strconv" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-mysql/schema" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/stringutil" +) + +var json = jsoniter.ConfigCompatibleWithStandardLibrary + +const defaultDateFormatter = "2006-01-02" + +type Endpoint interface { + Connect() error + Ping() error + Consume(mysql.Position, []*model.RowRequest) error + Stock([]*model.RowRequest) int64 + Close() +} + +func NewEndpoint(ds *canal.Canal) Endpoint { + cfg := global.Cfg() + luaengine.InitActuator(ds) + + if cfg.IsRedis() { + return newRedisEndpoint() + } + + if cfg.IsMongodb() { + return newMongoEndpoint() + } + + if cfg.IsRocketmq() { + return newRocketEndpoint() + } + + if cfg.IsRabbitmq() { + return newRabbitEndpoint() + } + + if cfg.IsKafka() { + return newKafkaEndpoint() + } + + if cfg.IsEls() { + if cfg.ElsVersion == 6 { + return newElastic6Endpoint() + } + if cfg.ElsVersion == 7 { + return newElastic7Endpoint() + } + } + + if cfg.IsScript() { + return newScriptEndpoint() + } + + return nil +} + +func convertColumnData(value interface{}, col *schema.TableColumn, rule *global.Rule) interface{} { + if value == nil { + return nil + } + + switch col.Type { + case schema.TYPE_ENUM: + switch value := value.(type) { + case int64: + eNum := value - 1 + if eNum < 0 || eNum >= int64(len(col.EnumValues)) { + // we insert invalid enum value before, so return empty + logs.Warnf("invalid binlog enum index %d, for enum %v", eNum, col.EnumValues) + return "" + } + return col.EnumValues[eNum] + case string: + return value + case []byte: + return string(value) + } + case schema.TYPE_SET: + switch value := value.(type) { + case int64: + bitmask := value + sets := make([]string, 0, len(col.SetValues)) + for i, s := range col.SetValues { + if bitmask&int64(1<+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "github.com/siddontang/go-mysql/canal" + "log" + "strings" + "sync" + + "github.com/Shopify/sarama" + "github.com/juju/errors" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logs" +) + +type KafkaEndpoint struct { + client sarama.Client + producer sarama.AsyncProducer + + retryLock sync.Mutex +} + +func newKafkaEndpoint() *KafkaEndpoint { + r := &KafkaEndpoint{} + return r +} + +func (s *KafkaEndpoint) Connect() error { + cfg := sarama.NewConfig() + cfg.Producer.Partitioner = sarama.NewRandomPartitioner + + if global.Cfg().KafkaSASLUser != "" && global.Cfg().KafkaSASLPassword != "" { + cfg.Net.SASL.Enable = true + cfg.Net.SASL.User = global.Cfg().KafkaSASLUser + cfg.Net.SASL.Password = global.Cfg().KafkaSASLPassword + } + + var err error + var client sarama.Client + ls := strings.Split(global.Cfg().KafkaAddr, ",") + client, err = sarama.NewClient(ls, cfg) + if err != nil { + return errors.Errorf("unable to create kafka client: %q", err) + } + + var producer sarama.AsyncProducer + producer, err = sarama.NewAsyncProducerFromClient(client) + if err != nil { + return errors.Errorf("unable to create kafka producer: %q", err) + } + + s.producer = producer + s.client = client + + return nil +} + +func (s *KafkaEndpoint) Ping() error { + return s.client.RefreshMetadata() +} + +func (s *KafkaEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + var ms []*sarama.ProducerMessage + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + ls, err := s.buildMessages(row, rule) + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + ms = append(ms, ls...) + } else { + m, err := s.buildMessage(row, rule) + if err != nil { + return errors.Errorf(errors.ErrorStack(err)) + } + ms = append(ms, m) + } + } + + for _, m := range ms { + s.producer.Input() <- m + select { + case err := <-s.producer.Errors(): + return err + default: + } + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *KafkaEndpoint) Stock(rows []*model.RowRequest) int64 { + expect := true + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + ls, err := s.buildMessages(row, rule) + if err != nil { + logs.Errorf(errors.ErrorStack(err)) + expect = false + break + } + for _, m := range ls { + s.producer.Input() <- m + select { + case err := <-s.producer.Errors(): + logs.Error(err.Error()) + expect = false + break + default: + } + } + if !expect { + break + } + } else { + m, err := s.buildMessage(row, rule) + if err != nil { + logs.Errorf(errors.ErrorStack(err)) + expect = false + break + } + s.producer.Input() <- m + select { + case err := <-s.producer.Errors(): + logs.Error(err.Error()) + expect = false + break + default: + + } + } + } + + if !expect { + return 0 + } + + return int64(len(rows)) +} + +func (s *KafkaEndpoint) buildMessages(row *model.RowRequest, rule *global.Rule) ([]*sarama.ProducerMessage, error) { + var err error + var ls []*model.MQRespond + kvm := rowMap(row, rule, true) + if row.Action == canal.UpdateAction { + previous := oldRowMap(row, rule, true) + ls, err = luaengine.DoMQOps(kvm, previous, row.Action, rule) + } else { + ls, err = luaengine.DoMQOps(kvm, nil, row.Action, rule) + } + if err != nil { + return nil, errors.Errorf("lua 脚本执行失败 : %s ", err) + } + + var ms []*sarama.ProducerMessage + for _, resp := range ls { + m := &sarama.ProducerMessage{ + Topic: resp.Topic, + Value: sarama.ByteEncoder(resp.ByteArray), + } + logs.Infof("topic: %s, message: %s", resp.Topic, string(resp.ByteArray)) + ms = append(ms, m) + } + + return ms, nil +} + +func (s *KafkaEndpoint) buildMessage(row *model.RowRequest, rule *global.Rule) (*sarama.ProducerMessage, error) { + kvm := rowMap(row, rule, false) + resp := new(model.MQRespond) + resp.Action = row.Action + resp.Timestamp = row.Timestamp + if rule.ValueEncoder == global.ValEncoderJson { + resp.Date = kvm + } else { + resp.Date = encodeValue(rule, kvm) + } + + if rule.ReserveRawData && canal.UpdateAction == row.Action { + resp.Raw = oldRowMap(row, rule, false) + } + + body, err := json.Marshal(resp) + if err != nil { + return nil, err + } + m := &sarama.ProducerMessage{ + Topic: rule.KafkaTopic, + Value: sarama.ByteEncoder(body), + } + logs.Infof("topic: %s, message: %s", rule.KafkaTopic, string(body)) + return m, nil +} + +func (s *KafkaEndpoint) Close() { + if s.producer != nil { + s.producer.Close() + } + if s.client != nil { + s.client.Close() + } +} diff --git a/godb/service/endpoint/mongo.go b/godb/service/endpoint/mongo.go new file mode 100644 index 0000000..b7e8a25 --- /dev/null +++ b/godb/service/endpoint/mongo.go @@ -0,0 +1,377 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "context" + "log" + "strings" + "sync" + + "github.com/juju/errors" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/stringutil" +) + +type cKey struct { + database string + collection string +} + +type MongoEndpoint struct { + options *options.ClientOptions + client *mongo.Client + lock sync.Mutex + collections map[cKey]*mongo.Collection + collLock sync.RWMutex + + retryLock sync.Mutex +} + +func newMongoEndpoint() *MongoEndpoint { + addrList := strings.Split(global.Cfg().MongodbAddr, ",") + opts := &options.ClientOptions{ + Hosts: addrList, + } + + if global.Cfg().MongodbUsername != "" && global.Cfg().MongodbPassword != "" { + opts.Auth = &options.Credential{ + Username: global.Cfg().MongodbUsername, + Password: global.Cfg().MongodbPassword, + } + } + + r := &MongoEndpoint{} + r.options = opts + r.collections = make(map[cKey]*mongo.Collection) + return r +} + +func (s *MongoEndpoint) Connect() error { + client, err := mongo.Connect(context.Background(), s.options) + if err != nil { + return err + } + + s.client = client + + s.collLock.Lock() + for _, rule := range global.RuleInsList() { + cc := s.client.Database(rule.MongodbDatabase).Collection(rule.MongodbCollection) + s.collections[s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection)] = cc + } + s.collLock.Unlock() + + return nil +} + +func (s *MongoEndpoint) Ping() error { + return s.client.Ping(context.Background(), readpref.Primary()) +} + +func (s *MongoEndpoint) isDuplicateKeyError(stack string) bool { + return strings.Contains(stack, "E11000 duplicate key error") +} + +func (s *MongoEndpoint) collectionKey(database, collection string) cKey { + return cKey{ + database: database, + collection: collection, + } +} + +func (s *MongoEndpoint) collection(key cKey) *mongo.Collection { + s.collLock.RLock() + c, ok := s.collections[key] + s.collLock.RUnlock() + if ok { + return c + } + + s.collLock.Lock() + c = s.client.Database(key.database).Collection(key.collection) + s.collections[key] = c + s.collLock.Unlock() + + return c +} + +func (s *MongoEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + models := make(map[cKey][]mongo.WriteModel, 0) + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoMongoOps(kvm, row.Action, rule) + if err != nil { + return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + for _, resp := range ls { + var model mongo.WriteModel + switch resp.Action { + case canal.InsertAction: + model = mongo.NewInsertOneModel().SetDocument(resp.Table) + case canal.UpdateAction: + model = mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": resp.Id}).SetUpdate(bson.M{"$set": resp.Table}) + case global.UpsertAction: + model = mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": resp.Id}).SetUpsert(true).SetUpdate(bson.M{"$set": resp.Table}) + case canal.DeleteAction: + model = mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": resp.Id}) + } + + key := s.collectionKey(rule.MongodbDatabase, resp.Collection) + array, ok := models[key] + if !ok { + array = make([]mongo.WriteModel, 0) + } + + logs.Infof("action:%s, collection:%s, id:%v, data:%v", resp.Action, resp.Collection, resp.Id, resp.Table) + + array = append(array, model) + models[key] = array + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + kvm["_id"] = id + var model mongo.WriteModel + switch row.Action { + case canal.InsertAction: + model = mongo.NewInsertOneModel().SetDocument(kvm) + case canal.UpdateAction: + model = mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": id}).SetUpdate(bson.M{"$set": kvm}) + case canal.DeleteAction: + model = mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": id}) + } + + ccKey := s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection) + array, ok := models[ccKey] + if !ok { + array = make([]mongo.WriteModel, 0) + } + + logs.Infof("action:%s, collection:%s, id:%v, data:%v", row.Action, rule.MongodbCollection, id, kvm) + + array = append(array, model) + models[ccKey] = array + } + } + + var slowly bool + for key, model := range models { + collection := s.collection(key) + _, err := collection.BulkWrite(context.Background(), model) + if err != nil { + if s.isDuplicateKeyError(err.Error()) { + slowly = true + } else { + return err + } + logs.Error(errors.ErrorStack(err)) + break + } + } + if slowly { + _, err := s.doConsumeSlowly(rows) + if err != nil { + return err + } + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *MongoEndpoint) Stock(rows []*model.RowRequest) int64 { + expect := true + models := make(map[cKey][]mongo.WriteModel, 0) + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoMongoOps(kvm, row.Action, rule) + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + expect = false + break + } + + for _, resp := range ls { + ccKey := s.collectionKey(rule.MongodbDatabase, resp.Collection) + model := mongo.NewInsertOneModel().SetDocument(resp.Table) + array, ok := models[ccKey] + if !ok { + array = make([]mongo.WriteModel, 0) + } + array = append(array, model) + models[ccKey] = array + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + kvm["_id"] = id + + ccKey := s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection) + model := mongo.NewInsertOneModel().SetDocument(kvm) + array, ok := models[ccKey] + if !ok { + array = make([]mongo.WriteModel, 0) + } + array = append(array, model) + models[ccKey] = array + } + } + + if !expect { + return 0 + } + + var slowly bool + var sum int64 + for key, vs := range models { + collection := s.collection(key) + rr, err := collection.BulkWrite(context.Background(), vs) + if err != nil { + if s.isDuplicateKeyError(err.Error()) { + slowly = true + } + logs.Error(errors.ErrorStack(err)) + break + } + sum += rr.InsertedCount + } + + if slowly { + logs.Info("do consume slowly ... ... ") + slowlySum, err := s.doConsumeSlowly(rows) + if err != nil { + logs.Warnf(err.Error()) + } + return slowlySum + } + + return sum +} + +func (s *MongoEndpoint) doConsumeSlowly(rows []*model.RowRequest) (int64, error) { + var sum int64 + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoMongoOps(kvm, row.Action, rule) + if err != nil { + logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + return sum, err + } + for _, resp := range ls { + collection := s.collection(s.collectionKey(rule.MongodbDatabase, resp.Collection)) + switch resp.Action { + case canal.InsertAction: + _, err := collection.InsertOne(context.Background(), resp.Table) + if err != nil { + if s.isDuplicateKeyError(err.Error()) { + logs.Warnf("duplicate key [ %v ]", stringutil.ToJsonString(resp.Table)) + } else { + return sum, err + } + } + case canal.UpdateAction: + _, err := collection.UpdateOne(context.Background(), bson.M{"_id": resp.Id}, bson.M{"$set": resp.Table}) + if err != nil { + return sum, err + } + case canal.DeleteAction: + _, err := collection.DeleteOne(context.Background(), bson.M{"_id": resp.Id}) + if err != nil { + return sum, err + } + } + logs.Infof("action:%s, collection:%s, id:%v, data:%v", + row.Action, collection.Name(), resp.Id, resp.Table) + } + } else { + kvm := rowMap(row, rule, false) + id := primaryKey(row, rule) + kvm["_id"] = id + + collection := s.collection(s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection)) + + switch row.Action { + case canal.InsertAction: + _, err := collection.InsertOne(context.Background(), kvm) + if err != nil { + if s.isDuplicateKeyError(err.Error()) { + logs.Warnf("duplicate key [ %v ]", stringutil.ToJsonString(kvm)) + } else { + return sum, err + } + } + case canal.UpdateAction: + _, err := collection.UpdateOne(context.Background(), bson.M{"_id": id}, bson.M{"$set": kvm}) + if err != nil { + return sum, err + } + case canal.DeleteAction: + _, err := collection.DeleteOne(context.Background(), bson.M{"_id": id}) + if err != nil { + return sum, err + } + } + + logs.Infof("action:%s, collection:%s, id:%v, data:%v", row.Action, collection.Name(), id, kvm) + } + sum++ + } + return sum, nil +} + +func (s *MongoEndpoint) Close() { + if s.client != nil { + s.client.Disconnect(context.Background()) + } +} diff --git a/godb/service/endpoint/mongo_test.go b/godb/service/endpoint/mongo_test.go new file mode 100644 index 0000000..eee7b56 --- /dev/null +++ b/godb/service/endpoint/mongo_test.go @@ -0,0 +1,54 @@ +package endpoint + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + "testing" +) + +func TestMongoPing(t *testing.T) { + opts := &options.ClientOptions{ + Hosts: []string{"127.0.0.1:27018"}, + } + + // 连接数据库 + client, err := mongo.Connect(context.Background(), opts) + if err != nil { + t.Error(err.Error()) + } + + err = client.Ping(context.Background(), readpref.Primary()) + if err != nil { + t.Error(err.Error()) + } +} + +func TestCheckData(t *testing.T) { + opts := &options.ClientOptions{ + Hosts: []string{"127.0.0.1:27018"}, + } + opts.Auth = &options.Credential{ + Username: "test", + Password: "test", + } + + // 连接数据库 + client, err := mongo.Connect(context.Background(), opts) + if err != nil { + t.Error(err.Error()) + } + + db := client.Database("test") + cc := db.Collection("ss") + + for i := 0; i < 22032; i++ { + r := cc.FindOne(context.Background(), bson.M{"_id": i}) + if r.Err() != nil { + t.Error(r.Err()) + } + } + +} diff --git a/godb/service/endpoint/rabbit.go b/godb/service/endpoint/rabbit.go new file mode 100644 index 0000000..e4a6127 --- /dev/null +++ b/godb/service/endpoint/rabbit.go @@ -0,0 +1,232 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "github.com/siddontang/go-mysql/canal" + "log" + "strconv" + + "github.com/juju/errors" + "github.com/siddontang/go-mysql/mysql" + "github.com/streadway/amqp" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/nets" +) + +type RabbitEndpoint struct { + rabCon *amqp.Connection + rabChl *amqp.Channel + queues map[string]bool + serverUrl string +} + +func newRabbitEndpoint() *RabbitEndpoint { + r := &RabbitEndpoint{} + r.queues = make(map[string]bool) + return r +} + +func (s *RabbitEndpoint) Connect() error { + if s.rabChl != nil { + s.rabChl.Close() + s.rabChl = nil + } + if s.rabCon != nil { + s.rabCon.Close() + s.rabCon = nil + } + + con, err := amqp.Dial(global.Cfg().RabbitmqAddr) + if err != nil { + return err + } + + uri, _ := amqp.ParseURI(global.Cfg().RabbitmqAddr) + s.serverUrl = uri.Host + ":" + strconv.Itoa(uri.Port) + + var chl *amqp.Channel + chl, err = con.Channel() + if err != nil { + return err + } + + s.rabCon = con + s.rabChl = chl + + if len(s.queues) == 0 { + for _, rule := range global.RuleInsList() { + _, err := s.rabChl.QueueDeclare( + rule.RabbitmqQueue, false, false, false, false, nil, + ) + if err != nil { + return err + } + s.queues[rule.RabbitmqQueue] = true + } + } + + return nil +} + +func (s *RabbitEndpoint) Ping() error { + _, err := nets.IsActiveTCPAddr(s.serverUrl) + return err +} + +func (s *RabbitEndpoint) mergeQueue(name string) { + _, ok := s.queues[name] + if ok { + return + } + + s.rabChl.QueueDeclare(name, false, false, false, false, nil) + s.queues[name] = true +} + +func (s *RabbitEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + err := s.doLuaConsume(row, rule) + if err != nil { + return err + } + } else { + err := s.doRuleConsume(row, rule) + if err != nil { + return err + } + } + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *RabbitEndpoint) Stock(rows []*model.RowRequest) int64 { + var sum int64 + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + err := s.doLuaConsume(row, rule) + if err != nil { + logs.Errorf(errors.ErrorStack(err)) + break + } + } else { + err := s.doRuleConsume(row, rule) + if err != nil { + logs.Errorf(errors.ErrorStack(err)) + break + } + } + sum++ + } + + return sum +} + +func (s *RabbitEndpoint) doLuaConsume(req *model.RowRequest, rule *global.Rule) error { + var err error + var ls []*model.MQRespond + kvm := rowMap(req, rule, true) + if req.Action == canal.UpdateAction { + previous := oldRowMap(req, rule, true) + ls, err = luaengine.DoMQOps(kvm, previous, req.Action, rule) + } else { + ls, err = luaengine.DoMQOps(kvm, nil, req.Action, rule) + } + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("lua 脚本执行失败 : %s ", err) + } + + for _, resp := range ls { + s.mergeQueue(resp.Topic) + err := s.rabChl.Publish("", resp.Topic, false, false, + amqp.Publishing{ + ContentType: "text/plain", + Body: resp.ByteArray, + }) + + logs.Infof("topic: %s, message: %s", resp.Topic, string(resp.ByteArray)) + if err != nil { + return err + } + } + + return nil +} + +func (s *RabbitEndpoint) doRuleConsume(req *model.RowRequest, rule *global.Rule) error { + kvm := rowMap(req, rule, false) + + resp := new(model.MQRespond) + resp.Action = req.Action + resp.Timestamp = req.Timestamp + if rule.ValueEncoder == global.ValEncoderJson { + resp.Date = kvm + } else { + resp.Date = encodeValue(rule, kvm) + } + + if rule.ReserveRawData && canal.UpdateAction == req.Action { + resp.Raw = oldRowMap(req, rule, false) + } + + body, err := json.Marshal(resp) + if err != nil { + return err + } + err = s.rabChl.Publish("", rule.RabbitmqQueue, false, false, + amqp.Publishing{ + ContentType: "text/plain", + Body: body, + }) + + logs.Infof("topic: %s, message: %s", rule.RabbitmqQueue, string(body)) + + return err +} + +func (s *RabbitEndpoint) Close() { + if s.rabChl != nil { + s.rabChl.Close() + } + if s.rabCon != nil { + s.rabCon.Close() + } +} diff --git a/godb/service/endpoint/redis.go b/godb/service/endpoint/redis.go new file mode 100644 index 0000000..5985644 --- /dev/null +++ b/godb/service/endpoint/redis.go @@ -0,0 +1,333 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "bytes" + "log" + "strings" + "sync" + + "github.com/go-redis/redis" + "github.com/pingcap/errors" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logs" + "go-mysql-transfer/util/stringutil" +) + +type RedisEndpoint struct { + isCluster bool + client *redis.Client + cluster *redis.ClusterClient + retryLock sync.Mutex +} + +func newRedisEndpoint() *RedisEndpoint { + cfg := global.Cfg() + r := &RedisEndpoint{} + + list := strings.Split(cfg.RedisAddr, ",") + if len(list) == 1 { + r.client = redis.NewClient(&redis.Options{ + Addr: cfg.RedisAddr, + Password: cfg.RedisPass, + DB: cfg.RedisDatabase, + }) + } else { + if cfg.RedisGroupType == global.RedisGroupTypeSentinel { + r.client = redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: cfg.RedisMasterName, + SentinelAddrs: list, + Password: cfg.RedisPass, + DB: cfg.RedisDatabase, + }) + } + if cfg.RedisGroupType == global.RedisGroupTypeCluster { + r.isCluster = true + r.cluster = redis.NewClusterClient(&redis.ClusterOptions{ + Addrs: list, + Password: cfg.RedisPass, + }) + } + } + + return r +} + +func (s *RedisEndpoint) Connect() error { + return s.Ping() +} + +func (s *RedisEndpoint) Ping() error { + var err error + if s.isCluster { + _, err = s.cluster.Ping().Result() + } else { + _, err = s.client.Ping().Result() + } + return err +} + +func (s *RedisEndpoint) pipe() redis.Pipeliner { + var pipe redis.Pipeliner + if s.isCluster { + pipe = s.cluster.Pipeline() + } else { + pipe = s.client.Pipeline() + } + return pipe +} + +func (s *RedisEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + pipe := s.pipe() + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + var err error + var ls []*model.RedisRespond + kvm := rowMap(row, rule, true) + if row.Action == canal.UpdateAction { + previous := oldRowMap(row, rule, true) + ls, err = luaengine.DoRedisOps(kvm, previous, row.Action, rule) + } else { + ls, err = luaengine.DoRedisOps(kvm, nil, row.Action, rule) + } + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("Lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + for _, resp := range ls { + s.preparePipe(resp, pipe) + logs.Infof("action: %s, structure: %s ,key: %s ,field: %s, value: %v", resp.Action, resp.Structure, resp.Key, resp.Field, resp.Val) + } + kvm = nil + } else { + resp := s.ruleRespond(row, rule) + s.preparePipe(resp, pipe) + logs.Infof("action: %s, structure: %s ,key: %s ,field: %s, value: %v", resp.Action, resp.Structure, resp.Key, resp.Field, resp.Val) + } + } + + _, err := pipe.Exec() + if err != nil { + return err + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *RedisEndpoint) Stock(rows []*model.RowRequest) int64 { + pipe := s.pipe() + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + kvm := rowMap(row, rule, true) + ls, err := luaengine.DoRedisOps(kvm, nil, row.Action, rule) + if err != nil { + logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + break + } + for _, resp := range ls { + s.preparePipe(resp, pipe) + } + } else { + resp := s.ruleRespond(row, rule) + resp.Action = row.Action + resp.Structure = rule.RedisStructure + s.preparePipe(resp, pipe) + } + } + + var counter int64 + res, err := pipe.Exec() + if err != nil { + logs.Error(err.Error()) + } + + for _, re := range res { + if re.Err() == nil { + counter++ + } + } + + return counter +} + +func (s *RedisEndpoint) ruleRespond(row *model.RowRequest, rule *global.Rule) *model.RedisRespond { + resp := new(model.RedisRespond) + resp.Action = row.Action + resp.Structure = rule.RedisStructure + + kvm := rowMap(row, rule, false) + resp.Key = s.encodeKey(row, rule) + if resp.Structure == global.RedisStructureHash { + resp.Field = s.encodeHashField(row, rule) + } + if resp.Structure == global.RedisStructureSortedSet { + resp.Score = s.encodeSortedSetScoreField(row, rule) + } + + if resp.Action == canal.InsertAction { + resp.Val = encodeValue(rule, kvm) + } else if resp.Action == canal.UpdateAction { + if rule.RedisStructure == global.RedisStructureList || + rule.RedisStructure == global.RedisStructureSet || + rule.RedisStructure == global.RedisStructureSortedSet { + oldKvm := oldRowMap(row, rule, false) + resp.OldVal = encodeValue(rule, oldKvm) + } + resp.Val = encodeValue(rule, kvm) + } else { + if rule.RedisStructure == global.RedisStructureList || + rule.RedisStructure == global.RedisStructureSet || + rule.RedisStructure == global.RedisStructureSortedSet { + resp.Val = encodeValue(rule, kvm) + } + } + + return resp +} + +func (s *RedisEndpoint) preparePipe(resp *model.RedisRespond, pipe redis.Cmdable) { + switch resp.Structure { + case global.RedisStructureString: + if resp.Action == canal.DeleteAction { + pipe.Del(resp.Key) + } else { + pipe.Set(resp.Key, resp.Val, 0) + } + case global.RedisStructureHash: + if resp.Action == canal.DeleteAction { + pipe.HDel(resp.Key, resp.Field) + } else { + pipe.HSet(resp.Key, resp.Field, resp.Val) + } + case global.RedisStructureList: + if resp.Action == canal.DeleteAction { + pipe.LRem(resp.Key, 0, resp.Val) + } else if resp.Action == canal.UpdateAction { + pipe.LRem(resp.Key, 0, resp.OldVal) + pipe.RPush(resp.Key, resp.Val) + } else { + pipe.RPush(resp.Key, resp.Val) + } + case global.RedisStructureSet: + if resp.Action == canal.DeleteAction { + pipe.SRem(resp.Key, resp.Val) + } else if resp.Action == canal.UpdateAction { + pipe.SRem(resp.Key, 0, resp.OldVal) + pipe.SAdd(resp.Key, resp.Val) + } else { + pipe.SAdd(resp.Key, resp.Val) + } + case global.RedisStructureSortedSet: + if resp.Action == canal.DeleteAction { + pipe.ZRem(resp.Key, resp.Val) + } else if resp.Action == canal.UpdateAction { + pipe.ZRem(resp.Key, 0, resp.OldVal) + val := redis.Z{Score: resp.Score, Member: resp.Val} + pipe.ZAdd(resp.Key, val) + } else { + val := redis.Z{Score: resp.Score, Member: resp.Val} + pipe.ZAdd(resp.Key, val) + } + } +} + +func (s *RedisEndpoint) encodeKey(req *model.RowRequest, rule *global.Rule) string { + if rule.RedisKeyValue != "" { + return rule.RedisKeyValue + } + + if rule.RedisKeyFormatter != "" { + kv := rowMap(req, rule, true) + var tmplBytes bytes.Buffer + err := rule.RedisKeyTmpl.Execute(&tmplBytes, kv) + if err != nil { + return "" + } + return tmplBytes.String() + } + + var key string + if rule.RedisKeyColumnIndex < 0 { + for _, v := range rule.RedisKeyColumnIndexs { + key += stringutil.ToString(req.Row[v]) + } + } else { + key = stringutil.ToString(req.Row[rule.RedisKeyColumnIndex]) + } + if rule.RedisKeyPrefix != "" { + key = rule.RedisKeyPrefix + key + } + + return key +} + +func (s *RedisEndpoint) encodeHashField(req *model.RowRequest, rule *global.Rule) string { + var field string + + if rule.RedisHashFieldColumnIndex < 0 { + for _, v := range rule.RedisHashFieldColumnIndexs { + field += stringutil.ToString(req.Row[v]) + } + } else { + field = stringutil.ToString(req.Row[rule.RedisHashFieldColumnIndex]) + } + + if rule.RedisHashFieldPrefix != "" { + field = rule.RedisHashFieldPrefix + field + } + + return field +} + +func (s *RedisEndpoint) encodeSortedSetScoreField(req *model.RowRequest, rule *global.Rule) float64 { + obj := req.Row[rule.RedisHashFieldColumnIndex] + if obj == nil { + return 0 + } + + str := stringutil.ToString(obj) + return stringutil.ToFloat64Safe(str) +} + +func (s *RedisEndpoint) Close() { + if s.client != nil { + s.client.Close() + } +} diff --git a/godb/service/endpoint/rocket.go b/godb/service/endpoint/rocket.go new file mode 100644 index 0000000..a131c01 --- /dev/null +++ b/godb/service/endpoint/rocket.go @@ -0,0 +1,266 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "context" + "github.com/siddontang/go-mysql/canal" + "log" + "strings" + "sync" + + "github.com/apache/rocketmq-client-go/v2" + "github.com/apache/rocketmq-client-go/v2/primitive" + "github.com/apache/rocketmq-client-go/v2/producer" + "github.com/apache/rocketmq-client-go/v2/rlog" + "github.com/juju/errors" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logagent" + "go-mysql-transfer/util/logs" +) + +const _rocketRetry = 2 + +type RocketEndpoint struct { + client rocketmq.Producer + retryLock sync.Mutex +} + +func newRocketEndpoint() *RocketEndpoint { + rlog.SetLogger(logagent.NewRocketmqLoggerAgent()) + cfg := global.Cfg() + + options := make([]producer.Option, 0) + serverList := strings.Split(cfg.RocketmqNameServers, ",") + options = append(options, producer.WithNameServer(serverList)) + options = append(options, producer.WithRetry(_rocketRetry)) + if cfg.RocketmqGroupName != "" { + options = append(options, producer.WithGroupName(cfg.RocketmqGroupName)) + } + if cfg.RocketmqInstanceName != "" { + options = append(options, producer.WithInstanceName(cfg.RocketmqInstanceName)) + } + if cfg.RocketmqAccessKey != "" && cfg.RocketmqSecretKey != "" { + options = append(options, producer.WithCredentials(primitive.Credentials{ + AccessKey: cfg.RocketmqAccessKey, + SecretKey: cfg.RocketmqSecretKey, + })) + } + + producer, _ := rocketmq.NewProducer(options...) + r := &RocketEndpoint{} + r.client = producer + return r +} + +func (s *RocketEndpoint) Connect() error { + return s.client.Start() +} + +func (s *RocketEndpoint) Ping() error { + ping := &primitive.Message{ + Topic: "BenchmarkTest", + Body: []byte("ping"), + } + _, err := s.client.SendSync(context.Background(), ping) + return err +} + +func (s *RocketEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + var ms []*primitive.Message + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + + if rule.LuaEnable() { + ls, err := s.buildMessages(row, rule) + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + ms = append(ms, ls...) + } else { + m, err := s.buildMessage(row, rule) + if err != nil { + return errors.New(errors.ErrorStack(err)) + } + ms = append(ms, m) + } + } + + if len(ms) ==0{ + return nil + } + + var wg sync.WaitGroup + wg.Add(1) + var callbackErr error + err := s.client.SendAsync(context.Background(), + func(ctx context.Context, result *primitive.SendResult, e error) { + if e != nil { + callbackErr = e + } + wg.Done() + }, ms...) + + if err != nil { + return err + } + wg.Wait() + + if callbackErr != nil { + return err + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *RocketEndpoint) Stock(rows []*model.RowRequest) int64 { + expect := true + var ms []*primitive.Message + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + if rule.LuaEnable() { + ls, err := s.buildMessages(row, rule) + if err != nil { + logs.Errorf(errors.ErrorStack(err)) + expect = false + break + } + ms = append(ms, ls...) + } else { + m, err := s.buildMessage(row, rule) + if err != nil { + logs.Errorf(errors.ErrorStack(err)) + expect = false + break + } + ms = append(ms, m) + } + } + + if !expect { + return 0 + } + + if len(ms) == 0 { + return 0 + } + + var wg sync.WaitGroup + wg.Add(1) + err := s.client.SendAsync(context.Background(), + func(ctx context.Context, result *primitive.SendResult, e error) { + if e != nil { + logs.Error(errors.ErrorStack(e)) + expect = false + } + wg.Done() + }, ms...) + + if err != nil { + logs.Error(errors.ErrorStack(err)) + return 0 + } + + wg.Wait() + + if expect { + return int64(len(ms)) + } + return 0 +} + +func (s *RocketEndpoint) buildMessages(req *model.RowRequest, rule *global.Rule) ([]*primitive.Message, error) { + var err error + var ls []*model.MQRespond + kvm := rowMap(req, rule, true) + if req.Action == canal.UpdateAction { + previous := oldRowMap(req, rule, true) + ls, err = luaengine.DoMQOps(kvm, previous, req.Action, rule) + } else { + ls, err = luaengine.DoMQOps(kvm, nil, req.Action, rule) + } + if err != nil { + return nil, errors.Errorf("lua 脚本执行失败 : %s ", err) + } + + var ms []*primitive.Message + for _, resp := range ls { + m := &primitive.Message{ + Topic: resp.Topic, + Body: resp.ByteArray, + } + logs.Infof("topic: %s, message: %s", m.Topic, string(m.Body)) + ms = append(ms, m) + } + + return ms, nil +} + +func (s *RocketEndpoint) buildMessage(req *model.RowRequest, rule *global.Rule) (*primitive.Message, error) { + kvm := rowMap(req, rule, false) + resp := new(model.MQRespond) + resp.Action = req.Action + resp.Timestamp = req.Timestamp + if rule.ValueEncoder == global.ValEncoderJson { + resp.Date = kvm + } else { + resp.Date = encodeValue(rule, kvm) + } + + if rule.ReserveRawData && canal.UpdateAction == req.Action { + resp.Raw = oldRowMap(req, rule, false) + } + + body, err := json.Marshal(resp) + if err != nil { + return nil, err + } + + m := &primitive.Message{ + Topic: rule.RocketmqTopic, + Body: body, + } + + logs.Infof("topic: %s, message: %s", m.Topic, string(m.Body)) + + return m, nil +} + +func (s *RocketEndpoint) Close() { + if s.client != nil { + s.client.Shutdown() + } +} diff --git a/godb/service/endpoint/script.go b/godb/service/endpoint/script.go new file mode 100644 index 0000000..eb92c11 --- /dev/null +++ b/godb/service/endpoint/script.go @@ -0,0 +1,93 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package endpoint + +import ( + "log" + + "github.com/pingcap/errors" + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/model" + "go-mysql-transfer/service/luaengine" + "go-mysql-transfer/util/logs" +) + +type ScriptEndpoint struct { +} + +func newScriptEndpoint() *ScriptEndpoint { + return &ScriptEndpoint{} +} + +func (s *ScriptEndpoint) Connect() error { + return nil +} + +func (s *ScriptEndpoint) Ping() error { + return nil +} + +func (s *ScriptEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + metrics.UpdateActionNum(row.Action, row.RuleKey) + kvm := rowMap(row, rule, true) + err := luaengine.DoScript(kvm, row.Action, rule) + if err != nil { + log.Println("Lua 脚本执行失败!!! ,详情请参见日志") + return errors.Errorf("Lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + } + kvm = nil + } + + logs.Infof("处理完成 %d 条数据", len(rows)) + return nil +} + +func (s *ScriptEndpoint) Stock(rows []*model.RowRequest) int64 { + var counter int64 + for _, row := range rows { + rule, _ := global.RuleIns(row.RuleKey) + if rule.TableColumnSize != len(row.Row) { + logs.Warnf("%s schema mismatching", row.RuleKey) + continue + } + + kvm := rowMap(row, rule, true) + err := luaengine.DoScript(kvm, row.Action, rule) + if err != nil { + logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) + break + } + counter++ + } + + return counter +} + +func (s *ScriptEndpoint) Close() { + +} diff --git a/godb/service/handler.go b/godb/service/handler.go new file mode 100644 index 0000000..452ab5a --- /dev/null +++ b/godb/service/handler.go @@ -0,0 +1,199 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package service + +import ( + "go-mysql-transfer/metrics" + "log" + "time" + + "github.com/juju/errors" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + "github.com/siddontang/go-mysql/replication" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" + "go-mysql-transfer/util/logs" +) + +type handler struct { + queue chan interface{} + stop chan struct{} +} + +func newHandler() *handler { + return &handler{ + queue: make(chan interface{}, 4096), + stop: make(chan struct{}, 1), + } +} + +func (s *handler) OnRotate(e *replication.RotateEvent) error { + s.queue <- model.PosRequest{ + Name: string(e.NextLogName), + Pos: uint32(e.Position), + Force: true, + } + return nil +} + +func (s *handler) OnTableChanged(schema, table string) error { + err := _transferService.updateRule(schema, table) + if err != nil { + return errors.Trace(err) + } + return nil +} + +func (s *handler) OnDDL(nextPos mysql.Position, _ *replication.QueryEvent) error { + s.queue <- model.PosRequest{ + Name: nextPos.Name, + Pos: nextPos.Pos, + Force: true, + } + return nil +} + +func (s *handler) OnXID(nextPos mysql.Position) error { + s.queue <- model.PosRequest{ + Name: nextPos.Name, + Pos: nextPos.Pos, + Force: false, + } + return nil +} + +func (s *handler) OnRow(e *canal.RowsEvent) error { + ruleKey := global.RuleKey(e.Table.Schema, e.Table.Name) + if !global.RuleInsExist(ruleKey) { + return nil + } + + var requests []*model.RowRequest + if e.Action != canal.UpdateAction { + // 定长分配 + requests = make([]*model.RowRequest, 0, len(e.Rows)) + } + + if e.Action == canal.UpdateAction { + for i := 0; i < len(e.Rows); i++ { + if (i+1)%2 == 0 { + v := new(model.RowRequest) + v.RuleKey = ruleKey + v.Action = e.Action + v.Timestamp = e.Header.Timestamp + if global.Cfg().IsReserveRawData() { + v.Old = e.Rows[i-1] + } + v.Row = e.Rows[i] + requests = append(requests, v) + } + } + } else { + for _, row := range e.Rows { + v := new(model.RowRequest) + v.RuleKey = ruleKey + v.Action = e.Action + v.Timestamp = e.Header.Timestamp + v.Row = row + requests = append(requests, v) + } + } + s.queue <- requests + + return nil +} + +func (s *handler) OnGTID(gtid mysql.GTIDSet) error { + return nil +} + +func (s *handler) OnPosSynced(pos mysql.Position, set mysql.GTIDSet, force bool) error { + return nil +} + +func (s *handler) String() string { + return "TransferHandler" +} + +func (s *handler) startListener() { + go func() { + interval := time.Duration(global.Cfg().FlushBulkInterval) + bulkSize := global.Cfg().BulkSize + ticker := time.NewTicker(time.Millisecond * interval) + defer ticker.Stop() + + lastSavedTime := time.Now() + requests := make([]*model.RowRequest, 0, bulkSize) + var current mysql.Position + from, _ := _transferService.positionDao.Get() + for { + needFlush := false + needSavePos := false + select { + case v := <-s.queue: + switch v := v.(type) { + case model.PosRequest: + now := time.Now() + if v.Force || now.Sub(lastSavedTime) > 3*time.Second { + lastSavedTime = now + needFlush = true + needSavePos = true + current = mysql.Position{ + Name: v.Name, + Pos: v.Pos, + } + } + case []*model.RowRequest: + requests = append(requests, v...) + needFlush = int64(len(requests)) >= global.Cfg().BulkSize + } + case <-ticker.C: + needFlush = true + case <-s.stop: + return + } + + if needFlush && len(requests) > 0 && _transferService.endpointEnable.Load() { + err := _transferService.endpoint.Consume(from, requests) + if err != nil { + _transferService.endpointEnable.Store(false) + metrics.SetDestState(metrics.DestStateFail) + logs.Error(err.Error()) + go _transferService.stopDump() + } + requests = requests[0:0] + } + if needSavePos && _transferService.endpointEnable.Load() { + logs.Infof("save position %s %d", current.Name, current.Pos) + if err := _transferService.positionDao.Save(current); err != nil { + logs.Errorf("save sync position %s err %v, close sync", current, err) + _transferService.Close() + return + } + from = current + } + } + }() +} + +func (s *handler) stopListener() { + log.Println("transfer stop") + s.stop <- struct{}{} +} diff --git a/godb/service/luaengine/actuator.go b/godb/service/luaengine/actuator.go new file mode 100644 index 0000000..0e2deb3 --- /dev/null +++ b/godb/service/luaengine/actuator.go @@ -0,0 +1,294 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "encoding/json" + "sync" + + luaJson "github.com/layeh/gopher-json" + "github.com/siddontang/go-mysql/canal" + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/util/byteutil" + "go-mysql-transfer/util/httpclient" + "go-mysql-transfer/util/stringutil" +) + +const ( + _globalRET = "___RET___" + _globalROW = "___ROW___" + _globalACT = "___ACT___" +) + +var ( + _pool *luaStatePool + _ds *canal.Canal + + _httpClient *httpclient.HttpClient +) + +type luaStatePool struct { + lock sync.Mutex + saved []*lua.LState +} + +func InitActuator(ds *canal.Canal) { + _ds = ds + _pool = &luaStatePool{ + saved: make([]*lua.LState, 0, 3), + } +} + +func (p *luaStatePool) Get() *lua.LState { + p.lock.Lock() + defer p.lock.Unlock() + + n := len(p.saved) + if n == 0 { + return p.New() + } + x := p.saved[n-1] + p.saved = p.saved[0 : n-1] + return x +} + +func (p *luaStatePool) New() *lua.LState { + _httpClient = httpclient.NewClient() + + L := lua.NewState() + + luaJson.Preload(L) + + L.PreloadModule("scriptOps", scriptModule) + L.PreloadModule("dbOps", dbModule) + L.PreloadModule("httpOps", httpModule) + + L.PreloadModule("redisOps", redisModule) + L.PreloadModule("mqOps", mqModule) + L.PreloadModule("mongodbOps", mongoModule) + L.PreloadModule("esOps", esModule) + + return L +} + +func (p *luaStatePool) Put(L *lua.LState) { + p.lock.Lock() + defer p.lock.Unlock() + + p.saved = append(p.saved, L) +} + +func (p *luaStatePool) Shutdown() { + for _, L := range p.saved { + L.Close() + } +} + +func rawRow(L *lua.LState) int { + row := L.GetGlobal(_globalROW) + L.Push(row) + return 1 +} + +func rawAction(L *lua.LState) int { + act := L.GetGlobal(_globalACT) + L.Push(act) + return 1 +} + +func paddingTable(l *lua.LState, table *lua.LTable, kv map[string]interface{}) { + for k, v := range kv { + switch v.(type) { + case float64: + ft := v.(float64) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case float32: + ft := v.(float32) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case int: + ft := v.(int) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case uint: + ft := v.(uint) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case int8: + ft := v.(int8) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case uint8: + ft := v.(uint8) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case int16: + ft := v.(int16) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case uint16: + ft := v.(uint16) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case int32: + ft := v.(int32) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case uint32: + ft := v.(uint32) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case int64: + ft := v.(int64) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case uint64: + ft := v.(uint64) + l.SetTable(table, lua.LString(k), lua.LNumber(ft)) + case string: + ft := v.(string) + l.SetTable(table, lua.LString(k), lua.LString(ft)) + case []byte: + ft := string(v.([]byte)) + l.SetTable(table, lua.LString(k), lua.LString(ft)) + case nil: + l.SetTable(table, lua.LString(k), lua.LNil) + default: + jsonValue, _ := json.Marshal(v) + l.SetTable(table, lua.LString(k), lua.LString(jsonValue)) + } + } +} + +func lvToString(lv lua.LValue) string { + if lua.LVCanConvToString(lv) { + return lua.LVAsString(lv) + } + + return lv.String() +} + +func lvToByteArray(lv lua.LValue) []byte { + switch lv.Type() { + case lua.LTNil: + return nil + case lua.LTBool: + return byteutil.JsonBytes(lua.LVAsBool(lv)) + case lua.LTNumber: + return []byte(lv.String()) + case lua.LTString: + return []byte(lua.LVAsString(lv)) + case lua.LTTable: + ret := lvToInterface(lv, false) + return byteutil.JsonBytes(ret) + default: + return byteutil.JsonBytes(lv) + } +} + +func lvToInterface(lv lua.LValue, tableToJson bool) interface{} { + switch lv.Type() { + case lua.LTNil: + return nil + case lua.LTBool: + return lua.LVAsBool(lv) + case lua.LTNumber: + return float64(lua.LVAsNumber(lv)) + case lua.LTString: + return lua.LVAsString(lv) + case lua.LTTable: + t, _ := lv.(*lua.LTable) + len := t.MaxN() + if len == 0 { // table + ret := make(map[string]interface{}) + t.ForEach(func(key, value lua.LValue) { + ret[lvToString(key)] = lvToInterface(value, false) + }) + if tableToJson { + return stringutil.ToJsonString(ret) + } + return ret + } else { // array + ret := make([]interface{}, 0, len) + for i := 1; i <= len; i++ { + ret = append(ret, lvToInterface(t.RawGetInt(i), false)) + } + if tableToJson { + return stringutil.ToJsonString(ret) + } + return ret + } + default: + return lv + } +} + +func lvToMap(lv lua.LValue) (map[string]interface{}, bool) { + switch lv.Type() { + case lua.LTTable: + t := lvToInterface(lv, false) + ret := t.(map[string]interface{}) + return ret, true + default: + return nil, false + } +} + +func interfaceToLv(v interface{}) lua.LValue { + switch v.(type) { + case float64: + ft := v.(float64) + return lua.LNumber(ft) + case float32: + ft := v.(float32) + return lua.LNumber(ft) + case int: + ft := v.(int) + return lua.LNumber(ft) + case uint: + ft := v.(uint) + return lua.LNumber(ft) + case int8: + ft := v.(int8) + return lua.LNumber(ft) + case uint8: + ft := v.(uint8) + return lua.LNumber(ft) + case int16: + ft := v.(int16) + return lua.LNumber(ft) + case uint16: + ft := v.(uint16) + return lua.LNumber(ft) + case int32: + ft := v.(int32) + return lua.LNumber(ft) + case uint32: + ft := v.(uint32) + return lua.LNumber(ft) + case int64: + ft := v.(int64) + return lua.LNumber(ft) + case uint64: + ft := v.(uint64) + return lua.LNumber(ft) + case string: + ft := v.(string) + return lua.LString(ft) + case []byte: + ft := string(v.([]byte)) + return lua.LString(ft) + case nil: + return lua.LNil + default: + jsonValue, _ := json.Marshal(v) + return lua.LString(jsonValue) + } + +} diff --git a/godb/service/luaengine/db_actuator.go b/godb/service/luaengine/db_actuator.go new file mode 100644 index 0000000..ce5738f --- /dev/null +++ b/godb/service/luaengine/db_actuator.go @@ -0,0 +1,124 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "github.com/layeh/gopher-json" + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/util/logs" +) + +func dbModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _dbModuleApi) + L.Push(t) + return 1 +} + +var _dbModuleApi = map[string]lua.LGFunction{ + "selectOne": selectOne, + "select": selectList, +} + +func selectOne(L *lua.LState) int { + sql := L.CheckString(1) + + logs.Infof("lua db module execute sql: %s", sql) + + rs, err := _ds.Execute(sql) + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + rowNumber := rs.RowNumber() + + table := L.NewTable() + if rowNumber > 1 { + logs.Error("return more than 1 row") + L.Push(lua.LNil) + L.Push(lua.LString("return more than 1 row")) + return 2 + } + + if rowNumber == 1 { + for field, index := range rs.FieldNames { + v, err := rs.GetValue(0, index) + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + val := interfaceToLv(v) + L.SetTable(table, lua.LString(field), val) + } + } + + if data, err := json.Encode(table); err == nil { + logs.Infof("lua db module result: %s", string(data)) + } + + L.Push(table) + return 1 +} + +func selectList(L *lua.LState) int { + sql := L.CheckString(1) + + logs.Infof("lua db module execute sql: %s", sql) + + rs, err := _ds.Execute(sql) + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + rowNumber := rs.RowNumber() + + ret := L.NewTable() + if rowNumber > 0 { + for i := 0; i < rowNumber; i++ { + table := L.NewTable() + for field, index := range rs.FieldNames { + v, err := rs.GetValue(i, index) + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + val := interfaceToLv(v) + L.SetTable(table, lua.LString(field), val) + } + L.SetTable(ret, lua.LNumber(i+1), table) + } + } + + if data, err := json.Encode(ret); err == nil { + logs.Infof("lua db module result: %s", string(data)) + } else { + logs.Error(err.Error()) + } + + L.Push(ret) + return 1 +} diff --git a/godb/service/luaengine/es_actuator.go b/godb/service/luaengine/es_actuator.go new file mode 100644 index 0000000..2061888 --- /dev/null +++ b/godb/service/luaengine/es_actuator.go @@ -0,0 +1,134 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "github.com/siddontang/go-mysql/canal" + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" + "go-mysql-transfer/util/stringutil" +) + +func esModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _esModuleApi) + L.Push(t) + return 1 +} + +var _esModuleApi = map[string]lua.LGFunction{ + "rawRow": rawRow, + "rawAction": rawAction, + + "INSERT": esInsert, + "UPDATE": esUpdate, + "DELETE": esDelete, +} + +func esInsert(L *lua.LState) int { + index := L.CheckAny(1) + id := L.CheckAny(2) + body := L.CheckAny(3) + + data := L.NewTable() + L.SetTable(data, lua.LString("index"), index) + L.SetTable(data, lua.LString("action"), lua.LString(canal.InsertAction)) + L.SetTable(data, lua.LString("id"), id) + L.SetTable(data, lua.LString("body"), body) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func esUpdate(L *lua.LState) int { + index := L.CheckAny(1) + id := L.CheckAny(2) + body := L.CheckAny(3) + + data := L.NewTable() + L.SetTable(data, lua.LString("index"), index) + L.SetTable(data, lua.LString("action"), lua.LString(canal.UpdateAction)) + L.SetTable(data, lua.LString("id"), id) + L.SetTable(data, lua.LString("body"), body) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func esDelete(L *lua.LState) int { + index := L.CheckAny(1) + id := L.CheckAny(2) + + data := L.NewTable() + L.SetTable(data, lua.LString("index"), index) + L.SetTable(data, lua.LString("action"), lua.LString(canal.DeleteAction)) + L.SetTable(data, lua.LString("id"), id) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func DoESOps(input map[string]interface{}, action string, rule *global.Rule) ([]*model.ESRespond, error) { + L := _pool.Get() + defer _pool.Put(L) + + row := L.NewTable() + paddingTable(L, row, input) + ret := L.NewTable() + L.SetGlobal(_globalRET, ret) + L.SetGlobal(_globalROW, row) + L.SetGlobal(_globalACT, lua.LString(action)) + + funcFromProto := L.NewFunctionFromProto(rule.LuaProto) + L.Push(funcFromProto) + err := L.PCall(0, lua.MultRet, nil) + if err != nil { + return nil, err + } + + responds := make([]*model.ESRespond, 0, ret.Len()) + ret.ForEach(func(k lua.LValue, v lua.LValue) { + resp := new(model.ESRespond) + resp.Index = lvToString(L.GetTable(v, lua.LString("index"))) + resp.Id = lvToString(L.GetTable(v, lua.LString("id"))) + resp.Action = lvToString(L.GetTable(v, lua.LString("action"))) + + var data string + body := L.GetTable(v, lua.LString("body")) + switch body.Type() { + case lua.LTNumber: + data = lua.LVAsString(body) + case lua.LTString: + data = lua.LVAsString(body) + case lua.LTTable: + mm, _ := lvToMap(body) + data = stringutil.ToJsonString(mm) + default: + data = stringutil.ToJsonString(body) + } + resp.Date = data + responds = append(responds, resp) + }) + + return responds, nil +} diff --git a/godb/service/luaengine/http_actuator.go b/godb/service/luaengine/http_actuator.go new file mode 100644 index 0000000..a8f8fd5 --- /dev/null +++ b/godb/service/luaengine/http_actuator.go @@ -0,0 +1,156 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/util/logs" +) + +func httpModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _httpModuleApi) + L.Push(t) + return 1 +} + +var _httpModuleApi = map[string]lua.LGFunction{ + "get": doGet, + "delete": doDelete, + "post": doPost, + "put": doPut, +} + +func doGet(L *lua.LState) int { + ret := L.NewTable() + paramUrl := L.CheckString(1) + paramOps := L.CheckTable(2) + + cli := _httpClient.GET(paramUrl) + if headers, ok := lvToMap(paramOps); ok { + cli.SetHeaders(headers) + } + + entity, err := cli.DoForEntity() + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + + ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) + ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) + + L.Push(ret) + return 1 +} + +func doDelete(L *lua.LState) int { + ret := L.NewTable() + paramUrl := L.CheckString(1) + paramOps := L.CheckTable(2) + + cli := _httpClient.DELETE(paramUrl) + if headers, ok := lvToMap(paramOps); ok { + cli.SetHeaders(headers) + } + + entity, err := cli.DoForEntity() + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + + ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) + ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) + + L.Push(ret) + return 1 +} + +func doPost(L *lua.LState) int { + ret := L.NewTable() + paramUrl := L.CheckString(1) + paramHeaders := L.CheckTable(2) + paramContents := L.CheckTable(3) + + cli := _httpClient.POST(paramUrl) + if headers, ok := lvToMap(paramHeaders); ok { + cli.SetHeaders(headers) + } + + contents, ok := lvToMap(paramContents) + if !ok { + logs.Error("The argument must Table") + L.Push(lua.LNil) + L.Push(lua.LString("The argument must Table")) + return 2 + } + + entity, err := cli.SetBodyAsForm(contents).DoForEntity() + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + + ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) + ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) + + L.Push(ret) + return 1 +} + +func doPut(L *lua.LState) int { + ret := L.NewTable() + paramUrl := L.CheckString(1) + paramHeaders := L.CheckTable(2) + paramContents := L.CheckTable(3) + + cli := _httpClient.PUT(paramUrl) + if headers, ok := lvToMap(paramHeaders); ok { + cli.SetHeaders(headers) + } + + contents, ok := lvToMap(paramContents) + if !ok { + logs.Error("The argument must Table") + L.Push(lua.LNil) + L.Push(lua.LString("The argument must Table")) + return 2 + } + + entity, err := cli.SetBodyAsForm(contents).DoForEntity() + if err != nil { + logs.Error(err.Error()) + L.Push(lua.LNil) + L.Push(lua.LString(err.Error())) + return 2 + } + + ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) + ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) + + L.Push(ret) + return 1 +} diff --git a/godb/service/luaengine/mongo_actuator.go b/godb/service/luaengine/mongo_actuator.go new file mode 100644 index 0000000..7ba8c50 --- /dev/null +++ b/godb/service/luaengine/mongo_actuator.go @@ -0,0 +1,161 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "github.com/juju/errors" + "github.com/siddontang/go-mysql/canal" + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" + "go-mysql-transfer/util/stringutil" +) + +func mongoModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _mongoModuleApi) + L.Push(t) + return 1 +} + +var _mongoModuleApi = map[string]lua.LGFunction{ + "rawRow": rawRow, + "rawAction": rawAction, + + "INSERT": mongoInsert, + "UPDATE": mongoUpdate, + "DELETE": mongoDelete, + "UPSERT": mongoUpsert, +} + +func mongoInsert(L *lua.LState) int { + collection := L.CheckAny(1) + table := L.CheckAny(2) + + data := L.NewTable() + L.SetTable(data, lua.LString("collection"), collection) + L.SetTable(data, lua.LString("action"), lua.LString(canal.InsertAction)) + L.SetTable(data, lua.LString("table"), table) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func mongoUpdate(L *lua.LState) int { + collection := L.CheckAny(1) + id := L.CheckAny(2) + table := L.CheckAny(3) + + data := L.NewTable() + L.SetTable(data, lua.LString("collection"), collection) + L.SetTable(data, lua.LString("action"), lua.LString(canal.UpdateAction)) + L.SetTable(data, lua.LString("id"), id) + L.SetTable(data, lua.LString("table"), table) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func mongoUpsert(L *lua.LState) int { + collection := L.CheckAny(1) + id := L.CheckAny(2) + table := L.CheckAny(3) + + data := L.NewTable() + L.SetTable(data, lua.LString("collection"), collection) + L.SetTable(data, lua.LString("action"), lua.LString(global.UpsertAction)) + L.SetTable(data, lua.LString("id"), id) + L.SetTable(data, lua.LString("table"), table) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func mongoDelete(L *lua.LState) int { + collection := L.CheckAny(1) + id := L.CheckAny(2) + + data := L.NewTable() + L.SetTable(data, lua.LString("collection"), collection) + L.SetTable(data, lua.LString("action"), lua.LString(canal.DeleteAction)) + L.SetTable(data, lua.LString("id"), id) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString(stringutil.UUID()), data) + return 0 +} + +func DoMongoOps(input map[string]interface{}, action string, rule *global.Rule) ([]*model.MongoRespond, error) { + L := _pool.Get() + defer _pool.Put(L) + + row := L.NewTable() + paddingTable(L, row, input) + ret := L.NewTable() + L.SetGlobal(_globalRET, ret) + L.SetGlobal(_globalROW, row) + L.SetGlobal(_globalACT, lua.LString(action)) + + funcFromProto := L.NewFunctionFromProto(rule.LuaProto) + L.Push(funcFromProto) + err := L.PCall(0, lua.MultRet, nil) + if err != nil { + return nil, err + } + + asserted := true + responds := make([]*model.MongoRespond, 0, ret.Len()) + ret.ForEach(func(k lua.LValue, v lua.LValue) { + resp := new(model.MongoRespond) + resp.Collection = lvToString(L.GetTable(v, lua.LString("collection"))) + resp.Action = lvToString(L.GetTable(v, lua.LString("action"))) + resp.Id = lvToInterface(L.GetTable(v, lua.LString("id")), true) + lvTable := L.GetTable(v, lua.LString("table")) + + var table map[string]interface{} + if action != canal.DeleteAction { + table, asserted = lvToMap(lvTable) + if !asserted { + return + } + resp.Table = table + } + + if action == canal.InsertAction { + _id, ok := table["_id"] + if !ok { + resp.Id = stringutil.UUID() + table["_id"] = resp.Id + } else { + resp.Id = _id + } + } + + responds = append(responds, resp) + }) + + if !asserted { + return nil, errors.New("The parameter must be of table type") + } + + return responds, nil +} diff --git a/godb/service/luaengine/mq_actuator.go b/godb/service/luaengine/mq_actuator.go new file mode 100644 index 0000000..28d89f9 --- /dev/null +++ b/godb/service/luaengine/mq_actuator.go @@ -0,0 +1,85 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "github.com/siddontang/go-mysql/canal" + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" +) + +func mqModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _mqModuleApi) + L.Push(t) + return 1 +} + +var _mqModuleApi = map[string]lua.LGFunction{ + "rawRow": rawRow, + "rawOldRow": rawOldRow, + "rawAction": rawAction, + + "SEND": msgSend, +} + +func msgSend(L *lua.LState) int { + topic := L.CheckAny(1) + msg := L.CheckAny(2) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, msg, topic) + return 0 +} + +func DoMQOps(input map[string]interface{}, previous map[string]interface{}, action string, rule *global.Rule) ([]*model.MQRespond, error) { + L := _pool.Get() + defer _pool.Put(L) + + row := L.NewTable() + paddingTable(L, row, input) + ret := L.NewTable() + L.SetGlobal(_globalRET, ret) + L.SetGlobal(_globalROW, row) + L.SetGlobal(_globalACT, lua.LString(action)) + + if action == canal.UpdateAction { + oldRow := L.NewTable() + paddingTable(L, oldRow, previous) + L.SetGlobal(_globalOLDROW, oldRow) + } + + funcFromProto := L.NewFunctionFromProto(rule.LuaProto) + L.Push(funcFromProto) + err := L.PCall(0, lua.MultRet, nil) + if err != nil { + return nil, err + } + + list := make([]*model.MQRespond, 0, ret.Len()) + ret.ForEach(func(k lua.LValue, v lua.LValue) { + resp := new(model.MQRespond) + resp.ByteArray = lvToByteArray(k) + resp.Topic = lvToString(v) + list = append(list, resp) + }) + + return list, nil +} diff --git a/godb/service/luaengine/redis_actuator.go b/godb/service/luaengine/redis_actuator.go new file mode 100644 index 0000000..a59ce81 --- /dev/null +++ b/godb/service/luaengine/redis_actuator.go @@ -0,0 +1,245 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package luaengine + +import ( + "github.com/siddontang/go-mysql/canal" + "github.com/yuin/gopher-lua" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" + "go-mysql-transfer/util/stringutil" +) + +const _globalOLDROW = "___OLDROW___" + +func redisModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _redisModuleApi) + L.Push(t) + return 1 +} + +var _redisModuleApi = map[string]lua.LGFunction{ + "rawRow": rawRow, + "rawOldRow": rawOldRow, + "rawAction": rawAction, + + "SET": redisSet, + "DEL": redisDel, + + "HSET": redisHSet, + "HDEL": redisHDel, + + "RPUSH": redisRPush, + "LREM": redisLRem, + + "SADD": redisSAdd, + "SREM": redisSRem, + + "ZADD": redisZAdd, + "ZREM": redisZRem, +} + +func rawOldRow(L *lua.LState) int { + row := L.GetGlobal(_globalOLDROW) + L.Push(row) + return 1 +} + +func redisSet(L *lua.LState) int { + key := L.CheckString(1) + val := L.CheckAny(2) + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("insert_1_"+key), val) + return 0 +} + +func redisDel(L *lua.LState) int { + key := L.CheckString(1) + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("delete_1_"+key), lua.LBool(true)) + return 0 +} + +func redisHSet(L *lua.LState) int { + key := L.CheckString(1) + field := L.CheckAny(2) + val := L.CheckAny(3) + + hash := L.NewTable() + L.SetTable(hash, lua.LString("key"), lua.LString(key)) + L.SetTable(hash, lua.LString("field"), field) + L.SetTable(hash, lua.LString("val"), val) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("insert_2_"+stringutil.UUID()), hash) + return 0 +} + +func redisHDel(L *lua.LState) int { + key := L.CheckAny(1) + field := L.CheckAny(2) + + hash := L.NewTable() + L.SetTable(hash, lua.LString("key"), key) + L.SetTable(hash, lua.LString("field"), field) + L.SetTable(hash, lua.LString("val"), lua.LNumber(1)) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("delete_2_"+stringutil.UUID()), hash) + return 0 +} + +func redisRPush(L *lua.LState) int { + key := L.CheckString(1) + val := L.CheckAny(2) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("insert_3_"+key), val) + return 0 +} + +func redisLRem(L *lua.LState) int { + key := L.CheckString(1) + val := L.CheckAny(2) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("delete_3_"+key), val) + return 0 +} + +func redisSAdd(L *lua.LState) int { + key := L.CheckString(1) + val := L.CheckAny(2) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("insert_4_"+key), val) + return 0 +} + +func redisSRem(L *lua.LState) int { + key := L.CheckString(1) + val := L.CheckAny(2) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("delete_4_"+key), val) + return 0 +} + +func redisZAdd(L *lua.LState) int { + key := L.CheckString(1) + score := L.CheckAny(2) + val := L.CheckAny(3) + + hash := L.NewTable() + L.SetTable(hash, lua.LString("key"), lua.LString(key)) + L.SetTable(hash, lua.LString("score"), score) + L.SetTable(hash, lua.LString("val"), val) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("insert_5_"+stringutil.UUID()), hash) + return 0 +} + +func redisZRem(L *lua.LState) int { + key := L.CheckString(1) + val := L.CheckAny(2) + + ret := L.GetGlobal(_globalRET) + L.SetTable(ret, lua.LString("delete_5_"+key), val) + return 0 +} + +func DoRedisOps(input map[string]interface{}, previous map[string]interface{}, action string, rule *global.Rule) ([]*model.RedisRespond, error) { + L := _pool.Get() + defer _pool.Put(L) + + row := L.NewTable() + paddingTable(L, row, input) + ret := L.NewTable() + L.SetGlobal(_globalRET, ret) + L.SetGlobal(_globalROW, row) + L.SetGlobal(_globalACT, lua.LString(action)) + + if action == canal.UpdateAction { + oldRow := L.NewTable() + paddingTable(L, oldRow, previous) + L.SetGlobal(_globalOLDROW, oldRow) + } + + funcFromProto := L.NewFunctionFromProto(rule.LuaProto) + L.Push(funcFromProto) + err := L.PCall(0, lua.MultRet, nil) + if err != nil { + return nil, err + } + + ls := make([]*model.RedisRespond, 0, ret.Len()) + ret.ForEach(func(k lua.LValue, v lua.LValue) { + resp := new(model.RedisRespond) + kk := lvToString(k) + resp.Action = kk[0:6] + resp.Structure = structureName(kk[7:8]) + if resp.Action == canal.DeleteAction { + resp.Key = kk[9:len(kk)] + resp.Val = lvToInterface(v, true) + } else { + if resp.Structure == global.RedisStructureHash { + key := L.GetTable(v, lua.LString("key")) + field := L.GetTable(v, lua.LString("field")) + val := L.GetTable(v, lua.LString("val")) + resp.Key = key.String() + resp.Field = lvToString(field) + resp.Val = lvToInterface(val, true) + } else if resp.Structure == global.RedisStructureSortedSet { + key := L.GetTable(v, lua.LString("key")) + score := L.GetTable(v, lua.LString("score")) + val := L.GetTable(v, lua.LString("val")) + resp.Key = key.String() + scoreTemp := lvToString(score) + resp.Score = stringutil.ToFloat64Safe(scoreTemp) + resp.Val = lvToInterface(val, true) + } else { + resp.Key = kk[9:len(kk)] + resp.Val = lvToInterface(v, true) + } + } + + ls = append(ls, resp) + }) + + return ls, nil +} + +func structureName(code string) string { + switch code { + case "1": + return global.RedisStructureString + case "2": + return global.RedisStructureHash + case "3": + return global.RedisStructureList + case "4": + return global.RedisStructureSet + case "5": + return global.RedisStructureSortedSet + } + + return "" +} diff --git a/godb/service/luaengine/script_actuator.go b/godb/service/luaengine/script_actuator.go new file mode 100644 index 0000000..2bd97e3 --- /dev/null +++ b/godb/service/luaengine/script_actuator.go @@ -0,0 +1,39 @@ +package luaengine + +import ( + lua "github.com/yuin/gopher-lua" + + "go-mysql-transfer/global" +) + +func scriptModule(L *lua.LState) int { + t := L.NewTable() + L.SetFuncs(t, _scriptModuleApi) + L.Push(t) + return 1 +} + +var _scriptModuleApi = map[string]lua.LGFunction{ + "rawRow": rawRow, + "rawAction": rawAction, +} + +func DoScript(input map[string]interface{}, action string, rule *global.Rule) error { + L := _pool.Get() + defer _pool.Put(L) + + row := L.NewTable() + paddingTable(L, row, input) + + L.SetGlobal(_globalROW, row) + L.SetGlobal(_globalACT, lua.LString(action)) + + funcFromProto := L.NewFunctionFromProto(rule.LuaProto) + L.Push(funcFromProto) + err := L.PCall(0, lua.MultRet, nil) + if err != nil { + return err + } + + return nil +} diff --git a/godb/service/service.go b/godb/service/service.go new file mode 100644 index 0000000..acbfae8 --- /dev/null +++ b/godb/service/service.go @@ -0,0 +1,69 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package service + +import ( + "go-mysql-transfer/global" + "go-mysql-transfer/service/election" +) + +var ( + _transferService *TransferService + _electionService election.Service + _clusterService *ClusterService +) + +func Initialize() error { + transferService := &TransferService{ + loopStopSignal: make(chan struct{}, 1), + } + err := transferService.initialize() + if err != nil { + return err + } + _transferService = transferService + + if global.Cfg().IsCluster() { + _clusterService = &ClusterService{ + electionSignal: make(chan bool, 1), + } + _electionService = election.NewElection(_clusterService.electionSignal) + } + + return nil +} + +func StartUp() { + if global.Cfg().IsCluster() { + _clusterService.boot() + } else { + _transferService.StartUp() + } +} + +func Close() { + _transferService.Close() +} + +func TransferServiceIns() *TransferService { + return _transferService +} + +func ClusterServiceIns() *ClusterService { + return _clusterService +} diff --git a/godb/service/stock_service.go b/godb/service/stock_service.go new file mode 100644 index 0000000..0ad37f9 --- /dev/null +++ b/godb/service/stock_service.go @@ -0,0 +1,382 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package service + +import ( + "fmt" + "github.com/juju/errors" + "github.com/siddontang/go-mysql/canal" + "go.uber.org/atomic" + "log" + "regexp" + "strings" + "sync" + + "go-mysql-transfer/global" + "go-mysql-transfer/model" + "go-mysql-transfer/service/endpoint" + "go-mysql-transfer/util/dates" + "go-mysql-transfer/util/logs" +) + +// 存量数据 +type StockService struct { + canal *canal.Canal + endpoint endpoint.Endpoint + + queueCh chan []*model.RowRequest + counter map[string]int64 + lockOfCounter sync.Mutex + totalRows map[string]int64 + wg sync.WaitGroup + shutoff *atomic.Bool +} + +func NewStockService() *StockService { + return &StockService{ + queueCh: make(chan []*model.RowRequest, global.Cfg().Maxprocs), + counter: make(map[string]int64), + totalRows: make(map[string]int64), + shutoff: atomic.NewBool(false), + } +} + +func (s *StockService) Run() error { + canalCfg := canal.NewDefaultConfig() + canalCfg.Addr = global.Cfg().Addr + canalCfg.User = global.Cfg().User + canalCfg.Password = global.Cfg().Password + canalCfg.Charset = global.Cfg().Charset + canalCfg.Flavor = global.Cfg().Flavor + canalCfg.ServerID = global.Cfg().SlaveID + canalCfg.Dump.ExecutionPath = global.Cfg().DumpExec + canalCfg.Dump.DiscardErr = false + canalCfg.Dump.SkipMasterData = global.Cfg().SkipMasterData + + if c, err := canal.NewCanal(canalCfg); err != nil { + errors.Trace(err) + } else { + s.canal = c + } + + if err := s.completeRules(); err != nil { + return errors.Trace(err) + } + s.addDumpDatabaseOrTable() + + endpoint := endpoint.NewEndpoint(s.canal) + if err := endpoint.Connect(); err != nil { + log.Println(err.Error()) + return errors.Trace(err) + } + s.endpoint = endpoint + + startTime := dates.NowMillisecond() + log.Println(fmt.Sprintf("bulk size: %d", global.Cfg().BulkSize)) + for _, rule := range global.RuleInsList() { + if rule.OrderByColumn == "" { + return errors.New("empty order_by_column not allowed") + } + + exportColumns := s.exportColumns(rule) + fullName := fmt.Sprintf("%s.%s", rule.Schema, rule.Table) + log.Println(fmt.Sprintf("开始导出 %s", fullName)) + + res, err := s.canal.Execute(fmt.Sprintf("select count(1) from %s", fullName)) + if err != nil { + return err + } + totalRow, err := res.GetInt(0, 0) + s.totalRows[fullName] = totalRow + log.Println(fmt.Sprintf("%s 共 %d 条数据", fullName, totalRow)) + + s.counter[fullName] = 0 + + var batch int64 + size := global.Cfg().BulkSize + if batch%size == 0 { + batch = totalRow / size + } else { + batch = (totalRow / size) + 1 + } + + var processed atomic.Int64 + for i := 0; i < global.Cfg().Maxprocs; i++ { + s.wg.Add(1) + go func(_fullName, _columns string, _rule *global.Rule) { + for { + processed.Inc() + requests, err := s.export(_fullName, _columns, processed.Load(), _rule) + if err != nil { + logs.Error(err.Error()) + s.shutoff.Store(true) + break + } + + s.imports(_fullName, requests) + if processed.Load() > batch { + break + } + } + s.wg.Done() + }(fullName, exportColumns, rule) + } + } + + s.wg.Wait() + + fmt.Println(fmt.Sprintf("共耗时 ：%d（毫秒）", dates.NowMillisecond()-startTime)) + + for k, v := range s.totalRows { + vv, ok := s.counter[k] + if ok { + fmt.Println(fmt.Sprintf("表： %s，共：%d 条数据，成功导入：%d 条", k, v, vv)) + if v > vv { + fmt.Println("存在导入错误的数据，具体请至日志查看") + } + } + } + + s.endpoint.Close() // 关闭客户端 + + return nil +} + +func (s *StockService) export(fullName, columns string, batch int64, rule *global.Rule) ([]*model.RowRequest, error) { + if s.shutoff.Load() { + return nil, errors.New("shutoff") + } + + offset := s.offset(batch) + sql := s.buildSql(fullName, columns, offset, rule) + logs.Infof("export sql : %s", sql) + resultSet, err := s.canal.Execute(sql) + if err != nil { + logs.Errorf("数据导出错误: %s - %s", sql, err.Error()) + return nil, err + } + rowNumber := resultSet.RowNumber() + requests := make([]*model.RowRequest, 0, rowNumber) + for i := 0; i < rowNumber; i++ { + rowValues := make([]interface{}, 0, len(rule.TableInfo.Columns)) + request := new(model.RowRequest) + for j := 0; j < len(rule.TableInfo.Columns); j++ { + val, err := resultSet.GetValue(i, j) + if err != nil { + logs.Errorf("数据导出错误: %s - %s", sql, err.Error()) + break + } + rowValues = append(rowValues, val) + request.Action = canal.InsertAction + request.RuleKey = global.RuleKey(rule.Schema, rule.Table) + request.Row = rowValues + } + requests = append(requests, request) + } + + return requests, nil +} + +// 构造SQL +func (s *StockService) buildSql(fullName, columns string, offset int64, rule *global.Rule) string { + size := global.Cfg().BulkSize + if len(rule.TableInfo.PKColumns) == 0 { + return fmt.Sprintf("select %s from %s order by %s limit %d,%d", columns, fullName, rule.OrderByColumn, offset, size) + } + + i := rule.TableInfo.PKColumns[0] + n := rule.TableInfo.GetPKColumn(i).Name + t := "select b.* from (select %s from %s order by %s limit %d,%d) a left join %s b on a.%s=b.%s" + sql := fmt.Sprintf(t, n, fullName, rule.OrderByColumn, offset, size, fullName, n, n) + return sql +} + +func (s *StockService) imports(fullName string, requests []*model.RowRequest) { + if s.shutoff.Load() { + return + } + + succeeds := s.endpoint.Stock(requests) + count := s.incCounter(fullName, succeeds) + log.Println(fmt.Sprintf("%s 导入数据 %d 条", fullName, count)) +} + +func (s *StockService) exportColumns(rule *global.Rule) string { + if rule.IncludeColumnConfig != "" { + var columns string + includes := strings.Split(rule.IncludeColumnConfig, ",") + for _, c := range rule.TableInfo.Columns { + for _, e := range includes { + var column string + if strings.ToUpper(e) == strings.ToUpper(c.Name) { + column = c.Name + } else { + column = "null as " + c.Name + } + + if columns != "" { + columns = columns + "," + } + columns = columns + column + } + } + return columns + } + + if rule.ExcludeColumnConfig != "" { + var columns string + excludes := strings.Split(rule.ExcludeColumnConfig, ",") + for _, c := range rule.TableInfo.Columns { + for _, e := range excludes { + var column string + if strings.ToUpper(e) == strings.ToUpper(c.Name) { + column = "null as " + c.Name + } else { + column = c.Name + } + + if columns != "" { + columns = columns + "," + } + columns = columns + column + } + } + return columns + } + + return "*" +} + +func (s *StockService) offset(currentPage int64) int64 { + var offset int64 + + if currentPage > 0 { + offset = (currentPage - 1) * global.Cfg().BulkSize + } + + return offset +} + +func (s *StockService) Close() { + s.canal.Close() +} + +func (s *StockService) incCounter(name string, n int64) int64 { + s.lockOfCounter.Lock() + defer s.lockOfCounter.Unlock() + + c, ok := s.counter[name] + if ok { + c = c + n + s.counter[name] = c + } + + return c +} + +func (s *StockService) completeRules() error { + wildcards := make(map[string]bool) + for _, rc := range global.Cfg().RuleConfigs { + if rc.Table == "*" { + return errors.Errorf("wildcard * is not allowed for table name") + } + + if regexp.QuoteMeta(rc.Table) != rc.Table { //通配符 + if _, ok := wildcards[global.RuleKey(rc.Schema, rc.Schema)]; ok { + return errors.Errorf("duplicate wildcard table defined for %s.%s", rc.Schema, rc.Table) + } + + tableName := rc.Table + if rc.Table == "*" { + tableName = "." + rc.Table + } + sql := fmt.Sprintf(`SELECT table_name FROM information_schema.tables WHERE + table_name RLIKE "%s" AND table_schema = "%s";`, tableName, rc.Schema) + res, err := s.canal.Execute(sql) + if err != nil { + return errors.Trace(err) + } + for i := 0; i < res.Resultset.RowNumber(); i++ { + tableName, _ := res.GetString(i, 0) + newRule, err := global.RuleDeepClone(rc) + if err != nil { + return errors.Trace(err) + } + newRule.Table = tableName + ruleKey := global.RuleKey(rc.Schema, tableName) + global.AddRuleIns(ruleKey, newRule) + } + } else { + newRule, err := global.RuleDeepClone(rc) + if err != nil { + return errors.Trace(err) + } + ruleKey := global.RuleKey(rc.Schema, rc.Table) + global.AddRuleIns(ruleKey, newRule) + } + } + + for _, rule := range global.RuleInsList() { + tableMata, err := s.canal.GetTable(rule.Schema, rule.Table) + if err != nil { + return errors.Trace(err) + } + if len(tableMata.PKColumns) == 0 { + if !global.Cfg().SkipNoPkTable { + return errors.Errorf("%s.%s must have a PK for a column", rule.Schema, rule.Table) + } + } + if len(tableMata.PKColumns) > 1 { + rule.IsCompositeKey = true // 组合主键 + } + rule.TableInfo = tableMata + rule.TableColumnSize = len(tableMata.Columns) + + if err := rule.Initialize(); err != nil { + return errors.Trace(err) + } + + if rule.LuaEnable() { + if err := rule.CompileLuaScript(global.Cfg().DataDir); err != nil { + return err + } + } + } + + return nil +} + +func (s *StockService) addDumpDatabaseOrTable() { + var schema string + schemas := make(map[string]int) + tables := make([]string, 0, global.RuleInsTotal()) + for _, rule := range global.RuleInsList() { + schema = rule.Table + schemas[rule.Schema] = 1 + tables = append(tables, rule.Table) + } + if len(schemas) == 1 { + s.canal.AddDumpTables(schema, tables...) + } else { + keys := make([]string, 0, len(schemas)) + for key := range schemas { + keys = append(keys, key) + } + s.canal.AddDumpDatabases(keys...) + } +} diff --git a/godb/service/transfer_service.go b/godb/service/transfer_service.go new file mode 100644 index 0000000..63a5731 --- /dev/null +++ b/godb/service/transfer_service.go @@ -0,0 +1,354 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package service + +import ( + "fmt" + "log" + "regexp" + "sync" + "time" + + "github.com/juju/errors" + "github.com/siddontang/go-mysql/canal" + "github.com/siddontang/go-mysql/mysql" + "go.uber.org/atomic" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/service/endpoint" + "go-mysql-transfer/storage" + "go-mysql-transfer/util/logs" +) + +const _transferLoopInterval = 1 + +type TransferService struct { + canal *canal.Canal + canalCfg *canal.Config + canalHandler *handler + canalEnable atomic.Bool + lockOfCanal sync.Mutex + firstsStart atomic.Bool + + wg sync.WaitGroup + endpoint endpoint.Endpoint + endpointEnable atomic.Bool + positionDao storage.PositionStorage + loopStopSignal chan struct{} +} + +func (s *TransferService) initialize() error { + s.canalCfg = canal.NewDefaultConfig() + s.canalCfg.Addr = global.Cfg().Addr + s.canalCfg.User = global.Cfg().User + s.canalCfg.Password = global.Cfg().Password + s.canalCfg.Charset = global.Cfg().Charset + s.canalCfg.Flavor = global.Cfg().Flavor + s.canalCfg.ServerID = global.Cfg().SlaveID + s.canalCfg.Dump.ExecutionPath = global.Cfg().DumpExec + s.canalCfg.Dump.DiscardErr = false + s.canalCfg.Dump.SkipMasterData = global.Cfg().SkipMasterData + + if err := s.createCanal(); err != nil { + return errors.Trace(err) + } + + if err := s.completeRules(); err != nil { + return errors.Trace(err) + } + + s.addDumpDatabaseOrTable() + + positionDao := storage.NewPositionStorage() + if err := positionDao.Initialize(); err != nil { + return errors.Trace(err) + } + s.positionDao = positionDao + + // endpoint + endpoint := endpoint.NewEndpoint(s.canal) + if err := endpoint.Connect(); err != nil { + return errors.Trace(err) + } + // 异步，必须要ping下才能确定连接成功 + if global.Cfg().IsMongodb() { + err := endpoint.Ping() + if err != nil { + return err + } + } + s.endpoint = endpoint + s.endpointEnable.Store(true) + metrics.SetDestState(metrics.DestStateOK) + + s.firstsStart.Store(true) + s.startLoop() + + return nil +} + +func (s *TransferService) run() error { + current, err := s.positionDao.Get() + if err != nil { + return err + } + + s.wg.Add(1) + go func(p mysql.Position) { + s.canalEnable.Store(true) + log.Println(fmt.Sprintf("transfer run from position(%s %d)", p.Name, p.Pos)) + if err := s.canal.RunFrom(p); err != nil { + log.Println(fmt.Sprintf("start transfer : %v", err)) + logs.Errorf("canal : %v", errors.ErrorStack(err)) + if s.canalHandler != nil { + s.canalHandler.stopListener() + } + s.canalEnable.Store(false) + } + + logs.Info("Canal is Closed") + s.canalEnable.Store(false) + s.canal = nil + s.wg.Done() + }(current) + + // canal未提供回调，停留一秒，确保RunFrom启动成功 + time.Sleep(time.Second) + return nil +} + +func (s *TransferService) StartUp() { + s.lockOfCanal.Lock() + defer s.lockOfCanal.Unlock() + + if s.firstsStart.Load() { + s.canalHandler = newHandler() + s.canal.SetEventHandler(s.canalHandler) + s.canalHandler.startListener() + s.firstsStart.Store(false) + s.run() + } else { + s.restart() + } +} + +func (s *TransferService) restart() { + if s.canal != nil { + s.canal.Close() + s.wg.Wait() + } + + s.createCanal() + s.addDumpDatabaseOrTable() + s.canalHandler = newHandler() + s.canal.SetEventHandler(s.canalHandler) + s.canalHandler.startListener() + s.run() +} + +func (s *TransferService) stopDump() { + s.lockOfCanal.Lock() + defer s.lockOfCanal.Unlock() + + if s.canal == nil { + return + } + + if !s.canalEnable.Load() { + return + } + + if s.canalHandler != nil { + s.canalHandler.stopListener() + s.canalHandler = nil + } + + s.canal.Close() + s.wg.Wait() + + log.Println("dumper stopped") +} + +func (s *TransferService) Close() { + s.stopDump() + s.loopStopSignal <- struct{}{} +} + +func (s *TransferService) Position() (mysql.Position, error) { + return s.positionDao.Get() +} + +func (s *TransferService) createCanal() error { + for _, rc := range global.Cfg().RuleConfigs { + s.canalCfg.IncludeTableRegex = append(s.canalCfg.IncludeTableRegex, rc.Schema+"\\."+rc.Table) + } + var err error + s.canal, err = canal.NewCanal(s.canalCfg) + return errors.Trace(err) +} + +func (s *TransferService) completeRules() error { + wildcards := make(map[string]bool) + for _, rc := range global.Cfg().RuleConfigs { + if rc.Table == "*" { + return errors.Errorf("wildcard * is not allowed for table name") + } + + if regexp.QuoteMeta(rc.Table) != rc.Table { //通配符 + if _, ok := wildcards[global.RuleKey(rc.Schema, rc.Schema)]; ok { + return errors.Errorf("duplicate wildcard table defined for %s.%s", rc.Schema, rc.Table) + } + + tableName := rc.Table + if rc.Table == "*" { + tableName = "." + rc.Table + } + sql := fmt.Sprintf(`SELECT table_name FROM information_schema.tables WHERE + table_name RLIKE "%s" AND table_schema = "%s";`, tableName, rc.Schema) + res, err := s.canal.Execute(sql) + if err != nil { + return errors.Trace(err) + } + for i := 0; i < res.Resultset.RowNumber(); i++ { + tableName, _ := res.GetString(i, 0) + newRule, err := global.RuleDeepClone(rc) + if err != nil { + return errors.Trace(err) + } + newRule.Table = tableName + ruleKey := global.RuleKey(rc.Schema, tableName) + global.AddRuleIns(ruleKey, newRule) + } + } else { + newRule, err := global.RuleDeepClone(rc) + if err != nil { + return errors.Trace(err) + } + ruleKey := global.RuleKey(rc.Schema, rc.Table) + global.AddRuleIns(ruleKey, newRule) + } + } + + for _, rule := range global.RuleInsList() { + tableMata, err := s.canal.GetTable(rule.Schema, rule.Table) + if err != nil { + return errors.Trace(err) + } + if len(tableMata.PKColumns) == 0 { + if !global.Cfg().SkipNoPkTable { + return errors.Errorf("%s.%s must have a PK for a column", rule.Schema, rule.Table) + } + } + if len(tableMata.PKColumns) > 1 { + rule.IsCompositeKey = true // 组合主键 + } + rule.TableInfo = tableMata + rule.TableColumnSize = len(tableMata.Columns) + + if err := rule.Initialize(); err != nil { + return errors.Trace(err) + } + + if rule.LuaEnable() { + if err := rule.CompileLuaScript(global.Cfg().DataDir); err != nil { + return err + } + } + } + + return nil +} + +func (s *TransferService) addDumpDatabaseOrTable() { + var schema string + schemas := make(map[string]int) + tables := make([]string, 0, global.RuleInsTotal()) + for _, rule := range global.RuleInsList() { + schema = rule.Table + schemas[rule.Schema] = 1 + tables = append(tables, rule.Table) + } + if len(schemas) == 1 { + s.canal.AddDumpTables(schema, tables...) + } else { + keys := make([]string, 0, len(schemas)) + for key := range schemas { + keys = append(keys, key) + } + s.canal.AddDumpDatabases(keys...) + } +} + +func (s *TransferService) updateRule(schema, table string) error { + rule, ok := global.RuleIns(global.RuleKey(schema, table)) + if ok { + tableInfo, err := s.canal.GetTable(schema, table) + if err != nil { + return errors.Trace(err) + } + + if len(tableInfo.PKColumns) == 0 { + if !global.Cfg().SkipNoPkTable { + return errors.Errorf("%s.%s must have a PK for a column", rule.Schema, rule.Table) + } + } + + if len(tableInfo.PKColumns) > 1 { + rule.IsCompositeKey = true + } + + rule.TableInfo = tableInfo + rule.TableColumnSize = len(tableInfo.Columns) + + err = rule.AfterUpdateTableInfo() + if err != nil { + return err + } + } + + return nil +} + +func (s *TransferService) startLoop() { + go func() { + ticker := time.NewTicker(_transferLoopInterval * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if !s.endpointEnable.Load() { + err := s.endpoint.Ping() + if err != nil { + log.Println("destination not available,see the log file for details") + logs.Error(err.Error()) + } else { + s.endpointEnable.Store(true) + if global.Cfg().IsRabbitmq() { + s.endpoint.Connect() + } + s.StartUp() + metrics.SetDestState(metrics.DestStateOK) + } + } + case <-s.loopStopSignal: + return + } + } + }() +} diff --git a/godb/storage/bolt_position_storage.go b/godb/storage/bolt_position_storage.go new file mode 100644 index 0000000..47fb49f --- /dev/null +++ b/godb/storage/bolt_position_storage.go @@ -0,0 +1,71 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package storage + +import ( + "github.com/juju/errors" + "github.com/siddontang/go-mysql/mysql" + "github.com/vmihailenco/msgpack" + "go.etcd.io/bbolt" +) + +type boltPositionStorage struct { + Name string + Pos uint32 +} + +func (s *boltPositionStorage) Initialize() error { + return _bolt.Update(func(tx *bbolt.Tx) error { + bt := tx.Bucket(_positionBucket) + data := bt.Get(_fixPositionId) + if data != nil { + return nil + } + + bytes, err := msgpack.Marshal(mysql.Position{}) + if err != nil { + return err + } + return bt.Put(_fixPositionId, bytes) + }) +} + +func (s *boltPositionStorage) Save(pos mysql.Position) error { + return _bolt.Update(func(tx *bbolt.Tx) error { + bt := tx.Bucket(_positionBucket) + data, err := msgpack.Marshal(pos) + if err != nil { + return err + } + return bt.Put(_fixPositionId, data) + }) +} + +func (s *boltPositionStorage) Get() (mysql.Position, error) { + var entity mysql.Position + err := _bolt.View(func(tx *bbolt.Tx) error { + bt := tx.Bucket(_positionBucket) + data := bt.Get(_fixPositionId) + if data == nil { + return errors.NotFoundf("PositionStorage") + } + return msgpack.Unmarshal(data, &entity) + }) + + return entity, err +} diff --git a/godb/storage/election_storage.go b/godb/storage/election_storage.go new file mode 100644 index 0000000..4f6859f --- /dev/null +++ b/godb/storage/election_storage.go @@ -0,0 +1,36 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package storage + +import "go-mysql-transfer/global" + +type ElectionStorage interface { + Elect() error +} + +func NewElectionStorage(conf *global.Config) PositionStorage { + if conf.IsCluster() { + if conf.IsZk() { + return &zkPositionStorage{} + } + if conf.IsEtcd() { + } + } + + return nil +} diff --git a/godb/storage/etcd_position_storage.go b/godb/storage/etcd_position_storage.go new file mode 100644 index 0000000..c79c300 --- /dev/null +++ b/godb/storage/etcd_position_storage.go @@ -0,0 +1,66 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package storage + +import ( + "encoding/json" + + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/util/etcds" +) + +type etcdPositionStorage struct { +} + +func (s *etcdPositionStorage) Initialize() error { + data, err := json.Marshal(mysql.Position{}) + if err != nil { + return err + } + + err = etcds.CreateIfNecessary(global.Cfg().ZkPositionDir(), string(data), _etcdOps) + if err != nil { + return err + } + + return nil +} + +func (s *etcdPositionStorage) Save(pos mysql.Position) error { + data, err := json.Marshal(pos) + if err != nil { + return err + } + + return etcds.Save(global.Cfg().ZkPositionDir(), string(data), _etcdOps) +} + +func (s *etcdPositionStorage) Get() (mysql.Position, error) { + var entity mysql.Position + + data, _, err := etcds.Get(global.Cfg().ZkPositionDir(), _etcdOps) + if err != nil { + return entity, err + } + + err = json.Unmarshal(data, &entity) + + return entity, err +} diff --git a/godb/storage/position_storage.go b/godb/storage/position_storage.go new file mode 100644 index 0000000..cf45e98 --- /dev/null +++ b/godb/storage/position_storage.go @@ -0,0 +1,43 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package storage + +import ( + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" +) + +type PositionStorage interface { + Initialize() error + Save(pos mysql.Position) error + Get() (mysql.Position, error) +} + +func NewPositionStorage() PositionStorage { + if global.Cfg().IsCluster() { + if global.Cfg().IsZk() { + return &zkPositionStorage{} + } + if global.Cfg().IsEtcd() { + return &etcdPositionStorage{} + } + } + + return &boltPositionStorage{} +} diff --git a/godb/storage/storage.go b/godb/storage/storage.go new file mode 100644 index 0000000..401d209 --- /dev/null +++ b/godb/storage/storage.go @@ -0,0 +1,185 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package storage + +import ( + "errors" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/samuel/go-zookeeper/zk" + "go.etcd.io/bbolt" + "go.etcd.io/etcd/clientv3" + etcdlog "go.etcd.io/etcd/pkg/logutil" + + "go-mysql-transfer/global" + "go-mysql-transfer/util/byteutil" + "go-mysql-transfer/util/files" + "go-mysql-transfer/util/logagent" + "go-mysql-transfer/util/zookeepers" +) + +const ( + _boltFilePath = "db" + _boltFileName = "data.db" + _boltFileMode = 0600 +) + +var ( + _positionBucket = []byte("Position") + _fixPositionId = byteutil.Uint64ToBytes(uint64(1)) + + _bolt *bbolt.DB + _zkConn *zk.Conn + _zkStatusSignal <-chan zk.Event + _zkAddresses []string + + _etcdConn *clientv3.Client + _etcdOps clientv3.KV +) + +func Initialize() error { + if err := initBolt(); err != nil { + return err + } + + if global.Cfg().IsZk() { + if err := initZk(); err != nil { + return err + } + } + + if global.Cfg().IsEtcd() { + if err := initEtcd(); err != nil { + return err + } + } + + return nil +} + +func initBolt() error { + blotStorePath := filepath.Join(global.Cfg().DataDir, _boltFilePath) + if err := files.MkdirIfNecessary(blotStorePath); err != nil { + return errors.New(fmt.Sprintf("create boltdb store : %s", err.Error())) + } + + boltFilePath := filepath.Join(blotStorePath, _boltFileName) + bolt, err := bbolt.Open(boltFilePath, _boltFileMode, bbolt.DefaultOptions) + if err != nil { + return errors.New(fmt.Sprintf("open boltdb: %s", err.Error())) + } + + err = bolt.Update(func(tx *bbolt.Tx) error { + tx.CreateBucketIfNotExists(_positionBucket) + return nil + }) + + _bolt = bolt + + return err +} + +func initZk() error { + option := zk.WithLogger(logagent.NewZkLoggerAgent()) + list := strings.Split(global.Cfg().Cluster.ZkAddrs, ",") + conn, sig, err := zk.Connect(list, time.Second, option) //*10) + + if err != nil { + return err + } + + if global.Cfg().Cluster.ZkAuthentication != "" { + err = conn.AddAuth("digest", []byte(global.Cfg().Cluster.ZkAuthentication)) + if err != nil { + return err + } + } + + err = zookeepers.CreateDirIfNecessary(global.Cfg().ZkRootDir(), conn) + if err != nil { + return err + } + + err = zookeepers.CreateDirIfNecessary(global.Cfg().ZkClusterDir(), conn) + if err != nil { + return err + } + + _zkAddresses = list + _zkConn = conn + _zkStatusSignal = sig + + return nil +} + +func initEtcd() error { + etcdlog.DefaultZapLoggerConfig = logagent.EtcdZapLoggerConfig() + clientv3.SetLogger(logagent.NewEtcdLoggerAgent()) + + list := strings.Split(global.Cfg().Cluster.EtcdAddrs, ",") + config := clientv3.Config{ + Endpoints: list, + Username: global.Cfg().Cluster.EtcdUser, + Password: global.Cfg().Cluster.EtcdPassword, + DialTimeout: 1 * time.Second, + } + + client, err := clientv3.New(config) + if err != nil { + return err + } + _etcdConn = client + _etcdOps = clientv3.NewKV(_etcdConn) + + return nil +} + +func ZKConn() *zk.Conn { + return _zkConn +} + +func ZKStatusSignal() <-chan zk.Event { + return _zkStatusSignal +} + +func ZKAddresses() []string { + return _zkAddresses +} + +func EtcdConn() *clientv3.Client { + return _etcdConn +} + +func EtcdOps() clientv3.KV { + return _etcdOps +} + +func Close() { + if _bolt != nil { + _bolt.Close() + } + if _zkConn != nil { + _zkConn.Close() + } + if _etcdConn != nil { + _etcdConn.Close() + } +} diff --git a/godb/storage/zk_position_storage.go b/godb/storage/zk_position_storage.go new file mode 100644 index 0000000..0df5125 --- /dev/null +++ b/godb/storage/zk_position_storage.go @@ -0,0 +1,74 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package storage + +import ( + "encoding/json" + + "github.com/siddontang/go-mysql/mysql" + + "go-mysql-transfer/global" + "go-mysql-transfer/util/zookeepers" +) + +type zkPositionStorage struct { +} + +func (s *zkPositionStorage) Initialize() error { + pos, err := json.Marshal(mysql.Position{}) + if err != nil { + return err + } + + err = zookeepers.CreateDirWithDataIfNecessary(global.Cfg().ZkPositionDir(), pos ,_zkConn) + if err != nil { + return err + } + + err = zookeepers.CreateDirIfNecessary(global.Cfg().ZkNodesDir(), _zkConn) + return err +} + +func (s *zkPositionStorage) Save(pos mysql.Position) error { + _, stat, err := _zkConn.Get(global.Cfg().ZkPositionDir()) + if err != nil { + return err + } + + data, err := json.Marshal(pos) + if err != nil { + return err + } + + _, err = _zkConn.Set(global.Cfg().ZkPositionDir(), data, stat.Version) + + return err +} + +func (s *zkPositionStorage) Get() (mysql.Position, error) { + var entity mysql.Position + + data, _, err := _zkConn.Get(global.Cfg().ZkPositionDir()) + if err != nil { + return entity, err + } + + err = json.Unmarshal(data, &entity) + + return entity, err +} diff --git a/godb/util/byteutil/byte.go b/godb/util/byteutil/byte.go new file mode 100644 index 0000000..7601192 --- /dev/null +++ b/godb/util/byteutil/byte.go @@ -0,0 +1,98 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package byteutil + +import ( + "bytes" + "encoding/binary" + "encoding/json" +) + +func StrToBytes(u string) []byte { + return []byte(u) +} + +func BytesToStr(u []byte) string { + return string(u) +} + +func Uint64ToBytes(u uint64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, u) + return buf +} + +func Int64ToBytes(u int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(u)) + return buf +} + +func BytesToUint64(b []byte) uint64 { + if b == nil { + return 0 + } + return binary.BigEndian.Uint64(b) +} + +func BytesToInt64(b []byte) int64 { + if b == nil { + return 0 + } + return int64(binary.BigEndian.Uint64(b)) +} + +func Uint8ToBytes(u uint8) ([]byte, error) { + bytesBuffer := bytes.NewBuffer([]byte{}) + err := binary.Write(bytesBuffer, binary.BigEndian, &u) + if err != nil { + return nil, err + } + return bytesBuffer.Bytes(), nil +} + +func BytesToUint8(b []byte) (uint8, error) { + bytesBuffer := bytes.NewBuffer(b) + var tmp uint8 + err := binary.Read(bytesBuffer, binary.BigEndian, &tmp) + if err != nil { + return 0, err + } + return tmp, nil +} + +func BytesToUint32(b []byte) uint32 { + if b == nil { + return 0 + } + return binary.BigEndian.Uint32(b) +} + +func Uint32ToBytes(u uint32) []byte { + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf, u) + return buf +} + +func JsonBytes(v interface{}) []byte { + bytes, err := json.Marshal(v) + if nil != err { + return nil + } + return bytes +} diff --git a/godb/util/collections/array_util.go b/godb/util/collections/array_util.go new file mode 100644 index 0000000..c00ecd9 --- /dev/null +++ b/godb/util/collections/array_util.go @@ -0,0 +1,13 @@ +package collections + +import "go-mysql-transfer/util/stringutil" + +func Contain(array []string, v interface{}) bool { + vvv := stringutil.ToString(v) + for _, vv := range array { + if vv == vvv { + return true + } + } + return false +} diff --git a/godb/util/collections/blocking_queue.go b/godb/util/collections/blocking_queue.go new file mode 100644 index 0000000..569848e --- /dev/null +++ b/godb/util/collections/blocking_queue.go @@ -0,0 +1,82 @@ +package collections + +import ( + "sync" +) + +type BlockingQueue struct { + q *Queue + cond *sync.Cond +} + +func NewBlockingQueue() *BlockingQueue { + bq := &BlockingQueue{} + bq.q = NewQueue() + bq.cond = sync.NewCond(&bq.q.lock) + return bq +} + +// 将元素插入到队列末尾 +func (bq *BlockingQueue) Offer(value interface{}) { + bq.cond.L.Lock() + defer bq.cond.L.Unlock() + + element := &node{value: value} + if bq.q.size == 0 { + bq.q.first = element + bq.q.last = element + } else { + bq.q.last.next = element + bq.q.last = element + } + bq.q.size++ + + bq.cond.Signal() +} + +//获取队首元素，若成功，则返回队首元素；否则返回null +func (bq *BlockingQueue) Peek() (interface{}, bool) { + return bq.q.Peek() +} + +//移除并获取队首元素，若成功，则返回队首元素；否则返回null +func (bq *BlockingQueue) Poll() (interface{}, bool) { + return bq.q.Poll() +} + +func (bq *BlockingQueue) Size() int { + return bq.q.Size() +} + +func (bq *BlockingQueue) Clear() { + bq.q.Clear() +} + +// 移除并返回队列头部的元素, 如果队列为空，则阻塞 +func (bq *BlockingQueue) Take() interface{} { + bq.cond.L.Lock() + defer bq.cond.L.Unlock() + + for { + if bq.q.size == 0 { + bq.cond.Wait() // 此处唤醒 + } else { + break + } + } + + if bq.q.size == 1 { + val := bq.q.first.value + bq.q.first = nil + bq.q.last = nil + bq.q.size = 0 + return val + } + + element := bq.q.first + bq.q.first = bq.q.first.next + val := element.value + element = nil + bq.q.size-- + return val +} diff --git a/godb/util/collections/list.go b/godb/util/collections/list.go new file mode 100644 index 0000000..17bc2f4 --- /dev/null +++ b/godb/util/collections/list.go @@ -0,0 +1,80 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package collections + +import "sync" + +type List struct { + lock sync.RWMutex //互斥锁 + + elements []interface{} + size int +} + +func NewList(values ...interface{}) *List { + l := &List{} + if len(values) > 0 { + l.Add(values...) + } + return l +} + +func (s *List) within(index int) bool { + return index >= 0 && index < s.size +} + +func (s *List) Add(values ...interface{}) { + s.lock.Lock() + defer s.lock.Unlock() + + if len(values) > 0 { + s.elements = append(s.elements, values) + s.size = s.size + len(values) + } +} + +func (s *List) Get(index int) interface{} { + s.lock.RLock() + defer s.lock.RUnlock() + + if !s.within(index) { + return nil + } + + return s.elements[index] +} + +func (s *List) Remove(index int) { + s.lock.Lock() + defer s.lock.Unlock() + + if !s.within(index) { + return + } + + s.elements = append(s.elements[:index], s.elements[index+1:]...) + + s.size-- +} + +func (s *List) Size() int { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.size +} diff --git a/godb/util/collections/queue.go b/godb/util/collections/queue.go new file mode 100644 index 0000000..5778c27 --- /dev/null +++ b/godb/util/collections/queue.go @@ -0,0 +1,111 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package collections + +import "sync" + +type Queue struct { + lock sync.RWMutex + + first *node + last *node + size int +} + +type node struct { + value interface{} + next *node +} + +func NewQueue(values ...interface{}) *Queue { + q := &Queue{} + if len(values) > 0 { + q.Offer(values...) + } + return q +} + +// 将元素插入到队列末尾 +func (q *Queue) Offer(values ...interface{}) { + q.lock.Lock() + defer q.lock.Unlock() + + for _, value := range values { + element := &node{value: value} + if q.size == 0 { + q.first = element + q.last = element + } else { + q.last.next = element + q.last = element + } + q.size++ + } +} + +//获取队首元素，若成功，则返回队首元素；否则返回null +func (q *Queue) Peek() (interface{}, bool) { + q.lock.RLock() + defer q.lock.RUnlock() + + if q.size == 0 { + return nil, false + } + return q.first.value, true +} + +//移除并获取队首元素，若成功，则返回队首元素；否则返回null +func (q *Queue) Poll() (interface{}, bool) { + q.lock.Lock() + defer q.lock.Unlock() + + if q.size == 0 { + return nil, false + } + + if q.size == 1 { + val := q.first.value + q.first = nil + q.last = nil + q.size = 0 + return val, true + } + + element := q.first + q.first = q.first.next + val := element.value + element = nil + q.size-- + return val, true +} + +func (q *Queue) Size() int { + q.lock.RLock() + defer q.lock.RUnlock() + + return q.size +} + +func (q *Queue) Clear() { + q.lock.Lock() + defer q.lock.Unlock() + + q.size = 0 + q.first = nil + q.last = nil +} diff --git a/godb/util/dates/date.go b/godb/util/dates/date.go new file mode 100644 index 0000000..da259ee --- /dev/null +++ b/godb/util/dates/date.go @@ -0,0 +1,87 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package dates + +import ( + "time" +) + +const ( + // 日期格式 + DayFormatter = "2006-01-02" + // 日期时间格式--分 + DayTimeMinuteFormatter = "2006-01-02 15:04" + // 日期时间格式--秒 + DayTimeSecondFormatter = "2006-01-02 15:04:05" + // 日期时间格式--毫秒 + DayTimeMillisecondFormatter = "2006-01-02 15:04:05.sss" + // 时间格式--秒 + TimeSecondFormatter = "15:04:05" +) + +// 默认格式2006-01-02 15:04:05 +func NowFormatted() string { + return time.Now().Format(DayTimeSecondFormatter) +} + +func NowLayout(layout string) string { + return time.Now().Format(layout) +} + +func Layout(date time.Time, layout string) string { + return date.Format(layout) +} + +func DefaultLayout(time time.Time) string { + return time.Format(DayTimeSecondFormatter) +} + +func FromDefaultLayout(str string) time.Time { + loc, _ := time.LoadLocation("Local") + theTime, _ := time.ParseInLocation(DayTimeSecondFormatter, str, loc) + return theTime +} + +// 当前的毫秒时间戳 +func NowMillisecond() int64 { + return time.Now().UnixNano() / 1e6 +} + +func PastDayDate(pastDay int) time.Time { + return time.Now().AddDate(0, 0, -pastDay) +} + +func FutureDayDate(futureDay int) time.Time { + return time.Now().AddDate(0, 0, futureDay) +} + +func WeekStartDayDate() time.Time { + now := time.Now() + offset := int(time.Monday - now.Weekday()) + if offset > 0 { + offset = -6 + } + weekStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) + return weekStart +} + +func MonthStartDayDate() time.Time { + year, month, _ := time.Now().Date() + monthStart := time.Date(year, month, 1, 0, 0, 0, 0, time.Local) + return monthStart +} diff --git a/godb/util/dates/date_format.go b/godb/util/dates/date_format.go new file mode 100644 index 0000000..8db44bd --- /dev/null +++ b/godb/util/dates/date_format.go @@ -0,0 +1,124 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package dates + +import ( + "strings" +) + +const ( + yyyy = "2006" + yy = "06" + mmmm = "January" + mmm = "Jan" + mm = "04" + m = "1" + + dddd = "Monday" + ddd = "Mon" + dd = "02" + + HHT = "03" + HH = "15" + MM = "01" + SS = "05" + ss = "05" + tt = "PM" + Z = "MST" + ZZZ = "MST" + + o = "Z07:00" +) + +func ConvertGoFormat(format string) string { + var goFormate = format + if strings.Contains(goFormate, "YYYY") { + goFormate = strings.Replace(goFormate, "YYYY", yyyy, -1) + } else if strings.Contains(goFormate, "yyyy") { + goFormate = strings.Replace(goFormate, "yyyy", yyyy, -1) + } else if strings.Contains(goFormate, "YY") { + goFormate = strings.Replace(goFormate, "YY", yy, -1) + } else if strings.Contains(goFormate, "yy") { + goFormate = strings.Replace(goFormate, "yy", yy, -1) + } + + if strings.Contains(goFormate, "MMMM") { + goFormate = strings.Replace(goFormate, "MMMM", mmmm, -1) + } else if strings.Contains(goFormate, "mmmm") { + goFormate = strings.Replace(goFormate, "mmmm", mmmm, -1) + } else if strings.Contains(goFormate, "MMM") { + goFormate = strings.Replace(goFormate, "MMM", mmm, -1) + } else if strings.Contains(goFormate, "mmm") { + goFormate = strings.Replace(goFormate, "mmm", mmm, -1) + } else if strings.Contains(goFormate, "mm") { + goFormate = strings.Replace(goFormate, "mm", mm, -1) + } + + if strings.Contains(goFormate, "dddd") { + goFormate = strings.Replace(goFormate, "dddd", dddd, -1) + } else if strings.Contains(goFormate, "ddd") { + goFormate = strings.Replace(goFormate, "ddd", ddd, -1) + } else if strings.Contains(goFormate, "dd") { + goFormate = strings.Replace(goFormate, "dd", dd, -1) + } + + if strings.Contains(goFormate, "tt") { + if strings.Contains(goFormate, "HH") { + goFormate = strings.Replace(goFormate, "HH", HHT, -1) + } else if strings.Contains(goFormate, "hh") { + goFormate = strings.Replace(goFormate, "hh", HHT, -1) + } + goFormate = strings.Replace(goFormate, "tt", tt, -1) + } else { + if strings.Contains(goFormate, "HH") { + goFormate = strings.Replace(goFormate, "HH", HH, -1) + } else if strings.Contains(goFormate, "hh") { + goFormate = strings.Replace(goFormate, "hh", HH, -1) + } + goFormate = strings.Replace(goFormate, "tt", "", -1) + } + + if strings.Contains(goFormate, "MM") { + goFormate = strings.Replace(goFormate, "MM", MM, -1) + } + + if strings.Contains(goFormate, "SS") { + goFormate = strings.Replace(goFormate, "SS", SS, -1) + } else if strings.Contains(goFormate, "ss") { + goFormate = strings.Replace(goFormate, "ss", SS, -1) + } + + if strings.Contains(goFormate, "ZZZ") { + goFormate = strings.Replace(goFormate, "ZZZ", ZZZ, -1) + } else if strings.Contains(goFormate, "zzz") { + goFormate = strings.Replace(goFormate, "zzz", ZZZ, -1) + } else if strings.Contains(goFormate, "Z") { + goFormate = strings.Replace(goFormate, "Z", Z, -1) + } else if strings.Contains(goFormate, "z") { + goFormate = strings.Replace(goFormate, "z", Z, -1) + } + + if strings.Contains(goFormate, "tt") { + goFormate = strings.Replace(goFormate, "tt", tt, -1) + } + if strings.Contains(goFormate, "o") { + goFormate = strings.Replace(goFormate, "o", o, -1) + } + + return goFormate +} diff --git a/godb/util/etcds/etcd_util.go b/godb/util/etcds/etcd_util.go new file mode 100644 index 0000000..106785d --- /dev/null +++ b/godb/util/etcds/etcd_util.go @@ -0,0 +1,166 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package etcds + +import ( + "context" + "time" + + "github.com/juju/errors" + "go.etcd.io/etcd/clientv3" +) + +const _etcdOpsTimeout = 1 * time.Second + +type Node struct { + Key string + Value []byte + Revision int64 +} + +func CreateIfNecessary(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + _, err := ops.Txn(ctx).If( + clientv3.Compare(clientv3.ModRevision(key), "=", 0), + ).Then( + clientv3.OpPut(key, val, opts...), + ).Commit() + + if err != nil { + return errors.Trace(err) + } + + //if !resp.Succeeded { + // return errors.AlreadyExistsf("key %s in etcd", key) + //} + + return nil +} + +func Get(key string, ops clientv3.KV) ([]byte, int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + resp, err := ops.Get(ctx, key) + if err != nil { + return nil, -1, errors.Trace(err) + } + + if len(resp.Kvs) == 0 { + return nil, -1, errors.NotFoundf("key %s in etcd", key) + } + + return resp.Kvs[0].Value, resp.Header.Revision, nil +} + +func HasChildren(key string, ops clientv3.KV) (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + resp, err := ops.Get(ctx, key, clientv3.WithPrefix()) + if err != nil { + return false, errors.Trace(err) + } + + length := len(key) + for _, kv := range resp.Kvs { + key := string(kv.Key) + if len(key) > length { + return true, nil + } + } + + return false, nil +} + +func List(key string, ops clientv3.KV) (map[string]*Node, error) { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + ret := make(map[string]*Node) + + resp, err := ops.Get(ctx, key, clientv3.WithPrefix()) + if err != nil { + return ret, errors.Trace(err) + } + + length := len(key) + for _, kv := range resp.Kvs { + key := string(kv.Key) + if len(key) <= length { + continue + } + node := &Node{ + Key: key, + Value: kv.Value, + Revision: kv.Version, + } + ret[key] = node + } + + return ret, nil +} + +func Save(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + resp, err := ops.Txn(ctx).If( + clientv3.Compare(clientv3.ModRevision(key), ">", 0), + ).Then( + clientv3.OpPut(key, val, opts...), + ).Commit() + + if err != nil { + return errors.Trace(err) + } + + if !resp.Succeeded { + return errors.NotFoundf("key %s in etcd", key) + } + + return nil +} + +// UpdateOrCreate updates a key/value, if the key does not exist then create, or update +func UpdateOrCreate(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + _, err := ops.Do(ctx, clientv3.OpPut(key, val, opts...)) + if err != nil { + return errors.Trace(err) + } + + return nil +} + +func Delete(key string, ops clientv3.KV, opts ...clientv3.OpOption) error { + ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) + defer cancel() + + _, err := ops.Delete(ctx, key, opts...) + + if err != nil { + return errors.Trace(err) + } + + return nil +} diff --git a/godb/util/files/util.go b/godb/util/files/util.go new file mode 100644 index 0000000..76e13ac --- /dev/null +++ b/godb/util/files/util.go @@ -0,0 +1,49 @@ +package files + +import ( + "os" +) + +// 判断给定的文件路径是否存在 +func IsExist(path string) bool { + _, err := os.Stat(path) + if err == nil { + return true + } + if os.IsNotExist(err) { + return false + } + return false +} + +// 判断给定的路径是否是文件夹 +func IsDir(path string) bool { + if stat, err := os.Stat(path); err == nil { + return stat.IsDir() + } + return false +} + +// 给定的文件不存在则创建 +func CreateFileIfNecessary(path string) bool { + _, err := os.Stat(path) + if err != nil && os.IsNotExist(err) { + if file, err := os.Create(path); err == nil { + file.Close() + } + } + exist := IsExist(path) + return exist +} + +// 给定的目录不存在则创建 +func MkdirIfNecessary(path string) error { + _, err := os.Stat(path) + if err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(path, os.ModePerm); err != nil { + // os.Chmod(path, 0777) + return err + } + } + return nil +} diff --git a/godb/util/httpclient/criteria.go b/godb/util/httpclient/criteria.go new file mode 100644 index 0000000..748d614 --- /dev/null +++ b/godb/util/httpclient/criteria.go @@ -0,0 +1,68 @@ +package httpclient + +import "net/http" + +// 参数 +type H map[string]interface{} + +// 重试条件 +type RetryConditionFunc func(*http.Response) bool + +// 请求条件 +type requestCriteria struct { + timeout int + retryCount int + retryInterval int + retryConditions []RetryConditionFunc + headers H +} + +func newRequestCriteria() *requestCriteria { + return &requestCriteria{ + headers: make(H), + } +} + +// 设置请求头 +func (c *requestCriteria) AddHeader(name string, value interface{}) { + c.headers[name] = value +} + +// 设置请求头 +func (c *requestCriteria) AddHeaders(values H) { + for k, v := range values { + c.headers[k] = v + } +} + +// 设置超时时间，单位为秒 +func (c *requestCriteria) SetTimeout(_timeout int) { + c.timeout = _timeout +} + +// 设置重试次数 +func (c *requestCriteria) SetRetryCount(_retryCount int) { + c.retryCount = _retryCount +} + +// 设置重试间隔时间，单位为秒 +func (c *requestCriteria) SetRetryInterval(_retryInterval int) { + c.retryInterval = _retryInterval +} + +// 添加重试条件 +func (c *requestCriteria) AddRetryConditionFunc(_retryCondition RetryConditionFunc) { + if _retryCondition != nil { + c.retryConditions = append(c.retryConditions, _retryCondition) + } +} + +// 判断是否需要重试 +func (c *requestCriteria) needRetry(res *http.Response) bool { + for _, condition := range c.retryConditions { + if condition(res) { + return true + } + } + return false +} diff --git a/godb/util/httpclient/executor.go b/godb/util/httpclient/executor.go new file mode 100644 index 0000000..81e8482 --- /dev/null +++ b/godb/util/httpclient/executor.go @@ -0,0 +1,389 @@ +package httpclient + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/pkg/errors" + + "go-mysql-transfer/util/stringutil" +) + +const ( + _contentTypeForm = 1 + _contentTypeJson = 2 +) + +type FormFile string + +// 执行器 +type executor struct { + client *HttpClient + criteria *requestCriteria + + addr string + method string + expectStatus int +} + +type GetOrDeleteExecutor struct { + executor + parameters H +} + +type PostOrPutExecutor struct { + executor + body interface{} + contentType int +} + +// 全局Criteria覆盖本地Criteria +func (s *executor) overrideCriteria() { + global := s.client.criteria + local := s.criteria + + if local.retryCount == 0 { + local.retryCount = global.retryCount + } + + if local.retryInterval == 0 { + local.retryInterval = global.retryInterval + } + + for _, retryCondition := range global.retryConditions { + local.retryConditions = append(local.retryConditions, retryCondition) + } + + for k, v := range global.headers { + if _, exist := local.headers[k]; !exist { + local.headers[k] = v + } + } +} + +// 执行Request +func (s *executor) execute(request *http.Request) (*http.Response, error) { + s.overrideCriteria() + + for k, v := range s.criteria.headers { + request.Header.Add(k, stringutil.ToString(v)) + } + + startTime := time.Now().UnixNano() + res, err := s.client.inner.Do(request) + latency := (time.Now().UnixNano() - startTime) / int64(time.Millisecond) + + if nil == err { + s.client.logger.Sugar().Infof("请求成功, %s | %s | %d | %d(毫秒)", request.Method, request.URL.String(), res.StatusCode, latency) + } + + if s.criteria.retryCount > 0 && s.criteria.needRetry(res) { + for i := 0; i < s.criteria.retryCount; i++ { + s.client.logger.Sugar().Infof("第%d次重试： %s | %s )", i+1, request.Method, request.URL.String()) + + res, err = s.client.inner.Do(request) + if err != nil { + s.client.logger.Error(err.Error()) + } + if !s.criteria.needRetry(res) || (i+1) == s.criteria.retryCount { + return res, err + } + <-time.After(time.Duration(s.criteria.retryInterval) * time.Second) + } + } + + if s.expectStatus != 0 && s.expectStatus != res.StatusCode { + defer res.Body.Close() + return nil, errors.Errorf("Response status code : %d (%s)", res.StatusCode, http.StatusText(res.StatusCode)) + } + + return res, err +} + +// 转换Response为string +func (s *executor) responseAsString(response *http.Response) (string, error) { + defer response.Body.Close() + + if data, err := ioutil.ReadAll(response.Body); err == nil { + return string(data), nil + } + + return "", nil +} + +// 转换Response为string +func (s *executor) responseAsEntity(response *http.Response) (*RespondEntity, error) { + defer response.Body.Close() + + data, err := ioutil.ReadAll(response.Body) + if nil != err { + return nil, err + } + + return &RespondEntity{ + statusCode: response.StatusCode, + data: data, + }, nil +} + +// 设置请求头 +func (r *GetOrDeleteExecutor) AddHeader(name string, value interface{}) *GetOrDeleteExecutor { + r.criteria.AddHeader(name, value) + return r +} + +// 设置请求头 +func (r *GetOrDeleteExecutor) SetHeaders(values H) *GetOrDeleteExecutor { + r.criteria.AddHeaders(values) + return r +} + +// 设置重试次数 +func (r *GetOrDeleteExecutor) SetRetryCount(_retryCount int) *GetOrDeleteExecutor { + r.criteria.SetRetryCount(_retryCount) + return r +} + +// 设置重试间隔时间，单位为秒 +func (r *GetOrDeleteExecutor) SetRetryInterval(_retryInterval int) *GetOrDeleteExecutor { + r.criteria.SetRetryInterval(_retryInterval) + return r +} + +// 添加重试条件 +func (r *GetOrDeleteExecutor) AddRetryConditionFunc(_retryCondition RetryConditionFunc) *GetOrDeleteExecutor { + r.criteria.AddRetryConditionFunc(_retryCondition) + return r +} + +// 设置查询参数 +func (r *GetOrDeleteExecutor) AddParameter(name string, value interface{}) *GetOrDeleteExecutor { + r.parameters[name] = value + return r +} + +// 设置查询参数 +func (r *GetOrDeleteExecutor) AddParameters(values H) *GetOrDeleteExecutor { + for k, v := range values { + r.parameters[k] = v + } + return r +} + +// 设置预期请求状态 +func (r *GetOrDeleteExecutor) SetExpectStatus(expect int) *GetOrDeleteExecutor { + r.expectStatus = expect + return r +} + +// 执行请求 +func (r *GetOrDeleteExecutor) Do() (*http.Response, error) { + values := make(url.Values) + for k, v := range r.parameters { + values.Set(k, stringutil.ToString(v)) + } + + url := stringutil.UrlValuesToQueryString(r.addr, values) + req, err := http.NewRequest(r.method, url, nil) + if nil != err { + return nil, err + } + + return r.execute(req) +} + +func (r *GetOrDeleteExecutor) DoForString() (string, error) { + res, err := r.Do() + if nil != err { + return "", err + } + return r.responseAsString(res) +} + +func (r *GetOrDeleteExecutor) DoForEntity() (*RespondEntity, error) { + res, err := r.Do() + if nil != err { + return nil, err + } + return r.responseAsEntity(res) +} + +// 设置请求头 +func (r *PostOrPutExecutor) AddHeader(name string, value interface{}) *PostOrPutExecutor { + r.criteria.AddHeader(name, value) + return r +} + +// 设置请求头 +func (r *PostOrPutExecutor) SetHeaders(values H) *PostOrPutExecutor { + r.criteria.AddHeaders(values) + return r +} + +// 设置重试次数 +func (r *PostOrPutExecutor) SetRetryCount(_retryCount int) *PostOrPutExecutor { + r.criteria.SetRetryCount(_retryCount) + return r +} + +// 设置重试间隔时间，单位为秒 +func (r *PostOrPutExecutor) SetRetryInterval(_retryInterval int) *PostOrPutExecutor { + r.criteria.SetRetryInterval(_retryInterval) + return r +} + +// 添加重试条件 +func (r *PostOrPutExecutor) AddRetryConditionFunc(_retryCondition RetryConditionFunc) *PostOrPutExecutor { + r.criteria.AddRetryConditionFunc(_retryCondition) + return r +} + +// 设置预期请求状态 +func (r *PostOrPutExecutor) SetExpectStatus(expect int) *PostOrPutExecutor { + r.expectStatus = expect + return r +} + +func (r *PostOrPutExecutor) SetBodyAsForm(body H) *PostOrPutExecutor { + r.contentType = _contentTypeForm + r.body = body + return r +} + +// 设置请求的contentType 为： +// "application/json" +func (r *PostOrPutExecutor) SetBodyAsJson(body interface{}) *PostOrPutExecutor { + r.contentType = _contentTypeJson + r.body = body + return r +} + +// 执行请求 +func (r *PostOrPutExecutor) Do() (*http.Response, error) { + if _contentTypeForm == r.contentType { + return r.doFormRequest() + } + + var data []byte + switch r.body.(type) { + case string: + ft := r.body.(string) + data = []byte(ft) + case []byte: + data = r.body.([]byte) + default: + temp, err := json.Marshal(r.body) + if err != nil { + return nil, err + } + data = temp + } + + req, err := http.NewRequest(r.method, r.addr, bytes.NewReader(data)) + if nil != err { + return nil, err + } + req.Header.Add("Content-Type", "application/json") + return r.execute(req) +} + +func (r *PostOrPutExecutor) DoForString() (string, error) { + res, err := r.Do() + if nil != err { + return "", err + } + return r.responseAsString(res) +} + +func (r *PostOrPutExecutor) DoForEntity() (*RespondEntity, error) { + res, err := r.Do() + if nil != err { + return nil, err + } + return r.responseAsEntity(res) +} + +func (r *PostOrPutExecutor) doFormRequest() (*http.Response, error) { + data := r.body.(H) + isMultipart := false + values := make(url.Values) + for k, v := range data { + if _, ok := v.(FormFile); ok { + isMultipart = true + values = nil + break + } + values.Set(k, stringutil.ToString(v)) + } + + if !isMultipart { + req, err := http.NewRequest(r.method, r.addr, strings.NewReader(values.Encode())) + if nil != err { + return nil, err + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + return r.execute(req) + } + + var err error + bodyBuffer := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuffer) + + var closings []*os.File + defer func() { + for _, closing := range closings { + closing.Close() + } + }() + + for k, v := range data { + switch v.(type) { + case FormFile: + vv := v.(FormFile) + var fw io.Writer + fw, err = bodyWriter.CreateFormFile(k, string(vv)) + if err != nil { + break + } + + var file *os.File + file, err = os.Open(string(vv)) + if err != nil { + break + } + closings = append(closings, file) + + _, err = io.Copy(fw, file) + if err != nil { + break + } + default: + if err := bodyWriter.WriteField(k, stringutil.ToString(v)); err != nil { + return nil, err + } + } + } + + if err != nil { + return nil, err + } + + bodyWriter.Close() + + var req *http.Request + req, err = http.NewRequest(r.method, r.addr, bodyBuffer) + if nil != err { + return nil, err + } + + req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) + return r.execute(req) +} diff --git a/godb/util/httpclient/http_client.go b/godb/util/httpclient/http_client.go new file mode 100644 index 0000000..220a0be --- /dev/null +++ b/godb/util/httpclient/http_client.go @@ -0,0 +1,156 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ + +package httpclient + +import ( + "net/http" + "time" + + "go.uber.org/zap" + + "go-mysql-transfer/util/logs" +) + +var DefaultClient = NewClient() + +type HttpClient struct { + logger *zap.Logger + inner *http.Client + criteria *requestCriteria +} + +// 创建Client +func NewClient() *HttpClient { + return &HttpClient{ + logger: logs.Logger(), + inner: &http.Client{}, + criteria: newRequestCriteria(), + } +} + +// 设置超时时间，单位为秒 +func (c *HttpClient) SetTimeout(timeout int) *HttpClient { + if timeout > 0 { + c.criteria.SetTimeout(timeout) + c.inner.Timeout = time.Duration(timeout) * time.Second + } + return c +} + +func (c *HttpClient) SetLogger(logger *zap.Logger) { + c.logger = logger +} + +// 获取超时时间 +func (c *HttpClient) GetTimeout() int { + return c.criteria.timeout +} + +// 设置重试次数 +func (c *HttpClient) SetRetryCount(retryCount int) *HttpClient { + c.criteria.SetRetryCount(retryCount) + return c +} + +// 获取重试次数 +func (c *HttpClient) GetRetryCount() int { + return c.criteria.retryCount +} + +// 设置重试间隔时间，单位为秒 +func (c *HttpClient) SetRetryInterval(retryInterval int) *HttpClient { + c.criteria.SetRetryInterval(retryInterval) + return c +} + +// 获取重试间隔时间 +func (c *HttpClient) GetRetryInterval() int { + return c.criteria.retryInterval +} + +// 添加重试条件 +func (c *HttpClient) AddRetryConditionFunc(retryCondition RetryConditionFunc) *HttpClient { + c.criteria.AddRetryConditionFunc(retryCondition) + return c +} + +// 设置Transport （用于确定HTTP请求的创建机制） +// 如果为空，将会使用DefaultTransport +func (c *HttpClient) SetTransport(transport http.RoundTripper) *HttpClient { + if transport != nil { + c.inner.Transport = transport + } + return c +} + +// 添加请求头 +func (c *HttpClient) AddHeader(key string, val string) *HttpClient { + c.criteria.AddHeader(key, val) + return c +} + +// 添加请求头 +func (c *HttpClient) AddHeaders(values H) *HttpClient { + c.criteria.AddHeaders(values) + return c +} + +// Get请求 +func (c *HttpClient) GET(url string) *GetOrDeleteExecutor { + t := &GetOrDeleteExecutor{ + parameters: make(H), + } + t.client = c + t.method = http.MethodGet + t.addr = url + t.criteria = newRequestCriteria() + return t +} + +// Delete请求 +func (c *HttpClient) DELETE(url string) *GetOrDeleteExecutor { + t := &GetOrDeleteExecutor{ + parameters: make(H), + } + t.client = c + t.method = http.MethodDelete + t.addr = url + t.criteria = newRequestCriteria() + return t +} + +// Post请求 +func (c *HttpClient) POST(url string) *PostOrPutExecutor { + t := &PostOrPutExecutor{} + t.client = c + t.method = http.MethodPost + t.addr = url + t.criteria = newRequestCriteria() + return t +} + +// Put请求 +func (c *HttpClient) PUT(url string) *PostOrPutExecutor { + t := &PostOrPutExecutor{} + t.client = c + t.method = http.MethodPut + t.addr = url + t.criteria = newRequestCriteria() + return t +} diff --git a/godb/util/httpclient/http_util_test.go b/godb/util/httpclient/http_util_test.go new file mode 100644 index 0000000..0233e58 --- /dev/null +++ b/godb/util/httpclient/http_util_test.go @@ -0,0 +1,269 @@ +package httpclient + +import ( + "fmt" + "io/ioutil" + "net/http" + "testing" +) + +func TestHttpClientGet(t *testing.T) { + res, err := DefaultClient.GET("http://httpbin.org/get").Do() + if err != nil { + t.Error("get failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } +} + +func TestHttpClientTimeout(t *testing.T) { + res, err := DefaultClient.SetTimeout(10).GET("http://127.0.0.1:8801/zombie").Do() + if err != nil { + t.Error("get failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } +} + +func TestHttpClientGetForString(t *testing.T) { + str, err := DefaultClient.GET("http://httpbin.org/get").DoForString() + if err != nil { + t.Error("get failed", err) + } + + println(str) +} + +func TestHttpClientGetWithParameters(t *testing.T) { + // 实际请求地址：http://httpbin.org/get?age=18&name=wf + str, err := DefaultClient.GET("http://httpbin.org/get").AddParameters(H{ + "token": "12596358412", + }).AddParameters(H{ + "name": "wf", + "age": 18, + }).DoForString() + + if err != nil { + t.Error("get failed", err) + } + + println(str) +} + +func retryCondition(res *http.Response) bool { + if res == nil { // Response为空时候重试 + return true + } + if res.StatusCode != http.StatusOK { // 状态不是200的时候重试 + return true + } + + return true +} + +func TestHttpClientRetry(t *testing.T) { + NewClient(). + SetRetryCount(3). // 重试次数 + SetRetryInterval(5). // 重试间隔 （秒） + AddRetryConditionFunc(retryCondition). // 重试条件 + GET("http://test.org/get"). + Do() + + // log like: + //{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"} + //{"level":"info","msg":"第1次重试： GET | http://test.org/get )"} + //{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"} + //{"level":"info","msg":"第2次重试： GET | http://test.org/get )"} + //{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"} + //{"level":"info","msg":"第3次重试： GET | http://test.org/get )"} + //{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"} +} + +func TestHttpClientPostForm(t *testing.T) { + // Content-Type is : application/x-www-form-urlencoded + res, err := DefaultClient.POST("http://httpbin.org/post"). + SetBodyAsForm(H{ + "name": "wf", + "age": 18, + }).Do() + + if err != nil { + t.Error("post failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } + + data, err := ioutil.ReadAll(res.Body) + if nil != err { + t.Error("read failed", err) + } + + fmt.Println(string(data)) +} + +type Person struct { + Name string + Age int +} + +func TestHttpClientPostJson(t *testing.T) { + // Content-Type is : application/json + res, err := DefaultClient.POST("http://httpbin.org/post"). + SetBodyAsJson(H{ + "name": "wf", + "age": 18, + }).Do() + + // 支持对象类型 + //person := &Person{ + // Name: "wf", + // Age: 18, + //} + //res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(person).Do() + + // 支持字符串类型 + //jsonText := "{\"Name\":\"wf\",\"Age\":18}" + //res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(jsonText).Do() + + if err != nil { + t.Error("post failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } + + data, err := ioutil.ReadAll(res.Body) + if nil != err { + t.Error("read failed", err) + } + + fmt.Println(string(data)) +} + +func TestHttpClientPostMultipart(t *testing.T) { + // Content-Type is : multipart/form-data + res, err := DefaultClient.POST("http://httpbin.org/post"). + SetBodyAsForm(H{ + "name": "wf", + "age": 18, + "photo": FormFile("D:\\4B.jpg"), + "resume": FormFile("D:\\resume.rtf"), + }).Do() + + if err != nil { + t.Error("post failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } + + data, err := ioutil.ReadAll(res.Body) + if nil != err { + t.Error("read failed", err) + } + + fmt.Println(string(data)) +} + +func TestHttpClientDelete(t *testing.T) { + str, err := DefaultClient.DELETE("http://httpbin.org/delete").DoForString() + if err != nil { + t.Error("delete failed", err) + } + + println(str) +} + +func TestHttpClientPutForm(t *testing.T) { + // Content-Type is : application/x-www-form-urlencoded + res, err := DefaultClient.PUT("http://httpbin.org/put"). + SetBodyAsForm(H{ + "name": "wf", + "age": 18, + }).Do() + + if err != nil { + t.Error("put failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } + + data, err := ioutil.ReadAll(res.Body) + if nil != err { + t.Error("read failed", err) + } + + fmt.Println(string(data)) +} + +func TestHttpClientPutJson(t *testing.T) { + // Content-Type is : application/json + res, err := DefaultClient.PUT("http://httpbin.org/put"). + SetBodyAsJson(H{ + "name": "wf", + "age": 18, + }).Do() + + // 支持对象类型 + //person := &Person{ + // Name: "wf", + // Age: 18, + //} + //res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(person).Do() + + // 支持字符串类型 + //jsonText := "{\"Name\":\"wf\",\"Age\":18}" + //res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(jsonText).Do() + + if err != nil { + t.Error("post failed", err) + } + + defer res.Body.Close() + + if res.StatusCode != 200 { + t.Error("Status Code not 200") + } + + data, err := ioutil.ReadAll(res.Body) + if nil != err { + t.Error("read failed", err) + } + + fmt.Println(string(data)) +} + +func TestResources(t *testing.T) { + entity, err := DefaultClient. + AddHeader("Authorization","adc8620e5164462e854f6f2e4e33ee53"). + GET("http://localhost:8090/portal/users/admin/resources"). + DoForEntity() + + if err != nil { + fmt.Println(err) + } + + fmt.Println("RespondText:",entity.DataAsString()) +} diff --git a/godb/util/httpclient/respond.go b/godb/util/httpclient/respond.go new file mode 100644 index 0000000..1152754 --- /dev/null +++ b/godb/util/httpclient/respond.go @@ -0,0 +1,36 @@ +package httpclient + +import ( + "encoding/json" + "net/http" +) + +// 应答实体 +type RespondEntity struct { + statusCode int + data []byte +} + +func (t *RespondEntity) StatusCode() int { + return t.statusCode +} + +func (t *RespondEntity) StatusText() string { + return http.StatusText(t.statusCode) +} + +func (t *RespondEntity) Data() []byte { + return t.data +} + +func (t *RespondEntity) DataAsString() string { + return string(t.data) +} + +func (t *RespondEntity) Unmarshal(entity interface{}) error { + err := json.Unmarshal(t.data, entity) + if err != nil { + return err + } + return nil +} diff --git a/godb/util/logagent/elastic_log_agent.go b/godb/util/logagent/elastic_log_agent.go new file mode 100644 index 0000000..905a1df --- /dev/null +++ b/godb/util/logagent/elastic_log_agent.go @@ -0,0 +1,31 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package logagent + +import "go-mysql-transfer/util/logs" + +type ElsLoggerAgent struct { +} + +func NewElsLoggerAgent() *ElsLoggerAgent { + return &ElsLoggerAgent{} +} + +func (s *ElsLoggerAgent) Printf(format string, v ...interface{}) { + logs.Infof(format, v) +} diff --git a/godb/util/logagent/etcd_log_agent.go b/godb/util/logagent/etcd_log_agent.go new file mode 100644 index 0000000..67fac81 --- /dev/null +++ b/godb/util/logagent/etcd_log_agent.go @@ -0,0 +1,146 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package logagent + +import ( + "go-mysql-transfer/util/logs" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func EtcdZapLoggerConfig() zap.Config { + return zap.Config{ + Level: zap.NewAtomicLevelAt(zap.InfoLevel), + Development: false, + Sampling: &zap.SamplingConfig{ + Initial: 100, + Thereafter: 100, + }, + Encoding: "console", + // copied from "zap.NewProductionEncoderConfig" with some updates + EncoderConfig: zapcore.EncoderConfig{ + TimeKey: "ts", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + + // Use "/dev/null" to discard all + OutputPaths: []string{"stderr"}, + ErrorOutputPaths: []string{"stderr"}, + } +} + +type EtcdLoggerAgent struct { +} + +func NewEtcdLoggerAgent() *EtcdLoggerAgent { + return &EtcdLoggerAgent{} +} + +// Info logs to INFO log. Arguments are handled in the manner of fmt.Print. +func (s *EtcdLoggerAgent) Info(args ...interface{}) { + for _, arg := range args { + logs.Infof("%v", arg) + } +} + +// Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. +func (s *EtcdLoggerAgent) Infoln(args ...interface{}) { + for _, arg := range args { + logs.Infof("%v", arg) + } +} + +// Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. +func (s *EtcdLoggerAgent) Infof(format string, args ...interface{}) { + logs.Infof(format, args...) +} + +// Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. +func (s *EtcdLoggerAgent) Warning(args ...interface{}) { + for _, arg := range args { + logs.Warnf("%v", arg) + } +} + +// Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. +func (s *EtcdLoggerAgent) Warningln(args ...interface{}) { + for _, arg := range args { + logs.Warnf("%v", arg) + } +} + +// Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. +func (s *EtcdLoggerAgent) Warningf(format string, args ...interface{}) { + logs.Warnf(format, args...) +} + +// Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. +func (s *EtcdLoggerAgent) Error(args ...interface{}) { + for _, arg := range args { + logs.Errorf("%v", arg) + } +} + +// Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. +func (s *EtcdLoggerAgent) Errorln(args ...interface{}) { + for _, arg := range args { + logs.Errorf("%v", arg) + } +} + +// Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. +func (s *EtcdLoggerAgent) Errorf(format string, args ...interface{}) { + logs.Errorf(format, args) +} + +// Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. +// gRPC ensures that all Fatal logs will exit with os.Exit(1). +// Implementations may also call os.Exit() with a non-zero exit code. +func (s *EtcdLoggerAgent) Fatal(args ...interface{}) { + for _, arg := range args { + logs.Errorf("%v", arg) + } +} + +// Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. +// gRPC ensures that all Fatal logs will exit with os.Exit(1). +// Implementations may also call os.Exit() with a non-zero exit code. +func (s *EtcdLoggerAgent) Fatalln(args ...interface{}) { + for _, arg := range args { + logs.Errorf("%v", arg) + } +} + +// Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. +// gRPC ensures that all Fatal logs will exit with os.Exit(1). +// Implementations may also call os.Exit() with a non-zero exit code. +func (s *EtcdLoggerAgent) Fatalf(format string, args ...interface{}) { + logs.Errorf(format, args) +} + +// V reports whether verbosity level l is at least the requested verbose level. +func (s *EtcdLoggerAgent) V(l int) bool { return false } diff --git a/godb/util/logagent/metrics_log_agent.go b/godb/util/logagent/metrics_log_agent.go new file mode 100644 index 0000000..56d2305 --- /dev/null +++ b/godb/util/logagent/metrics_log_agent.go @@ -0,0 +1,32 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package logagent + +import "go-mysql-transfer/util/logs" + +type MetricsLoggerAgent struct { +} + +func NewMetricsLoggerAgent() *MetricsLoggerAgent { + return &MetricsLoggerAgent{} +} + +// Info logs to INFO log. Arguments are handled in the manner of fmt.Print. +func (s *MetricsLoggerAgent) Printf(format string, v ...interface{}) { + logs.Infof(format, v) +} diff --git a/godb/util/logagent/rocketmq_log_agent.go b/godb/util/logagent/rocketmq_log_agent.go new file mode 100644 index 0000000..234c9b0 --- /dev/null +++ b/godb/util/logagent/rocketmq_log_agent.go @@ -0,0 +1,84 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package logagent + +import ( + "fmt" + "go.uber.org/zap/zapcore" + + "go-mysql-transfer/util/logs" +) + +type RocketmqLoggerAgent struct { +} + +func NewRocketmqLoggerAgent() *RocketmqLoggerAgent { + return &RocketmqLoggerAgent{} +} + +func (s *RocketmqLoggerAgent) Debug(msg string, fields map[string]interface{}) { + zapFields := make([]zapcore.Field, 0, len(fields)) + for k, v := range fields { + zapFields = append(zapFields, zapcore.Field{ + Key: k, + Type: zapcore.StringType, + String: fmt.Sprintf("%v", v), + }) + } + logs.Debug(msg, zapFields...) +} + +func (s *RocketmqLoggerAgent) Info(msg string, fields map[string]interface{}) { + zapFields := make([]zapcore.Field, 0, len(fields)) + for k, v := range fields { + zapFields = append(zapFields, zapcore.Field{ + Key: k, + Type: zapcore.StringType, + String: fmt.Sprintf("%v", v), + }) + } + logs.Info(msg, zapFields...) +} + +func (s *RocketmqLoggerAgent) Warning(msg string, fields map[string]interface{}) { + zapFields := make([]zapcore.Field, 0, len(fields)) + for k, v := range fields { + zapFields = append(zapFields, zapcore.Field{ + Key: k, + Type: zapcore.StringType, + String: fmt.Sprintf("%v", v), + }) + } + logs.Warn(msg, zapFields...) +} + +func (s *RocketmqLoggerAgent) Error(msg string, fields map[string]interface{}) { + zapFields := make([]zapcore.Field, 0, len(fields)) + for k, v := range fields { + zapFields = append(zapFields, zapcore.Field{ + Key: k, + Type: zapcore.StringType, + String: fmt.Sprintf("%v", v), + }) + } + logs.Error(msg, zapFields...) +} + +func (s *RocketmqLoggerAgent) Fatal(msg string, fields map[string]interface{}) { + s.Error(msg, fields) +} diff --git a/godb/util/logagent/zk_log_agent.go b/godb/util/logagent/zk_log_agent.go new file mode 100644 index 0000000..ab9434c --- /dev/null +++ b/godb/util/logagent/zk_log_agent.go @@ -0,0 +1,31 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package logagent + +import "go-mysql-transfer/util/logs" + +type ZkLoggerAgent struct { +} + +func NewZkLoggerAgent() *ZkLoggerAgent { + return &ZkLoggerAgent{} +} + +func (s *ZkLoggerAgent) Printf(template string, args ...interface{}) { + logs.Infof(template, args...) +} diff --git a/godb/util/nets/nets_test.go b/godb/util/nets/nets_test.go new file mode 100644 index 0000000..13f4955 --- /dev/null +++ b/godb/util/nets/nets_test.go @@ -0,0 +1,13 @@ +package nets + +import ( + "fmt" + "testing" +) + +func TestIsUsableAddr(t *testing.T) { + fmt.Println(IsUsableAddr(":8080")) + fmt.Println(IsUsableAddr(":8070")) + fmt.Println(IsUsableAddr(":8060")) + fmt.Println(IsUsableAddr(":8050")) +} diff --git a/godb/util/nets/util.go b/godb/util/nets/util.go new file mode 100644 index 0000000..ce329e2 --- /dev/null +++ b/godb/util/nets/util.go @@ -0,0 +1,120 @@ +package nets + +import ( + "net" + "regexp" + "strconv" + "strings" +) + +// 检测是否为 IP 格式 +func CheckIp(addr string) bool { + if "" == addr { + return false + } + + a := net.ParseIP(addr) + if a == nil { + return false + } + + return true +} + +// 检测 地址是否为 IP:端口 格式 +func CheckHostAddr(addr string) bool { + + if "" == addr { + return false + } + + items := strings.Split(addr, ":") + if items == nil || len(items) != 2 { + return false + } + + a := net.ParseIP(items[0]) + if a == nil { + return false + } + + match, err := regexp.MatchString("^[0-9]*$", items[1]) + if err != nil { + return false + } + + i, err := strconv.Atoi(items[1]) + if err != nil { + return false + } + if i < 0 || i > 65535 { + return false + } + + if match == false { + return false + } + return true +} + +// 获取一个空闲的TCP端口 +func GetFreePort(bind string) int { + ip := ":" + if "" != bind { + ip = bind + ":" + } + + var port int + for i := 17070; i < 65536; i++ { + addr, _ := net.ResolveTCPAddr("tcp", ip+strconv.Itoa(i)) + listener, err := net.ListenTCP("tcp", addr) + if err == nil { + listener.Close() + port = i + break + } + } + return port +} + +func IsUsableTcpAddr(addr string) (bool, error) { + listener, err := net.Listen("tcp", addr) + if err != nil { + return false, err + } + listener.Close() + return true, nil +} + +func IsActiveTCPAddr(addr string) (bool, error) { + tcpAddr, _ := net.ResolveTCPAddr("tcp", addr) + conn, err := net.DialTCP("tcp", nil, tcpAddr) + if err != nil { + return false, err + } + defer conn.Close() + + return true, nil +} + +func GetIpList() ([]string, error) { + var ips []string + netInterfaces, err := net.Interfaces() + if err != nil { + return nil, err + } + + for i := 0; i < len(netInterfaces); i++ { + if (netInterfaces[i].Flags & net.FlagUp) != 0 { + ls, _ := netInterfaces[i].Addrs() + for _, address := range ls { + if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { + if ipNet.IP.To4() != nil { + ips = append(ips, ipNet.IP.String()) + } + } + } + } + } + return ips, nil +} diff --git a/godb/util/snowflake/snow_flake.go b/godb/util/snowflake/snow_flake.go new file mode 100644 index 0000000..1ecf721 --- /dev/null +++ b/godb/util/snowflake/snow_flake.go @@ -0,0 +1,31 @@ +package snowflake + +import ( + "github.com/sony/sonyflake" + + "go-mysql-transfer/util/logs" +) + +// 雪花ID工具 + +var _sf *sonyflake.Sonyflake + +func InitSnowflake(machineId uint16) { + if _sf != nil { + return + } + + var st sonyflake.Settings + st.MachineID = func() (u uint16, e error) { + return machineId, nil + } + _sf = sonyflake.NewSonyflake(st) +} + +func NextId() (uint64, error) { + id, err := _sf.NextID() + if err != nil { + logs.Errorf("snowflake NextId ：%s", err.Error()) + } + return id, nil +} diff --git a/godb/util/stringutil/string_util.go b/godb/util/stringutil/string_util.go new file mode 100644 index 0000000..e927c40 --- /dev/null +++ b/godb/util/stringutil/string_util.go @@ -0,0 +1,315 @@ +/* + * Copyright 2020-2021 the original author(https://github.com/wj596) + * + *+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *
+ */ +package stringutil + +import ( + "bytes" + "crypto/hmac" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "unicode" + + "github.com/satori/go.uuid" +) + +// 产生UUID +func UUID() string { + return strings.ReplaceAll(uuid.NewV4().String(), "-", "") +} + +// 转换为Int +func ToIntSafe(str string) int { + v, e := strconv.Atoi(str) + if nil != e { + return 0 + } + return v +} + +// 转换为Uint64 +func ToInt64Safe(str string) int64 { + v, e := strconv.ParseInt(str, 10, 64) + if nil != e { + return 0 + } + return v +} + +// 转换为Uint64 +func ToUint64Safe(str string) uint64 { + v, e := strconv.ParseUint(str, 10, 64) + if nil != e { + return 0 + } + return v +} + +// Uint64转换为String +func Uint64ToStr(u uint64) string { + return strconv.FormatUint(u, 10) +} + +// 逗号分隔键值对转MAP,类似"name=wangjie,age=20"或者"name=wangjie|age=20" +func CommasToMap(base string, sep string) map[string]interface{} { + ret := make(map[string]interface{}) + if "" != base && "" != sep { + kvs := strings.Split(base, sep) + for _, kv := range kvs { + temp := strings.Split(kv, "=") + if len(temp) < 2 { + continue + } + if temp[0] == "" { + continue + } + ret[temp[0]] = temp[1] + } + } + return ret +} + +func ToJsonBytes(v interface{}) []byte { + bytes, err := json.Marshal(v) + if nil != err { + return nil + } + return bytes +} + +func ToJsonString(v interface{}) string { + bytes, err := json.Marshal(v) + if nil != err { + return "" + } + return string(bytes) +} + +func ToJsonIndent(v interface{}) string { + bytes, err := json.MarshalIndent(v, "", "\t") + if nil != err { + return "" + } + return string(bytes) +} + +// url.Values转查询字符串 +func UrlValuesToQueryString(base string, parameters url.Values) string { + if len(parameters) == 0 { + return base + } + + if !strings.Contains(base, "?") { + base += "?" + } + + if strings.HasSuffix(base, "?") || strings.HasSuffix(base, "&") { + base += parameters.Encode() + } else { + base += "&" + parameters.Encode() + } + + return base +} + +// map转查询字符串 +func MapToQueryString(base string, parameters map[string]interface{}) string { + if len(parameters) == 0 { + return base + } + + exist := false + if strings.Contains(base, "?") { + exist = true + } + var buffer bytes.Buffer + buffer.WriteString(base) + for k, v := range parameters { + var temp string + if !exist { + temp = "?" + k + "=" + ToString(v) + exist = true + } else { + temp = "&" + k + "=" + ToString(v) + } + buffer.WriteString(temp) + } + return buffer.String() +} + +// 转换为字符串 +func ToString(value interface{}) string { + var key string + if value == nil { + return key + } + + switch value.(type) { + case float64: + ft := value.(float64) + key = strconv.FormatFloat(ft, 'f', -1, 64) + case float32: + ft := value.(float32) + key = strconv.FormatFloat(float64(ft), 'f', -1, 64) + case int: + it := value.(int) + key = strconv.Itoa(it) + case uint: + it := value.(uint) + key = strconv.Itoa(int(it)) + case int8: + it := value.(int8) + key = strconv.Itoa(int(it)) + case uint8: + it := value.(uint8) + key = strconv.Itoa(int(it)) + case int16: + it := value.(int16) + key = strconv.Itoa(int(it)) + case uint16: + it := value.(uint16) + key = strconv.Itoa(int(it)) + case int32: + it := value.(int32) + key = strconv.Itoa(int(it)) + case uint32: + it := value.(uint32) + key = strconv.Itoa(int(it)) + case int64: + it := value.(int64) + key = strconv.FormatInt(it, 10) + case uint64: + it := value.(uint64) + key = strconv.FormatUint(it, 10) + case string: + key = value.(string) + case []byte: + key = string(value.([]byte)) + default: + newValue, _ := json.Marshal(value) + key = string(newValue) + } + + return key +} + +// 是否为邮件格式 +func IsEmailFormat(email string) bool { + pattern := `\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*` + reg := regexp.MustCompile(pattern) + return reg.MatchString(email) +} + +// 是否为中文 +func IsChineseChar(str string) bool { + for _, r := range str { + if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) { + return true + } + } + return false +} + +// MD5编码 +func MD5(str string) string { + h := md5.New() + h.Write([]byte(str)) + return hex.EncodeToString(h.Sum(nil)) +} + +func HmacSHA256(plaintext string, key string) string { + hash := hmac.New(sha256.New, []byte(key)) // 创建哈希算法 + hash.Write([]byte(plaintext)) // 写入数据 + return fmt.Sprintf("%X", hash.Sum(nil)) +} + +func HmacMD5(plaintext string, key string) string { + hash := hmac.New(md5.New, []byte(key)) // 创建哈希算法 + hash.Write([]byte(plaintext)) // 写入数据 + return fmt.Sprintf("%X", hash.Sum(nil)) +} + +// 驼峰式写法转为下划线写法 +func Camel2Case(name string) string { + buffer := new(bytes.Buffer) + for i, r := range name { + if unicode.IsUpper(r) { + if i != 0 { + buffer.WriteByte('_') + } + buffer.WriteRune(unicode.ToLower(r)) + } else { + buffer.WriteString(strconv.FormatInt(int64(r), 10)) + } + } + return buffer.String() +} + +// 下划线写法转为驼峰写法 +func Case2Camel(name string) string { + name = strings.Replace(name, "_", " ", -1) + name = strings.Title(name) + name = strings.Replace(name, " ", "", -1) + return Lcfirst(name) +} + +// 首字母大写 +func Ucfirst(str string) string { + for i, v := range str { + return string(unicode.ToUpper(v)) + str[i+1:] + } + return "" +} + +// 首字母小写 +func Lcfirst(str string) string { + for i, v := range str { + return string(unicode.ToLower(v)) + str[i+1:] + } + return "" +} + +func ToFloat64Safe(str string) float64 { + v, e := strconv.ParseFloat(str, 64) + if nil != e { + return 0 + } + return v +} + +func ToUint32(str string) (uint32, error) { + v, e := strconv.ParseUint(str, 10, 32) + if nil != e { + return 0, e + } + return uint32(v), nil +} + +func ToUint32Safe(str string) uint32 { + v, e := strconv.ParseUint(str, 10, 32) + if nil != e { + return 0 + } + return uint32(v) +} diff --git a/godb/util/stringutil/string_util_test.go b/godb/util/stringutil/string_util_test.go new file mode 100644 index 0000000..024a6b6 --- /dev/null +++ b/godb/util/stringutil/string_util_test.go @@ -0,0 +1,12 @@ +package stringutil + +import ( + "testing" +) + +func TestIsChineseChar(t *testing.T) { + println(IsChineseChar("a")) + println(IsChineseChar(",")) + println(IsChineseChar("a我b")) + println(IsChineseChar("，")) +} diff --git a/godb/util/sys/util.go b/godb/util/sys/util.go new file mode 100644 index 0000000..ea90c57 --- /dev/null +++ b/godb/util/sys/util.go @@ -0,0 +1,36 @@ +package sys + +import ( + "fmt" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" +) + +type Closer interface { + Close() +} + +// 获取程序运行路径 +func CurrentDirectory() string { + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + fmt.Errorf(err.Error()) + } + return strings.Replace(dir, "\\", "/", -1) +} + +func WaitCloseSignals() { + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT) + <-signals +} + +func WaitCloseSignalsAndRelease(closer Closer) { + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT) + <-signals + closer.Close() +} diff --git a/godb/util/zookeepers/util.go b/godb/util/zookeepers/util.go new file mode 100644 index 0000000..46fc9f1 --- /dev/null +++ b/godb/util/zookeepers/util.go @@ -0,0 +1,63 @@ +package zookeepers + +import ( + "strings" + + "github.com/samuel/go-zookeeper/zk" + + "go-mysql-transfer/util/stringutil" +) + +func CreateDirIfNecessary(dir string, conn *zk.Conn) error { + exist, _, err := conn.Exists(dir) + if err != nil { + return err + } + if !exist { + _, err := conn.Create(dir, nil, 0, zk.WorldACL(zk.PermAll)) + if err != nil { + return err + } + } + + return nil +} + +func CreateDirWithDataIfNecessary(dir string,data []byte, conn *zk.Conn) error { + exist, _, err := conn.Exists(dir) + if err != nil { + return err + } + if !exist { + _, err := conn.Create(dir, data, 0, zk.WorldACL(zk.PermAll)) + if err != nil { + return err + } + } + + return nil +} + +func DeleteDir(dir string, conn *zk.Conn) error { + _, stat, err := conn.Get(dir) + if err != nil { + return err + } + + return conn.Delete(dir, stat.Version) +} + +func JoinDir(args ...interface{}) string { + path := "" + for _, arg := range args { + if arg != "" { + p := stringutil.ToString(arg) + if strings.HasPrefix(p, "/") { + path = path + p + } else { + path = path + "/" + p + } + } + } + return path +} diff --git a/godb/web/router.go b/godb/web/router.go new file mode 100644 index 0000000..43bd8f5 --- /dev/null +++ b/godb/web/router.go @@ -0,0 +1,137 @@ +package web + +import ( + "fmt" + "go-mysql-transfer/service" + "go-mysql-transfer/util/dates" + "go-mysql-transfer/util/nets" + "log" + "net/http" + "path" + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "go-mysql-transfer/global" + "go-mysql-transfer/metrics" + "go-mysql-transfer/util/logs" +) + +var _server *http.Server + +func Start() error { + if !global.Cfg().EnableWebAdmin { //哨兵 + return nil + } + + gin.SetMode(gin.ReleaseMode) + g := gin.New() + //statics := "D:\\statics" + //index := "D:\\statics\\index.html" + + statics := "statics" + index := path.Join(statics, "index.html") + g.Static("/statics", statics) + g.LoadHTMLFiles(index) + g.GET("/", webAdminFunc) + + port := global.Cfg().WebAdminPort + listen := fmt.Sprintf(":%s", strconv.Itoa(port)) + _server = &http.Server{ + Addr: listen, + Handler: g, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + ok, err := nets.IsUsableTcpAddr(listen) + if !ok { + return err + } + + log.Println(fmt.Sprintf("Web Admin Listen At %s", listen)) + go func() { + if err := _server.ListenAndServe(); err != nil { + logs.Error(err.Error()) + } + }() + + return nil +} + +func webAdminFunc(c *gin.Context) { + pos, _ := service.TransferServiceIns().Position() + + var tables []string + for _, v := range global.RuleKeyList() { + tables = append(tables, v) + } + + var insertAmounts []uint64 + for _, v := range tables { + insertAmounts = append(insertAmounts, metrics.LabInsertAmount(v)) + } + + var updateAmounts []uint64 + for _, v := range tables { + updateAmounts = append(updateAmounts, metrics.LabUpdateRecord(v)) + } + + var deleteAmounts []uint64 + for _, v := range tables { + deleteAmounts = append(deleteAmounts, metrics.LabDeleteRecord(v)) + } + + h := gin.H{ + "mysql": global.Cfg().Addr, + "binName": pos.Name, + "binPos": pos.Pos, + "destName": global.Cfg().DestStdName(), + "destAddr": global.Cfg().DestAddr(), + "destState": metrics.DestState(), + "bootTime": dates.Layout(global.BootTime(), dates.DayTimeMinuteFormatter), + "insertAmount": metrics.InsertAmount(), + "updateAmount": metrics.UpdateAmount(), + "deleteAmount": metrics.DeleteAmount(), + "tables": tables, + "insertAmounts": insertAmounts, + "updateAmounts": updateAmounts, + "deleteAmounts": deleteAmounts, + "isCluster": global.Cfg().IsCluster(), + "isRedirect": false, + } + + if global.Cfg().IsCluster() { + h["isZk"] = global.Cfg().IsZk() + h["zkAddrs"] = global.Cfg().Cluster.ZkAddrs + h["etcdAddrs"] = global.Cfg().Cluster.EtcdAddrs + h["isLeader"] = global.IsLeader() + h["leader"] = global.LeaderNode() + if !global.IsLeader() { + h["isRedirect"] = true + } + + var followers []string + for _, v := range service.ClusterServiceIns().Nodes() { + if v != global.LeaderNode() { + followers = append(followers, v) + } + } + h["followers"] = followers + } + + c.HTML(200, "index.html", h) +} + +func Close() { + if _server == nil { + return + } + + err := _server.Shutdown(nil) + if err != nil { + log.Println(err.Error()) + } +} diff --git a/godb/web/statics/LICENSE b/godb/web/statics/LICENSE new file mode 100644 index 0000000..56a29a6 --- /dev/null +++ b/godb/web/statics/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 layuimini + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/godb/web/statics/README.md b/godb/web/statics/README.md new file mode 100644 index 0000000..22287fb --- /dev/null +++ b/godb/web/statics/README.md @@ -0,0 +1,76 @@ +layuimini后台模板 +=============== +# 项目介绍 +最简洁、清爽、易用的layui后台框架模板。 + +项目会不定时进行更新，建议star和watch一份。 + +技术交流QQ群：[1165301500](https://jq.qq.com/?_wv=1027&k=eUm5xKG1)、[667813249🈵](https://jq.qq.com/?_wv=1027&k=5lyiE2Q)、[561838086🈵](https://jq.qq.com/?_wv=1027&k=5JRGVfe) + +# 主要特性 +* 界面足够简洁清爽，响应式且适配手机端。 +* 一个接口`几行代码而已`直接初始化整个框架，无需复杂操作。 +* 页面支持多配色方案，可自行选择喜欢的配色。 +* 支持多tab，可以打开多窗口。 +* 支持无限级菜单和对font-awesome图标库的完美支持。 +* 失效以及报错菜单无法直接打开，并给出弹出层提示`完美的线上用户体验`。 +* url地址hash定位，可以清楚看到当前tab的地址信息。 +* 刷新页面会保留当前的窗口，并且会定位当前窗口对应左侧菜单栏。 +* 支持font-awesome图标选择插件 + + +# 代码仓库(iframe 多tab版) + +### v2版 + * 在线预览地址：[http://layuimini.99php.cn/iframe/v2/index.html](http://layuimini.99php.cn/iframe/v2/index.html) + * GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/v2](https://github.com/zhongshaofa/layuimini/tree/v2) + * Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/v2](https://gitee.com/zhongshaofa/layuimini/tree/v2) +### v1版 + * 在线预览地址：[http://layuimini.99php.cn/iframe/v1/index.html](http://layuimini.99php.cn/iframe/v1/index.html) + * GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/master](https://github.com/zhongshaofa/layuimini/tree/master) + * Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/master](https://gitee.com/zhongshaofa/layuimini/tree/master) + +# 代码仓库(onepage 单页版) + +### v2版 + * 在线预览地址：[http://layuimini.99php.cn/onepage/v2/index.html](http://layuimini.99php.cn/onepage/v2/index.html) + * GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/v2-onepage](https://github.com/zhongshaofa/layuimini/tree/v2-onepage) + * Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/v2-onepage](https://gitee.com/zhongshaofa/layuimini/tree/v2-onepage) + +### v1版 + * 在线预览地址：[http://layuimini.99php.cn/onepage/v1/index.html](http://layuimini.99php.cn/onepage/v1/index.html) + * GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/onepage](https://github.com/zhongshaofa/layuimini/tree/onepage) + * Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/onepage](https://gitee.com/zhongshaofa/layuimini/tree/onepage) + +# 下载方式 + +### iframe v2版 + * GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b v2` + * Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b v2` +### iframe v1版 + * GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b master` + * Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b master` +### 单页版 v2版 + * GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b v2-onepage` + * Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b v2-onepage` +### 单页版 v1版 + * GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b onepage` + * Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b onepage` +### 发行版地址 +* GitHub发版地址：[https://github.com/zhongshaofa/layuimini/releases](https://github.com/zhongshaofa/layuimini/releases) +* Gitee发版地址：[https://gitee.com/zhongshaofa/layuimini/releases](https://gitee.com/zhongshaofa/layuimini/releases) + +# 效果预览 +> 总体预览 + + + +# 使用说明 + +文档地址：[查看文档](http://layuimini.99php.cn/docs/) + + # 捐赠支持 + +开源项目不易，若此项目能得到你的青睐，可以捐赠支持作者持续开发与维护，感谢所有支持开源的朋友。 + +  diff --git a/godb/web/statics/api/clear.json b/godb/web/statics/api/clear.json new file mode 100644 index 0000000..e0f5ed7 --- /dev/null +++ b/godb/web/statics/api/clear.json @@ -0,0 +1,4 @@ +{ + "code": 1, + "msg": "服务端清理缓存成功" +} \ No newline at end of file diff --git a/godb/web/statics/api/init.json b/godb/web/statics/api/init.json new file mode 100644 index 0000000..8c7383c --- /dev/null +++ b/godb/web/statics/api/init.json @@ -0,0 +1,245 @@ +{ + "homeInfo": { + "title": "首页", + "href": "page/welcome-1.html?t=1" + }, + "logoInfo": { + "title": "LAYUI MINI", + "image": "images/logo.png", + "href": "" + }, + "menuInfo": [ + { + "title": "常规管理", + "icon": "fa fa-address-book", + "href": "", + "target": "_self", + "child": [ + { + "title": "主页模板", + "href": "", + "icon": "fa fa-home", + "target": "_self", + "child": [ + { + "title": "主页一", + "href": "page/welcome-1.html", + "icon": "fa fa-tachometer", + "target": "_self" + }, + { + "title": "主页二", + "href": "page/welcome-2.html", + "icon": "fa fa-tachometer", + "target": "_self" + }, + { + "title": "主页三", + "href": "page/welcome-3.html", + "icon": "fa fa-tachometer", + "target": "_self" + } + ] + }, + { + "title": "菜单管理", + "href": "page/menu.html", + "icon": "fa fa-window-maximize", + "target": "_self" + }, + { + "title": "系统设置", + "href": "page/setting.html", + "icon": "fa fa-gears", + "target": "_self" + }, + { + "title": "表格示例", + "href": "page/table.html", + "icon": "fa fa-file-text", + "target": "_self" + }, + { + "title": "表单示例", + "href": "", + "icon": "fa fa-calendar", + "target": "_self", + "child": [ + { + "title": "普通表单", + "href": "page/form.html", + "icon": "fa fa-list-alt", + "target": "_self" + }, + { + "title": "分步表单", + "href": "page/form-step.html", + "icon": "fa fa-navicon", + "target": "_self" + } + ] + }, + { + "title": "登录模板", + "href": "", + "icon": "fa fa-flag-o", + "target": "_self", + "child": [ + { + "title": "登录-1", + "href": "page/login-1.html", + "icon": "fa fa-stumbleupon-circle", + "target": "_blank" + }, + { + "title": "登录-2", + "href": "page/login-2.html", + "icon": "fa fa-viacoin", + "target": "_blank" + }, + { + "title": "登录-3", + "href": "page/login-3.html", + "icon": "fa fa-tags", + "target": "_blank" + } + ] + }, + { + "title": "异常页面", + "href": "", + "icon": "fa fa-home", + "target": "_self", + "child": [ + { + "title": "404页面", + "href": "page/404.html", + "icon": "fa fa-hourglass-end", + "target": "_self" + } + ] + }, + { + "title": "其它界面", + "href": "", + "icon": "fa fa-snowflake-o", + "target": "", + "child": [ + { + "title": "按钮示例", + "href": "page/button.html", + "icon": "fa fa-snowflake-o", + "target": "_self" + }, + { + "title": "弹出层", + "href": "page/layer.html", + "icon": "fa fa-shield", + "target": "_self" + } + ] + } + ] + }, + { + "title": "组件管理", + "icon": "fa fa-lemon-o", + "href": "", + "target": "_self", + "child": [ + { + "title": "图标列表", + "href": "page/icon.html", + "icon": "fa fa-dot-circle-o", + "target": "_self" + }, + { + "title": "图标选择", + "href": "page/icon-picker.html", + "icon": "fa fa-adn", + "target": "_self" + }, + { + "title": "颜色选择", + "href": "page/color-select.html", + "icon": "fa fa-dashboard", + "target": "_self" + }, + { + "title": "下拉选择", + "href": "page/table-select.html", + "icon": "fa fa-angle-double-down", + "target": "_self" + }, + { + "title": "文件上传", + "href": "page/upload.html", + "icon": "fa fa-arrow-up", + "target": "_self" + }, + { + "title": "富文本编辑器", + "href": "page/editor.html", + "icon": "fa fa-edit", + "target": "_self" + }, + { + "title": "省市县区选择器", + "href": "page/area.html", + "icon": "fa fa-rocket", + "target": "_self" + } + ] + }, + { + "title": "其它管理", + "icon": "fa fa-slideshare", + "href": "", + "target": "_self", + "child": [ + { + "title": "多级菜单", + "href": "", + "icon": "fa fa-meetup", + "target": "", + "child": [ + { + "title": "按钮1", + "href": "page/button.html?v=1", + "icon": "fa fa-calendar", + "target": "_self", + "child": [ + { + "title": "按钮2", + "href": "page/button.html?v=2", + "icon": "fa fa-snowflake-o", + "target": "_self", + "child": [ + { + "title": "按钮3", + "href": "page/button.html?v=3", + "icon": "fa fa-snowflake-o", + "target": "_self" + }, + { + "title": "表单4", + "href": "page/form.html?v=1", + "icon": "fa fa-calendar", + "target": "_self" + } + ] + } + ] + } + ] + }, + { + "title": "失效菜单", + "href": "page/error.html", + "icon": "fa fa-superpowers", + "target": "_self" + } + ] + } + ] +} \ No newline at end of file diff --git a/godb/web/statics/api/menus.json b/godb/web/statics/api/menus.json new file mode 100644 index 0000000..e14d00e --- /dev/null +++ b/godb/web/statics/api/menus.json @@ -0,0 +1,254 @@ +{ + "code": 0, + "msg": "", + "count": 19, + "data": [ + { + "authorityId": 1, + "authorityName": "系统管理", + "orderNumber": 1, + "menuUrl": null, + "menuIcon": "layui-icon-set", + "createTime": "2018/06/29 11:05:41", + "authority": null, + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 0, + "parentId": -1 + }, + { + "authorityId": 2, + "authorityName": "用户管理", + "orderNumber": 2, + "menuUrl": "system/user", + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": null, + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 0, + "parentId": 1 + }, + { + "authorityId": 3, + "authorityName": "查询用户", + "orderNumber": 3, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/07/21 13:54:16", + "authority": "user:view", + "checked": 0, + "updateTime": "2018/07/21 13:54:16", + "isMenu": 1, + "parentId": 2 + }, + { + "authorityId": 4, + "authorityName": "添加用户", + "orderNumber": 4, + "menuUrl": null, + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": "user:add", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 2 + }, + { + "authorityId": 5, + "authorityName": "修改用户", + "orderNumber": 5, + "menuUrl": null, + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": "user:edit", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 2 + }, + { + "authorityId": 6, + "authorityName": "删除用户", + "orderNumber": 6, + "menuUrl": null, + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": "user:delete", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 2 + }, + { + "authorityId": 7, + "authorityName": "角色管理", + "orderNumber": 7, + "menuUrl": "system/role", + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": null, + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 0, + "parentId": 1 + }, + { + "authorityId": 8, + "authorityName": "查询角色", + "orderNumber": 8, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/07/21 13:54:59", + "authority": "role:view", + "checked": 0, + "updateTime": "2018/07/21 13:54:58", + "isMenu": 1, + "parentId": 7 + }, + { + "authorityId": 9, + "authorityName": "添加角色", + "orderNumber": 9, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/06/29 11:05:41", + "authority": "role:add", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 7 + }, + { + "authorityId": 10, + "authorityName": "修改角色", + "orderNumber": 10, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/06/29 11:05:41", + "authority": "role:edit", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 7 + }, + { + "authorityId": 11, + "authorityName": "删除角色", + "orderNumber": 11, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/06/29 11:05:41", + "authority": "role:delete", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 7 + }, + { + "authorityId": 12, + "authorityName": "角色权限管理", + "orderNumber": 12, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/06/29 11:05:41", + "authority": "role:auth", + "checked": 0, + "updateTime": "2018/07/13 15:27:18", + "isMenu": 1, + "parentId": 7 + }, + { + "authorityId": 13, + "authorityName": "权限管理", + "orderNumber": 13, + "menuUrl": "system/authorities", + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": null, + "checked": 0, + "updateTime": "2018/07/13 15:45:13", + "isMenu": 0, + "parentId": 1 + }, + { + "authorityId": 14, + "authorityName": "查询权限", + "orderNumber": 14, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/07/21 13:55:57", + "authority": "authorities:view", + "checked": 0, + "updateTime": "2018/07/21 13:55:56", + "isMenu": 1, + "parentId": 13 + }, + { + "authorityId": 15, + "authorityName": "添加权限", + "orderNumber": 15, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/06/29 11:05:41", + "authority": "authorities:add", + "checked": 0, + "updateTime": "2018/06/29 11:05:41", + "isMenu": 1, + "parentId": 13 + }, + { + "authorityId": 16, + "authorityName": "修改权限", + "orderNumber": 16, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/07/13 09:13:42", + "authority": "authorities:edit", + "checked": 0, + "updateTime": "2018/07/13 09:13:42", + "isMenu": 1, + "parentId": 13 + }, + { + "authorityId": 17, + "authorityName": "删除权限", + "orderNumber": 17, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/06/29 11:05:41", + "authority": "authorities:delete", + "checked": 0, + "updateTime": "2018/06/29 11:05:41", + "isMenu": 1, + "parentId": 13 + }, + { + "authorityId": 18, + "authorityName": "登录日志", + "orderNumber": 18, + "menuUrl": "system/loginRecord", + "menuIcon": null, + "createTime": "2018/06/29 11:05:41", + "authority": null, + "checked": 0, + "updateTime": "2018/06/29 11:05:41", + "isMenu": 0, + "parentId": 1 + }, + { + "authorityId": 19, + "authorityName": "查询登录日志", + "orderNumber": 19, + "menuUrl": "", + "menuIcon": "", + "createTime": "2018/07/21 13:56:43", + "authority": "loginRecord:view", + "checked": 0, + "updateTime": "2018/07/21 13:56:43", + "isMenu": 1, + "parentId": 18 + } + ] +} \ No newline at end of file diff --git a/godb/web/statics/api/table.json b/godb/web/statics/api/table.json new file mode 100644 index 0000000..7bda61b --- /dev/null +++ b/godb/web/statics/api/table.json @@ -0,0 +1,127 @@ +{ + "code": 0, + "msg": "", + "count": 1000, + "data": [ + { + "id": 10000, + "username": "user-0", + "sex": "女", + "city": "城市-0", + "sign": "签名-0", + "experience": 255, + "logins": 24, + "wealth": 82830700, + "classify": "作家", + "score": 57 + }, + { + "id": 10001, + "username": "user-1", + "sex": "男", + "city": "城市-1", + "sign": "签名-1", + "experience": 884, + "logins": 58, + "wealth": 64928690, + "classify": "词人", + "score": 27 + }, + { + "id": 10002, + "username": "user-2", + "sex": "女", + "city": "城市-2", + "sign": "签名-2", + "experience": 650, + "logins": 77, + "wealth": 6298078, + "classify": "酱油", + "score": 31 + }, + { + "id": 10003, + "username": "user-3", + "sex": "女", + "city": "城市-3", + "sign": "签名-3", + "experience": 362, + "logins": 157, + "wealth": 37117017, + "classify": "诗人", + "score": 68 + }, + { + "id": 10004, + "username": "user-4", + "sex": "男", + "city": "城市-4", + "sign": "签名-4", + "experience": 807, + "logins": 51, + "wealth": 76263262, + "classify": "作家", + "score": 6 + }, + { + "id": 10005, + "username": "user-5", + "sex": "女", + "city": "城市-5", + "sign": "签名-5", + "experience": 173, + "logins": 68, + "wealth": 60344147, + "classify": "作家", + "score": 87 + }, + { + "id": 10006, + "username": "user-6", + "sex": "女", + "city": "城市-6", + "sign": "签名-6", + "experience": 982, + "logins": 37, + "wealth": 57768166, + "classify": "作家", + "score": 34 + }, + { + "id": 10007, + "username": "user-7", + "sex": "男", + "city": "城市-7", + "sign": "签名-7", + "experience": 727, + "logins": 150, + "wealth": 82030578, + "classify": "作家", + "score": 28 + }, + { + "id": 10008, + "username": "user-8", + "sex": "男", + "city": "城市-8", + "sign": "签名-8", + "experience": 951, + "logins": 133, + "wealth": 16503371, + "classify": "词人", + "score": 14 + }, + { + "id": 10009, + "username": "user-9", + "sex": "女", + "city": "城市-9", + "sign": "签名-9", + "experience": 484, + "logins": 25, + "wealth": 86801934, + "classify": "词人", + "score": 75 + } + ] +} \ No newline at end of file diff --git a/godb/web/statics/api/tableSelect.json b/godb/web/statics/api/tableSelect.json new file mode 100644 index 0000000..37fb0ed --- /dev/null +++ b/godb/web/statics/api/tableSelect.json @@ -0,0 +1,23 @@ +{ + "code": 0, + "msg": "", + "count": 16, + "data": [ + { "id":"001", "username":"张玉林", "sex":"女" }, + { "id":"002", "username":"刘晓军", "sex":"男" }, + { "id":"003", "username":"张恒", "sex":"男" }, + { "id":"004", "username":"朱一", "sex":"男" }, + { "id":"005", "username":"刘佳能", "sex":"女" }, + { "id":"006", "username":"晓梅", "sex":"女" }, + { "id":"007", "username":"马冬梅", "sex":"女" }, + { "id":"008", "username":"刘晓庆", "sex":"女" }, + { "id":"009", "username":"刘晓庆", "sex":"女" }, + { "id":"010", "username":"刘晓庆", "sex":"女" }, + { "id":"011", "username":"刘晓庆", "sex":"女" }, + { "id":"012", "username":"刘晓庆", "sex":"女" }, + { "id":"013", "username":"刘晓庆", "sex":"女" }, + { "id":"014", "username":"刘晓庆", "sex":"女" }, + { "id":"015", "username":"刘晓庆", "sex":"女" }, + { "id":"016", "username":"刘晓庆", "sex":"女" } + ] +} \ No newline at end of file diff --git a/godb/web/statics/api/upload.json b/godb/web/statics/api/upload.json new file mode 100644 index 0000000..691902d --- /dev/null +++ b/godb/web/statics/api/upload.json @@ -0,0 +1,10 @@ +{ + "code": 1, + "msg": "上传成功", + "data": { + "url": [ + "../images/logo.png", + "../images/captcha.jpg" + ] + } +} diff --git a/godb/web/statics/css/layuimini.css b/godb/web/statics/css/layuimini.css new file mode 100644 index 0000000..5a152bf --- /dev/null +++ b/godb/web/statics/css/layuimini.css @@ -0,0 +1,803 @@ +/** +配色方案（如有需要，请自行配置） + */ +/**头部-配色*/ +.layui-layout-admin .layui-header { + background-color: #1aa094 !important; +} + +.layui-header > ul > .layui-nav-item.layui-this, .layuimini-tool i:hover { + background-color: #197971 !important; +} + +.layui-header .layuimini-header-content > ul > .layui-nav-item.layui-this, .layuimini-tool i:hover { + background-color: #197971 !important; +} + +/**logo-配色*/ +.layui-layout-admin .layuimini-logo { + background-color: #243346 !important; +} + +/**左侧-配色*/ +.layui-side.layui-bg-black, .layui-side.layui-bg-black > .layuimini-menu-left > ul { + background-color: #2f4056 !important; +} + +.layuimini-menu-left .layui-nav .layui-nav-child a:hover:not(.layui-this) { + background-color: #3b3f4b; +} + +/**左侧菜单选中-配色*/ +.layui-layout-admin .layui-nav-tree .layui-this, .layui-layout-admin .layui-nav-tree .layui-this > a, .layui-layout-admin .layui-nav-tree .layui-nav-child dd.layui-this, .layui-layout-admin .layui-nav-tree .layui-nav-child dd.layui-this a { + background-color: #1aa094 !important; +} + + +/**头部样式 */ +.layui-layout-admin .header { + position: fixed; + left: 0; + right: 0; + top: 0; + bottom: 0; +} + +.layuimini-header-menu, .layui-header { + height: 60px !important; +} + +.layuimini-header-menu > .layui-nav-item { + color: #1b1d21; + height: 60px !important; + line-height: 60px !important; +} + +.layui-header > .layui-layout-right > .layui-nav-item { + height: 60px !important; + line-height: 60px !important; +} + +.layui-layout-left { + left: 295px !important; +} + +.layui-nav.layui-layout-left.layuimini-header-menu.layuimini-pc-show { + font-weight: bold; + transition: all .2s; +} + + +/**logo演示（通用） */ +.layui-layout-admin .layuimini-logo { + font-weight: bold; + color: #ffffff !important; + height: 60px !important; + line-height: 60px !important; + overflow: hidden; + line-height: 64px; + transition: all .2s !important; +} + +.layui-layout-admin .layuimini-logo img { + display: inline-block; + height: 40px; + vertical-align: middle; +} + +.layui-layout-admin .layuimini-logo h1 { + display: inline-block; + margin: 0 0 0 12px; + color: #ffffff; + font-weight: 600; + font-size: 20px; + font-family: Avenir, Helvetica Neue, Arial, Helvetica, sans-serif; + vertical-align: middle; +} + +/**缩放工具（通用） */ +.layuimini-tool { + position: absolute !important; + top: 0; + left: 235px; + width: 60px; + height: 100%; + line-height: 60px; + text-align: center; + color: #ffffff !important; + transition: all .2s; +} + +/**缩放工具（缩放） */ +.layuimini-tool i { + display: block; + color: #bbe3df; + width: 32px; + height: 32px; + line-height: 32px; + border-radius: 3px; + text-align: center; + margin-top: 15px; + cursor: pointer; +} + + +.layuimini-page-header { + overflow: hidden; + display: block; + height: 35px; + line-height: 35px; + margin-bottom: 0; + border-radius: 0; + border-bottom: 1px solid #e1dddd; +} + +.layuimini-page-header .layui-breadcrumb { + border-top: 1px solid #f6f6f6; + padding: 0 15px; + visibility: visible; +} + + +/**左侧菜单栏 (通用) */ +.layui-side.layui-bg-black { + transition: all .2s; +} + +.layui-side.layui-bg-black > .layuimini-menu-left > ul { + transition: all .2s; +} + +.layui-side.layui-bg-black > .layuimini-menu-left > ul > .layui-nav-item:first-child { + border-top: 1px solid #4b5461; +} + +.layuimini-menu-left .layui-nav .layui-nav-item a { + height: 40px; + line-height: 40px; + padding-right: 30px; +} + +.layuimini-menu-left .layui-nav .layui-nav-item > a { + padding-top: 5px; + padding-bottom: 5px; +} + +.layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child { + background: 0 0 !important +} + +.layuimini-menu-left .layui-nav .layui-nav-more { + right: 15px; +} + +.layuimini-menu-left .layui-nav .layui-nav-item a:hover { + background-color: transparent !important; +} + +.layuimini-menu-left .layui-nav { + background-color: transparent !important; +} + + +/**左侧菜单栏 (正常) */ +.layui-layout-body .layui-nav-itemed .layui-nav-child a, .layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child a { + padding-left: 35px; +} + +.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child a { + padding-left: 45px; +} + +.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child .layui-nav-child a { + padding-left: 55px; +} + +.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child .layui-nav-child .layui-nav-child a { + padding-left: 65px; +} + +.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-itemed > .layui-nav-child { + padding: 5px 0; +} + +/**内容主体（通用） */ +.layui-layout-admin .layui-body { + /*position: fixed;*/ + overflow: hidden; + bottom: 0px !important; + top: 60px !important; + transition: all .2s; +} + +/**选择配色方案 */ +.layuimini-color .color-title { + padding: 10px 0 10px 20px; + border-bottom: 1px solid #d9dada; + margin-bottom: 8px; +} + +.layuimini-color .color-content { + padding: 10px 5px 0 5px; +} + +.layuimini-color .color-content ul { + list-style: none; + text-align: center; +} + +.layuimini-color .color-content ul li { + position: relative; + display: inline-block; + vertical-align: top; + width: 80px; + height: 50px; + margin: 0 15px 15px 0; + padding: 2px 2px 4px 2px; + background-color: #f2f2f2; + cursor: pointer; + font-size: 12px; + color: #666; +} + +.layuimini-color .color-content li.layui-this:after, .layuimini-color .color-content li:hover:after { + width: 100%; + height: 100%; + padding: 4px; + top: -5px; + left: -5px; + border-color: #d8d8d8; + opacity: 1; +} + +.layuimini-color .color-content li:after { + content: ''; + position: absolute; + z-index: 20; + top: 50%; + left: 50%; + width: 1px; + height: 0; + border: 1px solid #f2f2f2; + transition: all .3s; + -webkit-transition: all .3s; + opacity: 0; +} + + +/**其它 */ +.layui-tab-item { + width: 100% !important; + height: 100% !important; +} + +.layui-nav-item.layui-this { + background-color: #1b1d21; +} + +.layui-width-height { + width: 100%; + height: 95%; +} + +.layui-tab { + margin: 0 0 0 0; + z-index: 99999; +} + +.text-center { + height: 30px !important; + line-height: 30px !important; + text-align: center !important; +} + +.layui-nav { + padding: 0 !important; +} + +.layui-nav .layui-this:after, .layui-nav-bar, .layui-nav-tree .layui-nav-itemed:after { + width: 0 !important; + height: 0 !important; +} + +.layui-layout-admin .layui-side { + top: 60px !important; +} + +.layui-tab-card { + box-shadow: 0px 0px 0px #888888; + border-bottom: 0; +} + + +/*打开页面动画*/ +.layui-tab-item.layui-show { + animation: moveTop 1s; + -webkit-animation: moveTop 1s; + animation-fill-mode: both; + -webkit-animation-fill-mode: both; + position: relative; + height: 100%; + -webkit-overflow-scrolling: touch; + overflow: auto; +} + +@keyframes moveTop { + 0% { + opacity: 0; + -webkit-transform: translateY(30px); + -ms-transform: translateY(30px); + transform: translateY(30px); + } + 100% { + opacity: 1; + -webkit-transform: translateY(0); + -ms-transform: translateY(0); + transform: translateY(0); + } +} + +@-o-keyframes moveTop { + 0% { + opacity: 0; + -webkit-transform: translateY(30px); + -ms-transform: translateY(30px); + transform: translateY(30px); + } + 100% { + opacity: 1; + -webkit-transform: translateY(0); + -ms-transform: translateY(0); + transform: translateY(0); + } +} + +@-moz-keyframes moveTop { + 0% { + opacity: 0; + -webkit-transform: translateY(30px); + -ms-transform: translateY(30px); + transform: translateY(30px); + } + 100% { + opacity: 1; + -webkit-transform: translateY(0); + -ms-transform: translateY(0); + transform: translateY(0); + } +} + +@-webkit-keyframes moveTop { + 0% { + opacity: 0; + -webkit-transform: translateY(30px); + -ms-transform: translateY(30px); + transform: translateY(30px); + } + 100% { + opacity: 1; + -webkit-transform: translateY(0); + -ms-transform: translateY(0); + transform: translateY(0); + } +} + +/**自定义滚动条样式 */ +::-webkit-scrollbar { + width: 6px; + height: 6px +} + +::-webkit-scrollbar-track { + background-color: transparent; + -webkit-border-radius: 2em; + -moz-border-radius: 2em; + border-radius: 2em; +} + +::-webkit-scrollbar-thumb { + background-color: #9c9da0; + -webkit-border-radius: 2em; + -moz-border-radius: 2em; + border-radius: 2em +} + + +.layuimini-content-page { + overflow: auto; + width: 100%; + height: 100%; +} + + +/*移动端遮罩层*/ +.layuimini-make { + position: fixed; + left: 0; + right: 0; + bottom: 0; + top: 0; + z-index: 1000; + background: rgba(0, 0, 0, .5); + display: none; +} + +.layuimini-mini .layui-header { + z-index: 1001; +} + +/**初始化加载层*/ +.layuimini-loader { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: #ffffff; + z-index: 999999; +} + +.layuimini-loader .layuimini-loader-inner { + display: block; + position: relative; + left: 50%; + top: 50%; + width: 150px; + height: 150px; + margin: -75px 0 0 -75px; + border-radius: 50%; + border: 3px solid transparent; + border-top-color: #1E9FFF; + animation: spin 2s linear infinite; +} + +.layuimini-loader .layuimini-loader-inner:before { + content: ""; + position: absolute; + top: 5px; + left: 5px; + right: 5px; + bottom: 5px; + border-radius: 50%; + border: 3px solid transparent; + border-top-color: #1E9FFF; + animation: spin 3s linear infinite; +} + +.layuimini-loader .layuimini-loader-inner:after { + content: ""; + position: absolute; + top: 15px; + left: 15px; + right: 15px; + bottom: 15px; + border-radius: 50%; + border: 3px solid transparent; + border-top-color: #1E9FFF; + animation: spin 1.5s linear infinite; +} + +@keyframes spin { + 0% { + transform: rotate(0deg); + } + to { + transform: rotate(1turn); + } +} + +/*系统设置*/ + +.layuimini-color .layui-word-aux { + position: absolute; + left: 60px; + top: 12px; + font-size: 12px; +} + +.layuimini-color .layui-input-block { + margin-left: 15px; + min-height: 36px; +} + +.layuimini-color .more-menu-list { + width: 100%; + margin-top: 30px; +} + + +.layuimini-color .more-menu-item:first-child { + border-top: 1px solid #e8e8e8; +} + +.layuimini-color .more-menu-item .layui-icon { + font-size: 18px; + padding-right: 10px; +} + +.layuimini-color .more-menu-item { + color: #595959; + height: 50px; + line-height: 50px; + font-size: 16px; + padding: 0 25px; + border-bottom: 1px solid #e8e8e8; + font-style: normal; + display: block; +} + +.layuimini-color .more-menu-item:hover { + background-color: whitesmoke; +} + +.layuimini-color .more-menu-item:after { + color: #8c8c8c; + right: 16px; + content: "\e602"; + position: absolute; + font-family: layui-icon !important; +} + +/** +菜单缩放 + */ +.popup-tips .layui-layer-TipsG{ + display: none; +} +.popup-tips.layui-layer-tips .layui-layer-content{ + padding: 0; +} +.popup-tips .layui-nav-tree{ + width: 150px; + border-radius: 10px; +} + +/**左侧菜单字体间距*/ +.layuimini-menu-left .layui-nav-item a span { + letter-spacing: 1px; +} + +/**头部菜单字体间距*/ +.layui-layout-admin .layui-header .layuimini-header-menu.layuimini-pc-show,.layui-layout-admin .layui-header .layuimini-header-menu.layuimini-mobile-show { + letter-spacing: 1px; +} + + +/**左侧菜单更多下拉样式*/ +.layuimini-menu-left .layui-nav-more,.layuimini-menu-left-zoom .layui-nav-more { + font-family: layui-icon !important; + font-size: 12px; + font-style: normal; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + overflow: hidden; + width: auto; + height: auto; + line-height: normal; + border: none; + display: inline-block; + margin-top: -6px !important; +} + +.layuimini-menu-left .layui-nav-child .layui-nav-more { + margin-top: -6px !important; +} + +.layuimini-menu-left .layui-nav .layui-nav-mored,.layuimini-menu-left .layui-nav-itemed>a .layui-nav-more{ + margin-top: -9px!important; +} + +.layuimini-menu-left-zoom.layui-nav .layui-nav-mored,.layuimini-menu-left-zoom.layui-nav-itemed>a .layui-nav-more{ + margin-top: -9px!important; +} + +.layuimini-menu-left .layui-nav-more:before,.layuimini-menu-left-zoom .layui-nav-more:before { + content: "\e61a"; +} +.layuimini-menu-left .layui-nav-itemed > a > .layui-nav-more,.layuimini-menu-left-zoom .layui-nav-itemed > a > .layui-nav-more { + transform: rotate(180deg); + -ms-transform: rotate(180deg); + -moz-transform: rotate(180deg); + -webkit-transform: rotate(180deg); + -o-transform: rotate(180deg); + width: 12px; + text-align: center; + border-style:none; +} + +.layuimini-menu-left .layui-nav-itemed > a > .layui-nav-more:before,.layuimini-menu-left-zoom .layui-nav-itemed > a > .layui-nav-more:before { + content: '\e61a'; + background-color: transparent; + display: inline-block; + vertical-align: middle; +} + +/**修复左侧菜单字体不对齐的问题*/ +.layuimini-menu-left .layui-nav-item a .fa,.layuimini-menu-left .layui-nav-item a .layui-icon{ + width: 20px; +} + + + +/** + PC版样式 + */ +@media screen and (min-width: 1025px) { + /**头部样式（缩放） */ + .layuimini-mini .layui-layout-left.layuimini-header-menu.layuimini-pc-show { + left: 155px !important; + } + + /**logo演示（缩放） */ + .layuimini-mini .layui-layout-admin .layuimini-logo { + width: 60px !important; + } + + .layuimini-mini .layui-layout-admin .layuimini-logo h1 { + display: none; + } + + /**左侧菜单栏（缩放） */ + .layuimini-mini .layuimini-menu-left { + width: 80px !important; + } + + .layuimini-mini .layui-side.layui-bg-black, .layuimini-mini .layuimini-menu-left > ul, .layuimini-mini .layuimini-menu-left > ul li i { + width: 60px !important; + } + + .layuimini-mini .layuimini-menu-left > ul li span:first-child { + display: none; + } + + .layuimini-mini .layuimini-menu-left > ul li span:last-child { + float: right; + right: 7px; + } + + .layuimini-mini .layuimini-menu-left .layui-nav .layui-nav-item a { + height: 40px; + line-height: 40px; + padding-right: 0px !important; + } + + /**内容主体（缩放） */ + .layuimini-mini .layui-layout-admin { + left: 60px !important; + } + + .layuimini-mini .layuimini-tool { + left: 95px !important; + } + + .layuimini-pc-show{ + display: block; + } + .layuimini-mobile-show{ + display: none; + } + + /**菜单缩放*/ + .layuimini-mini .layuimini-menu-left .layui-nav-more,.layuimini-mini .layuimini-menu-left .layui-nav-child{ + display: none!important; + } + +} + +/** + 手机自适应样式 +*/ +@media screen and (max-width: 1024px) { + + .layuimini-pc-show{ + display: none; + } + .layuimini-mobile-show{ + display: block; + } + + .layuimini-header-content { + left: 0; + } + + .layui-layout-admin .layui-body .layui-tab-item.layui-show { + border-top: 1px solid #e2e2e2; + } + + .layuimini-all .layui-layout-left.layuimini-header-menu { + left: 15px !important + } + + .layuimini-mini .layui-layout-left.layuimini-header-menu { + left: 205px !important + } + + .layui-layout-admin .layui-nav.layui-layout-right > li:not(.layuimini-setting) { + width: 40px !important; + } + + .layui-layout-admin .layui-nav.layui-layout-right > li:not(.layuimini-setting) a { + padding: 0 15px; + } + + .layuimini-all .layui-layout-admin .layui-body { + left: 0px !important; + } + + .layuimini-mini .layui-layout-admin .layuimini-menu-left, .layuimini-mini .layui-header .layuimini-logo { + left: 0; + transition: left .2s; + z-index: 1001 !important; + } + + .layuimini-all .layui-layout-admin .layuimini-menu-left, .layuimini-all .layui-header .layuimini-logo { + left: -200px; + transition: left .2s; + top: 0; + z-index: 1002; + } + + .layuimini-mini .layui-layout-admin .layui-body { + left: 0!important; + transition: left .2s; + top: 0; + z-index: 998; + } + + .layuimini-mini .layuimini-make { + display: block; + } + + .layuimini-multi-module .layuimini-header-content .layuimini-tool { + display: none; + } + + .layuimini-single-module .layuimini-header-content .layuimini-tool { + left: 15px; + } + + .layuimini-mini .layuimini-site-mobile { + display: none !important; + } + + .layuimini-site-mobile { + display: block !important; + position: fixed; + z-index: 100000; + bottom: 15px; + left: 15px; + width: 40px; + height: 40px; + line-height: 40px; + border-radius: 2px; + text-align: center; + background-color: rgba(0, 0, 0, .7); + color: #fff; + } + + .layuimini-header-content { + z-index: 997; + } + + .layuimini-content-page { + -webkit-overflow-scrolling: touch; + } + + /*修复UC之类的浏览器点击无效*/ + .layuimini-make { + cursor: pointer; + } + + .layuimini-site-mobile { + cursor: pointer; + } + +} + +@media screen and (max-width: 550px){ + + /**头部右侧数据*/ + .layuimini-multi-module.layuimini-mini .layuimini-header-content .layui-layout-right { + display: none; + } +} diff --git a/godb/web/statics/css/public.css b/godb/web/statics/css/public.css new file mode 100644 index 0000000..6fbdf17 --- /dev/null +++ b/godb/web/statics/css/public.css @@ -0,0 +1,13 @@ +.layuimini-content-page{background-color:#f2f2f2!important;} +.layuimini-container {border:1px solid #f2f2f2;border-radius:5px;background-color:#f2f2f2} +.layuimini-main {margin:10px 10px 10px 10px;border:5px solid #ffffff;border-radius:5px;background-color:#ffffff} +.layui-breadcrumb>* {font-size: 13px;!important;} + +/**必填红点 */ +.layuimini-form>.layui-form-item>.required:after {content:'*';color:red;position:absolute;margin-left:4px;font-weight:bold;line-height:1.8em;top:6px;right:5px;} +.layuimini-form>.layui-form-item>.layui-form-label {width:120px !important;} +.layuimini-form>.layui-form-item>.layui-input-block {margin-left:150px !important;} +.layuimini-form>.layui-form-item>.layui-input-block >tip {display:inline-block;margin-top:10px;line-height:10px;font-size:10px;color:#a29c9c;} + +/**搜索框*/ +.layuimini-container .table-search-fieldset {margin: 0;border: 1px solid #e6e6e6;padding: 10px 20px 5px 20px;color: #6b6b6b;} diff --git a/godb/web/statics/css/themes/default.css b/godb/web/statics/css/themes/default.css new file mode 100644 index 0000000..4f3cefb --- /dev/null +++ b/godb/web/statics/css/themes/default.css @@ -0,0 +1,95 @@ +/*头部右侧背景色 headerRightBg */ +.layui-layout-admin .layui-header { + background-color: #ffffff !important; +} + +/*头部右侧选中背景色 headerRightBgThis */ +.layui-layout-admin .layui-header .layuimini-header-content > ul > .layui-nav-item.layui-this, .layuimini-tool i:hover { + background-color: #e4e4e4 !important; +} + +/*头部右侧字体颜色 headerRightColor */ +.layui-layout-admin .layui-header .layui-nav .layui-nav-item a { + color: rgba(107, 107, 107, 0.7); +} + +/**头部右侧下拉字体颜色 headerRightChildColor */ +.layui-layout-admin .layui-header .layui-nav .layui-nav-item .layui-nav-child a { + color: rgba(107, 107, 107, 0.7) !important; +} + +/*头部右侧鼠标选中 headerRightColorThis */ +.layui-header .layuimini-menu-header-pc.layui-nav .layui-nav-item a:hover, .layui-header .layuimini-header-menu.layuimini-pc-show.layui-nav .layui-this a { + color: #565656 !important; +} + +/*头部右侧更多下拉颜色 headerRightNavMore */ +.layui-header .layui-nav .layui-nav-more { + border-top-color: rgba(160, 160, 160, 0.7) !important; +} + +/*头部右侧更多下拉颜色 headerRightNavMore */ +.layui-header .layui-nav .layui-nav-mored, .layui-header .layui-nav-itemed > a .layui-nav-more { + border-color: transparent transparent rgba(160, 160, 160, 0.7) !important; +} + +/**头部右侧更多下拉配置色 headerRightNavMoreBg headerRightNavMoreColor */ +.layui-header .layui-nav .layui-nav-child dd.layui-this a, .layui-header .layui-nav-child dd.layui-this, .layui-layout-admin .layui-header .layui-nav .layui-nav-item .layui-nav-child .layui-this a { + background-color: #1E9FFF !important; + color: #ffffff !important; +} + +/*头部缩放按钮样式 headerRightToolColor */ +.layui-layout-admin .layui-header .layuimini-tool i { + color: #565656; +} + +/*logo背景颜色 headerLogoBg */ +.layui-layout-admin .layuimini-logo { + background-color: #192027 !important; +} + +/*logo字体颜色 headerLogoColor */ +.layui-layout-admin .layuimini-logo h1 { + color: rgb(191, 187, 187); +} + +/*左侧菜单更多下拉样式 leftMenuNavMore */ +.layuimini-menu-left .layui-nav .layui-nav-more, .layuimini-menu-left-zoom.layui-nav .layui-nav-more { + border-top-color: rgb(191, 187, 187); +} + +/*左侧菜单更多下拉样式 leftMenuNavMore */ +.layuimini-menu-left .layui-nav .layui-nav-mored, .layuimini-menu-left .layui-nav-itemed > a .layui-nav-more, .layuimini-menu-left-zoom.layui-nav .layui-nav-mored, .layuimini-menu-left-zoom.layui-nav-itemed > a .layui-nav-more { + border-color: transparent transparent rgb(191, 187, 187) !important; +} + +/*左侧菜单背景 leftMenuBg */ +.layui-side.layui-bg-black, .layui-side.layui-bg-black > .layuimini-menu-left > ul, .layuimini-menu-left-zoom > ul { + background-color: #28333E !important; +} + +/*左侧菜单选中背景 leftMenuBgThis */ +.layuimini-menu-left .layui-nav-tree .layui-this, .layuimini-menu-left .layui-nav-tree .layui-this > a, .layuimini-menu-left .layui-nav-tree .layui-nav-child dd.layui-this, .layuimini-menu-left .layui-nav-tree .layui-nav-child dd.layui-this a, .layuimini-menu-left-zoom.layui-nav-tree .layui-this, .layuimini-menu-left-zoom.layui-nav-tree .layui-this > a, .layuimini-menu-left-zoom.layui-nav-tree .layui-nav-child dd.layui-this, .layuimini-menu-left-zoom.layui-nav-tree .layui-nav-child dd.layui-this a { + background-color: #1E9FFF !important +} + +/*左侧菜单子菜单背景 leftMenuChildBg */ +.layuimini-menu-left .layui-nav-itemed > .layui-nav-child { + background-color: #0c0f13 !important; +} + +/*左侧菜单字体颜色 leftMenuColor */ +.layuimini-menu-left .layui-nav .layui-nav-item a, .layuimini-menu-left-zoom.layui-nav .layui-nav-item a { + color: rgb(191, 187, 187) !important; +} + +/*左侧菜单选中字体颜色 leftMenuColorThis */ +.layuimini-menu-left .layui-nav .layui-nav-item a:hover, .layuimini-menu-left .layui-nav .layui-this a, .layuimini-menu-left-zoom.layui-nav .layui-nav-item a:hover, .layuimini-menu-left-zoom.layui-nav .layui-this a { + color: #ffffff !important; +} + +/**tab选项卡选中颜色 tabActiveColor */ +.layuimini-tab .layui-tab-title .layui-this .layuimini-tab-active { + background-color: #1e9fff; +} diff --git a/godb/web/statics/images/bg.jpg b/godb/web/statics/images/bg.jpg new file mode 100644 index 0000000..d5870f5 Binary files /dev/null and b/godb/web/statics/images/bg.jpg differ diff --git a/godb/web/statics/images/captcha.jpg b/godb/web/statics/images/captcha.jpg new file mode 100644 index 0000000..92b2a33 Binary files /dev/null and b/godb/web/statics/images/captcha.jpg differ diff --git a/godb/web/statics/images/donate_qrcode.png b/godb/web/statics/images/donate_qrcode.png new file mode 100644 index 0000000..079ac3d Binary files /dev/null and b/godb/web/statics/images/donate_qrcode.png differ diff --git a/godb/web/statics/images/favicon.ico b/godb/web/statics/images/favicon.ico new file mode 100644 index 0000000..950f726 Binary files /dev/null and b/godb/web/statics/images/favicon.ico differ diff --git a/godb/web/statics/images/home.png b/godb/web/statics/images/home.png new file mode 100644 index 0000000..a26efa4 Binary files /dev/null and b/godb/web/statics/images/home.png differ diff --git a/godb/web/statics/images/icon-login.png b/godb/web/statics/images/icon-login.png new file mode 100644 index 0000000..1db2f96 Binary files /dev/null and b/godb/web/statics/images/icon-login.png differ diff --git a/godb/web/statics/images/loginbg.png b/godb/web/statics/images/loginbg.png new file mode 100644 index 0000000..675c74b Binary files /dev/null and b/godb/web/statics/images/loginbg.png differ diff --git a/godb/web/statics/images/logo.png b/godb/web/statics/images/logo.png new file mode 100644 index 0000000..c3e107d Binary files /dev/null and b/godb/web/statics/images/logo.png differ diff --git a/godb/web/statics/index.html b/godb/web/statics/index.html new file mode 100644 index 0000000..1db6611 --- /dev/null +++ b/godb/web/statics/index.html @@ -0,0 +1,68 @@ + + + + +