453530270@qq.com
2 years ago
294 changed files with 39529 additions and 0 deletions
@ -0,0 +1,25 @@ |
|||
FROM golang:1.14 as compiler |
|||
|
|||
ENV GO111MODULE=on \ |
|||
GOPROXY=https://goproxy.cn,direct |
|||
|
|||
WORKDIR /app |
|||
|
|||
COPY . . |
|||
|
|||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o transfer . |
|||
|
|||
RUN mkdir publish && cp transfer publish && \ |
|||
cp app.yml publish && cp -r web/statics publish |
|||
|
|||
# 第二阶段 |
|||
FROM alpine |
|||
|
|||
WORKDIR /app |
|||
|
|||
COPY --from=compiler /app/publish . |
|||
|
|||
# 注意修改端口 |
|||
EXPOSE 8060 |
|||
|
|||
ENTRYPOINT ["./transfer"] |
|||
@ -0,0 +1,201 @@ |
|||
Apache License |
|||
Version 2.0, January 2004 |
|||
http://www.apache.org/licenses/ |
|||
|
|||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
|||
|
|||
1. Definitions. |
|||
|
|||
"License" shall mean the terms and conditions for use, reproduction, |
|||
and distribution as defined by Sections 1 through 9 of this document. |
|||
|
|||
"Licensor" shall mean the copyright owner or entity authorized by |
|||
the copyright owner that is granting the License. |
|||
|
|||
"Legal Entity" shall mean the union of the acting entity and all |
|||
other entities that control, are controlled by, or are under common |
|||
control with that entity. For the purposes of this definition, |
|||
"control" means (i) the power, direct or indirect, to cause the |
|||
direction or management of such entity, whether by contract or |
|||
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
|||
outstanding shares, or (iii) beneficial ownership of such entity. |
|||
|
|||
"You" (or "Your") shall mean an individual or Legal Entity |
|||
exercising permissions granted by this License. |
|||
|
|||
"Source" form shall mean the preferred form for making modifications, |
|||
including but not limited to software source code, documentation |
|||
source, and configuration files. |
|||
|
|||
"Object" form shall mean any form resulting from mechanical |
|||
transformation or translation of a Source form, including but |
|||
not limited to compiled object code, generated documentation, |
|||
and conversions to other media types. |
|||
|
|||
"Work" shall mean the work of authorship, whether in Source or |
|||
Object form, made available under the License, as indicated by a |
|||
copyright notice that is included in or attached to the work |
|||
(an example is provided in the Appendix below). |
|||
|
|||
"Derivative Works" shall mean any work, whether in Source or Object |
|||
form, that is based on (or derived from) the Work and for which the |
|||
editorial revisions, annotations, elaborations, or other modifications |
|||
represent, as a whole, an original work of authorship. For the purposes |
|||
of this License, Derivative Works shall not include works that remain |
|||
separable from, or merely link (or bind by name) to the interfaces of, |
|||
the Work and Derivative Works thereof. |
|||
|
|||
"Contribution" shall mean any work of authorship, including |
|||
the original version of the Work and any modifications or additions |
|||
to that Work or Derivative Works thereof, that is intentionally |
|||
submitted to Licensor for inclusion in the Work by the copyright owner |
|||
or by an individual or Legal Entity authorized to submit on behalf of |
|||
the copyright owner. For the purposes of this definition, "submitted" |
|||
means any form of electronic, verbal, or written communication sent |
|||
to the Licensor or its representatives, including but not limited to |
|||
communication on electronic mailing lists, source code control systems, |
|||
and issue tracking systems that are managed by, or on behalf of, the |
|||
Licensor for the purpose of discussing and improving the Work, but |
|||
excluding communication that is conspicuously marked or otherwise |
|||
designated in writing by the copyright owner as "Not a Contribution." |
|||
|
|||
"Contributor" shall mean Licensor and any individual or Legal Entity |
|||
on behalf of whom a Contribution has been received by Licensor and |
|||
subsequently incorporated within the Work. |
|||
|
|||
2. Grant of Copyright License. Subject to the terms and conditions of |
|||
this License, each Contributor hereby grants to You a perpetual, |
|||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
|||
copyright license to reproduce, prepare Derivative Works of, |
|||
publicly display, publicly perform, sublicense, and distribute the |
|||
Work and such Derivative Works in Source or Object form. |
|||
|
|||
3. Grant of Patent License. Subject to the terms and conditions of |
|||
this License, each Contributor hereby grants to You a perpetual, |
|||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
|||
(except as stated in this section) patent license to make, have made, |
|||
use, offer to sell, sell, import, and otherwise transfer the Work, |
|||
where such license applies only to those patent claims licensable |
|||
by such Contributor that are necessarily infringed by their |
|||
Contribution(s) alone or by combination of their Contribution(s) |
|||
with the Work to which such Contribution(s) was submitted. If You |
|||
institute patent litigation against any entity (including a |
|||
cross-claim or counterclaim in a lawsuit) alleging that the Work |
|||
or a Contribution incorporated within the Work constitutes direct |
|||
or contributory patent infringement, then any patent licenses |
|||
granted to You under this License for that Work shall terminate |
|||
as of the date such litigation is filed. |
|||
|
|||
4. Redistribution. You may reproduce and distribute copies of the |
|||
Work or Derivative Works thereof in any medium, with or without |
|||
modifications, and in Source or Object form, provided that You |
|||
meet the following conditions: |
|||
|
|||
(a) You must give any other recipients of the Work or |
|||
Derivative Works a copy of this License; and |
|||
|
|||
(b) You must cause any modified files to carry prominent notices |
|||
stating that You changed the files; and |
|||
|
|||
(c) You must retain, in the Source form of any Derivative Works |
|||
that You distribute, all copyright, patent, trademark, and |
|||
attribution notices from the Source form of the Work, |
|||
excluding those notices that do not pertain to any part of |
|||
the Derivative Works; and |
|||
|
|||
(d) If the Work includes a "NOTICE" text file as part of its |
|||
distribution, then any Derivative Works that You distribute must |
|||
include a readable copy of the attribution notices contained |
|||
within such NOTICE file, excluding those notices that do not |
|||
pertain to any part of the Derivative Works, in at least one |
|||
of the following places: within a NOTICE text file distributed |
|||
as part of the Derivative Works; within the Source form or |
|||
documentation, if provided along with the Derivative Works; or, |
|||
within a display generated by the Derivative Works, if and |
|||
wherever such third-party notices normally appear. The contents |
|||
of the NOTICE file are for informational purposes only and |
|||
do not modify the License. You may add Your own attribution |
|||
notices within Derivative Works that You distribute, alongside |
|||
or as an addendum to the NOTICE text from the Work, provided |
|||
that such additional attribution notices cannot be construed |
|||
as modifying the License. |
|||
|
|||
You may add Your own copyright statement to Your modifications and |
|||
may provide additional or different license terms and conditions |
|||
for use, reproduction, or distribution of Your modifications, or |
|||
for any such Derivative Works as a whole, provided Your use, |
|||
reproduction, and distribution of the Work otherwise complies with |
|||
the conditions stated in this License. |
|||
|
|||
5. Submission of Contributions. Unless You explicitly state otherwise, |
|||
any Contribution intentionally submitted for inclusion in the Work |
|||
by You to the Licensor shall be under the terms and conditions of |
|||
this License, without any additional terms or conditions. |
|||
Notwithstanding the above, nothing herein shall supersede or modify |
|||
the terms of any separate license agreement you may have executed |
|||
with Licensor regarding such Contributions. |
|||
|
|||
6. Trademarks. This License does not grant permission to use the trade |
|||
names, trademarks, service marks, or product names of the Licensor, |
|||
except as required for reasonable and customary use in describing the |
|||
origin of the Work and reproducing the content of the NOTICE file. |
|||
|
|||
7. Disclaimer of Warranty. Unless required by applicable law or |
|||
agreed to in writing, Licensor provides the Work (and each |
|||
Contributor provides its Contributions) on an "AS IS" BASIS, |
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
|||
implied, including, without limitation, any warranties or conditions |
|||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
|||
PARTICULAR PURPOSE. You are solely responsible for determining the |
|||
appropriateness of using or redistributing the Work and assume any |
|||
risks associated with Your exercise of permissions under this License. |
|||
|
|||
8. Limitation of Liability. In no event and under no legal theory, |
|||
whether in tort (including negligence), contract, or otherwise, |
|||
unless required by applicable law (such as deliberate and grossly |
|||
negligent acts) or agreed to in writing, shall any Contributor be |
|||
liable to You for damages, including any direct, indirect, special, |
|||
incidental, or consequential damages of any character arising as a |
|||
result of this License or out of the use or inability to use the |
|||
Work (including but not limited to damages for loss of goodwill, |
|||
work stoppage, computer failure or malfunction, or any and all |
|||
other commercial damages or losses), even if such Contributor |
|||
has been advised of the possibility of such damages. |
|||
|
|||
9. Accepting Warranty or Additional Liability. While redistributing |
|||
the Work or Derivative Works thereof, You may choose to offer, |
|||
and charge a fee for, acceptance of support, warranty, indemnity, |
|||
or other liability obligations and/or rights consistent with this |
|||
License. However, in accepting such obligations, You may act only |
|||
on Your own behalf and on Your sole responsibility, not on behalf |
|||
of any other Contributor, and only if You agree to indemnify, |
|||
defend, and hold each Contributor harmless for any liability |
|||
incurred by, or claims asserted against, such Contributor by reason |
|||
of your accepting any such warranty or additional liability. |
|||
|
|||
END OF TERMS AND CONDITIONS |
|||
|
|||
APPENDIX: How to apply the Apache License to your work. |
|||
|
|||
To apply the Apache License to your work, attach the following |
|||
boilerplate notice, with the fields enclosed by brackets "[]" |
|||
replaced with your own identifying information. (Don't include |
|||
the brackets!) The text should be enclosed in the appropriate |
|||
comment syntax for the file format. We also recommend that a |
|||
file or class name and description of purpose be included on the |
|||
same "printed page" as the copyright notice for easier |
|||
identification within third-party archives. |
|||
|
|||
Copyright [yyyy] [name of copyright owner] |
|||
|
|||
Licensed under the Apache License, Version 2.0 (the "License"); |
|||
you may not use this file except in compliance with the License. |
|||
You may obtain a copy of the License at |
|||
|
|||
http://www.apache.org/licenses/LICENSE-2.0 |
|||
|
|||
Unless required by applicable law or agreed to in writing, software |
|||
distributed under the License is distributed on an "AS IS" BASIS, |
|||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
See the License for the specific language governing permissions and |
|||
limitations under the License. |
|||
@ -0,0 +1,204 @@ |
|||
[](https://www.apache.org/licenses/LICENSE-2.0.html) |
|||
|
|||
# 简介 |
|||
|
|||
go-mysql-transfer是一款MySQL数据库实时增量同步工具。 |
|||
|
|||
能够监听MySQL二进制日志(Binlog)的变动，将变更内容形成指定格式的消息，实时发送到接收端。从而在数据库和接收端之间形成一个高性能、低延迟的增量数据同步更新管道。 |
|||
|
|||
# 特性 |
|||
|
|||
1、简单，不依赖其它组件，一键部署 |
|||
|
|||
2、集成多种接收端，如：Redis、MongoDB、Elasticsearch、RocketMQ、Kafka、RabbitMQ、HTTP API等，无需编写客户端，开箱即用 |
|||
|
|||
3、内置丰富的数据解析、消息生成规则、模板语法 |
|||
|
|||
4、支持Lua脚本扩展，可处理复杂逻辑 |
|||
|
|||
5、集成Prometheus客户端，支持监控告警 |
|||
|
|||
6、集成Web Admin监控页面 |
|||
|
|||
7、支持高可用集群部署 |
|||
|
|||
8、数据同步失败重试 |
|||
|
|||
9、支持全量数据初始化 |
|||
|
|||
|
|||
# 原理 |
|||
|
|||
1、将自己伪装为MySQL的Slave监听binlog，获取binlog的变更数据 |
|||
|
|||
2、根据规则或者lua脚本解析数据，生成指定格式的消息 |
|||
|
|||
3、将生成的消息批量发送给接收端 |
|||
|
|||
# 与同类工具比较 |
|||
|
|||
<table> |
|||
<thead> |
|||
<tr> |
|||
<th width="20%">特色</th> |
|||
<th width="20%">Canal</th> |
|||
<th width="20%">mysql_stream</th> |
|||
<th width="40%">go-mysql-transfer</th> |
|||
</tr> |
|||
</thead> |
|||
<tbody> |
|||
<tr> |
|||
<td>开发语言</td> |
|||
<td>Java</td> |
|||
<td>Python</td> |
|||
<td>Golang</td> |
|||
</tr> |
|||
<tr> |
|||
<td>高可用</td> |
|||
<td>支持</td> |
|||
<td>支持</td> |
|||
<td>支持</td> |
|||
</tr> |
|||
<tr> |
|||
<td>接收端</td> |
|||
<td>编码定制</td> |
|||
<td>Kafka等(MQ)</td> |
|||
<td>Redis、MongoDB、Elasticsearch、RabbitMQ、Kafka、RocketMQ、HTTP API <br>后续支持更多</td> |
|||
</tr> |
|||
<tr> |
|||
<td>全量数据初始化</td> |
|||
<td>不支持</td> |
|||
<td>支持</td> |
|||
<td>支持</td> |
|||
</tr> |
|||
<tr> |
|||
<td>数据格式</td> |
|||
<td>编码定制</td> |
|||
<td>Json（固定格式）</td> |
|||
<td>Json（规则配置)<br>模板语法<br>Lua脚本</td> |
|||
</tr> |
|||
</tbody> |
|||
</table> |
|||
|
|||
# 安装包 |
|||
|
|||
**二进制安装包** |
|||
|
|||
直接下载安装包: [点击下载](https://github.com/wj596/go-mysql-transfer/releases) |
|||
|
|||
**源码编译** |
|||
|
|||
1、依赖Golang 1.14 及以上版本 |
|||
|
|||
2、设置' GO111MODULE=on ' |
|||
|
|||
3、拉取源码 ' git clone https://github.com/wj596/go-mysql-transfer.git ' |
|||
|
|||
4、进入目录，执行 ' go build '编译 |
|||
|
|||
# 全量数据初始化 |
|||
|
|||
go-mysql-transfer -stock |
|||
|
|||
# 运行 |
|||
|
|||
**开启MySQL的binlog** |
|||
|
|||
``` |
|||
#Linux在my.cnf文件 |
|||
#Windows在my.ini文件 |
|||
log-bin=mysql-bin # 开启 binlog |
|||
binlog-format=ROW # 选择 ROW 模式 |
|||
server_id=1 # 配置 MySQL replaction 需要定义，不要和 go-mysql-transfer 的 slave_id 重复 |
|||
``` |
|||
|
|||
**命令行运行** |
|||
|
|||
1、修改app.yml |
|||
|
|||
2、Windows直接运行 go-mysql-transfer.exe |
|||
|
|||
3、Linux执行 nohup go-mysql-transfer & |
|||
|
|||
|
|||
# gitee |
|||
|
|||
如果您的github访问不稳定，可以在码云(gitee)上star项目：[go-mysql-transfer 码云(gitee)](https://gitee.com/wj596/go-mysql-transfer) |
|||
|
|||
# 使用说明 |
|||
|
|||
* [高可用集群](https://www.kancloud.cn/wj596/go-mysql-transfer/2116627) |
|||
* [同步数据到Redis](https://www.kancloud.cn/wj596/go-mysql-transfer/2064427) |
|||
* [Redis配置](https://www.kancloud.cn/wj596/go-mysql-transfer/2111996) |
|||
* [基于规则同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2111997) |
|||
* [基于Lua脚本同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2111998) |
|||
* [同步数据到MongoDB](https://www.kancloud.cn/wj596/go-mysql-transfer/2064428) |
|||
* [MongoDB配置](https://www.kancloud.cn/wj596/go-mysql-transfer/2111999) |
|||
* [基于规则同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112000) |
|||
* [基于Lua脚本同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112001) |
|||
* [同步数据到RocketMQ](https://www.kancloud.cn/wj596/go-mysql-transfer/2064429) |
|||
* [RocketMQ配置](https://www.kancloud.cn/wj596/go-mysql-transfer/2112002) |
|||
* [基于规则同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112003) |
|||
* [基于Lua脚本同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112004) |
|||
* [同步数据到Kafka](https://www.kancloud.cn/wj596/go-mysql-transfer/2064430) |
|||
* [Kafka配置](https://www.kancloud.cn/wj596/go-mysql-transfer/2112005) |
|||
* [基于规则同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112006) |
|||
* [基于Lua脚本同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112007) |
|||
* [同步数据到RabbitMQ](https://www.kancloud.cn/wj596/go-mysql-transfer/2064431) |
|||
* [RabbitMQ配置](https://www.kancloud.cn/wj596/go-mysql-transfer/2112008) |
|||
* [基于规则同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112009) |
|||
* [基于Lua脚本同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112010) |
|||
* [同步数据到Elasticsearch](https://www.kancloud.cn/wj596/go-mysql-transfer/2064432) |
|||
* [Elasticsearch配置](https://www.kancloud.cn/wj596/go-mysql-transfer/2112011) |
|||
* [基于规则同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112012) |
|||
* [基于Lua脚本同步](https://www.kancloud.cn/wj596/go-mysql-transfer/2112013) |
|||
* [全量数据导入](https://www.kancloud.cn/wj596/go-mysql-transfer/2116628) |
|||
* [Lua脚本](https://www.kancloud.cn/wj596/go-mysql-transfer/2064433) |
|||
* [基础模块](https://www.kancloud.cn/wj596/go-mysql-transfer/2112014) |
|||
* [Json模块](https://www.kancloud.cn/wj596/go-mysql-transfer/2112015) |
|||
* [HttpClient模块](https://www.kancloud.cn/wj596/go-mysql-transfer/2112016) |
|||
* [DBClient模块](https://www.kancloud.cn/wj596/go-mysql-transfer/2112017) |
|||
* [监控](https://www.kancloud.cn/wj596/go-mysql-transfer/2064434) |
|||
* [性能测试](https://www.kancloud.cn/wj596/go-mysql-transfer/2116629) |
|||
* [常见问题](https://www.kancloud.cn/wj596/go-mysql-transfer/2064435) |
|||
|
|||
|
|||
# 感谢 |
|||
|
|||
* [go-mysql](github.com/siddontang/go-mysql) |
|||
|
|||
* [go-mysql-elasticsearch](https://github.com/siddontang/go-mysql-elasticsearch) |
|||
|
|||
* [go-redis](https://github.com/go-redis/redis) |
|||
|
|||
* [rocketmq-client-go](https://github.com/apache/rocketmq-client-go) |
|||
|
|||
|
|||
|
|||
|
|||
# 更新日志 |
|||
|
|||
**v1.0.0 bate** |
|||
|
|||
* 9.17 初始化提交bate版本 |
|||
|
|||
**v1.0.1 release** |
|||
|
|||
* 9.22 release |
|||
|
|||
**v1.0.2 release** |
|||
|
|||
* 添加dbOps（数据库操作）、httpOps（http操作）两个Lua模块 |
|||
* 修复enum类型字段出现的乱码问题 |
|||
* redis接收端增加*Sorted* Set数据类型支持 |
|||
* 修复了近来反馈的bug |
|||
|
|||
**v1.0.3 release** |
|||
|
|||
* 添加了Web Admin监控界面 |
|||
* 改进了全量数据同步的速度 |
|||
* 重构了失败重试机制 |
|||
* 功能优化，如：针对MongoDB添加UPSERT操作、针对消息队列添加了update原始数据保留，等等 |
|||
|
|||
**v1.0.4 release** |
|||
* 修复了 -position 命令，binlog 名称验证问题 |
|||
@ -0,0 +1,129 @@ |
|||
# mysql配置 |
|||
addr: 127.0.0.1:3306 |
|||
user: root |
|||
pass: root |
|||
charset : utf8 |
|||
slave_id: 1001 #slave ID |
|||
flavor: mysql #mysql or mariadb,默认mysql |
|||
|
|||
#系统相关配置 |
|||
#data_dir: D:\\transfer #应用产生的数据存放地址，包括日志、缓存数据等，默认当前运行目录下store文件夹 |
|||
#logger: |
|||
# level: info #日志级别；支持：debug|info|warn|error，默认info |
|||
|
|||
#maxprocs: 50 #并发协（线）程数量，默认为: CPU核数*2；一般情况下不需要设置此项 |
|||
#bulk_size: 1000 #每批处理数量，不写默认100，可以根据带宽、机器性能等调整;如果是全量数据初始化时redis建议设为1000，其他接收端酌情调大 |
|||
|
|||
#prometheus相关配置 |
|||
#enable_exporter: true #是否启用prometheus exporter，默认false |
|||
#exporter_addr: 9595 #prometheus exporter端口，默认9595 |
|||
|
|||
#web admin相关配置 |
|||
enable_web_admin: true #是否启用web admin，默认false |
|||
web_admin_port: 8060 #web监控端口,默认8060 |
|||
|
|||
#cluster: # 集群相关配置 |
|||
#name: myTransfer #集群名称，具有相同name的节点放入同一个集群 |
|||
#bind_ip: 127.0.0.1 # 绑定的IP,如果机器有多张网卡(包含虚拟网卡)会有多个IP，使用这个属性绑定一个 |
|||
#ZooKeeper地址，多个用逗号风格 |
|||
#zk_addrs: 192.168.1.10:2181,192.168.1.11:2182,192.168.1.12:2183 |
|||
#zk_authentication: 123456 #digest类型的访问秘钥，如：user:password，默认为空 |
|||
#etcd_addrs: 127.0.0.1:2379 #etcd连接地址，多个用逗号分隔 |
|||
#etcd_user: test #etcd用户名 |
|||
#etcd_password: 123456 #etcd密码 |
|||
|
|||
#目标类型 |
|||
target: redis # 支持redis、mongodb、elasticsearch、rocketmq、kafka、rabbitmq |
|||
|
|||
#redis连接配置 |
|||
redis_addrs: 127.0.0.1:6379 #redis地址，多个用逗号分隔 |
|||
#redis_group_type: cluster # 集群类型 sentinel或者cluster |
|||
#redis_master_name: mymaster # Master节点名称,如果group_type为sentinel则此项不能为空，为cluster此项无效 |
|||
#redis_pass: 123456 #redis密码 |
|||
#redis_database: 0 #redis数据库 0-16,默认0。如果group_type为cluster此项无效 |
|||
|
|||
#mongodb连接配置 |
|||
#mongodb_addrs: 127.0.0.1:27017 #mongodb连接地址，多个用逗号分隔 |
|||
#mongodb_username: #mongodb用户名，默认为空 |
|||
#mongodb_password: #mongodb密码，默认为空 |
|||
|
|||
#elasticsearch连接配置 |
|||
#es_addrs: 127.0.0.1:9200 #连接地址，多个用逗号分隔 |
|||
#es_version: 7 # Elasticsearch版本，支持6和7、默认为7 |
|||
#es_password: # 用户名 |
|||
#es_version: # 密码 |
|||
|
|||
#rocketmq连接配置 |
|||
#rocketmq_name_servers: 127.0.0.1:9876 #rocketmq命名服务地址，多个用逗号分隔 |
|||
#rocketmq_group_name: transfer_test_group #rocketmq group name,默认为空 |
|||
#rocketmq_instance_name: transfer_test_group_ins #rocketmq instance name,默认为空 |
|||
#rocketmq_access_key: RocketMQ #访问控制 accessKey,默认为空 |
|||
#rocketmq_secret_key: 12345678 #访问控制 secretKey,默认为空 |
|||
|
|||
#kafka连接配置 |
|||
#kafka_addrs: 127.0.0.1:9092 #kafka连接地址，多个用逗号分隔 |
|||
#kafka_sasl_user: #kafka SASL_PLAINTEXT认证模式 用户名 |
|||
#kafka_sasl_password: #kafka SASL_PLAINTEXT认证模式 密码 |
|||
|
|||
#rabbitmq连接配置 |
|||
#rabbitmq_addr: amqp://guest:guest@127.0.0.1:5672/ #连接字符串,如: amqp://guest:guest@localhost:5672/ |
|||
|
|||
#规则配置 |
|||
rule: |
|||
- |
|||
schema: eseap #数据库名称 |
|||
table: t_user #表名称 |
|||
#order_by_column: id #排序字段，存量数据同步时不能为空 |
|||
#column_lower_case:false #列名称转为小写,默认为false |
|||
#column_upper_case:false#列名称转为大写,默认为false |
|||
column_underscore_to_camel: true #列名称下划线转驼峰,默认为false |
|||
# 包含的列，多值逗号分隔，如：id,name,age,area_id 为空时表示包含全部列 |
|||
#include_columns: ID,USER_NAME,PASSWORD |
|||
#exclude_columns: BIRTHDAY,MOBIE # 排除掉的列，多值逗号分隔，如：id,name,age,area_id 默认为空 |
|||
#column_mappings: USER_NAME=account #列名称映射，多个映射关系用逗号分隔，如：USER_NAME=account 表示将字段名USER_NAME映射为account |
|||
#default_column_values: area_name=合肥 #默认的列-值，多个用逗号分隔，如：source=binlog,area_name=合肥 |
|||
#date_formatter: yyyy-MM-dd #date类型格式化， 不填写默认yyyy-MM-dd |
|||
#datetime_formatter: yyyy-MM-dd HH:mm:ss #datetime、timestamp类型格式化，不填写默认yyyy-MM-dd HH:mm:ss |
|||
#lua_file_path: lua/t_user.lua #lua脚本文件 |
|||
#lua_script: #lua 脚本 |
|||
value_encoder: json #值编码，支持json、kv-commas、v-commas；默认为json |
|||
#value_formatter: '{{.ID}}|{{.USER_NAME}}' # 值格式化表达式，如：{{.ID}}|{{.USER_NAME}},{{.ID}}表示ID字段的值、{{.USER_NAME}}表示USER_NAME字段的值 |
|||
|
|||
#redis相关 |
|||
redis_structure: string # 数据类型。 支持string、hash、list、set、sortedset类型(与redis的数据类型一致) |
|||
#redis_key_prefix: USER_ #key的前缀 |
|||
#redis_key_column: USER_NAME #使用哪个列的值作为key，不填写默认使用主键 |
|||
#redis_key_formatter: '{{.ID}}|{{.USER_NAME}}' |
|||
#redis_key_value: user #KEY的值（固定值）；当redis_structure为hash、list、set、sortedset此值不能为空 |
|||
#redis_hash_field_prefix: _CARD_ #hash的field前缀，仅redis_structure为hash时起作用 |
|||
#redis_hash_field_column: Cert_No #使用哪个列的值作为hash的field，仅redis_structure为hash时起作用，不填写默认使用主键 |
|||
#redis_sorted_set_score_column: id #sortedset的score，当数据类型为sortedset时，此项不能为空，此项的值应为数字类型 |
|||
|
|||
#mongodb相关 |
|||
#mongodb_database: transfer #mongodb database不能为空 |
|||
#mongodb_collection: transfer_test_topic #mongodb collection，可以为空，默认使用表名称 |
|||
|
|||
#elasticsearch相关 |
|||
#es_index: user_index #Index名称,可以为空，默认使用表(Table)名称 |
|||
#es_mappings: #索引映射，可以为空，为空时根据数据类型自行推导ES推导 |
|||
# - |
|||
# column: REMARK #数据库列名称 |
|||
# field: remark #映射后的ES字段名称 |
|||
# type: text #ES字段类型 |
|||
# analyzer: ik_smart #ES分词器，type为text此项有意义 |
|||
# #format: #日期格式，type为date此项有意义 |
|||
# - |
|||
# column: USER_NAME #数据库列名称 |
|||
# field: account #映射后的ES字段名称 |
|||
# type: keyword #ES字段类型 |
|||
|
|||
#rocketmq相关 |
|||
#rocketmq_topic: transfer_test_topic #rocketmq topic，可以为空，默认使用表名称 |
|||
|
|||
#kafka相关 |
|||
#kafka_topic: user_topic #rocketmq topic，可以为空，默认使用表名称 |
|||
|
|||
#rabbitmq相关 |
|||
#rabbitmq_queue: user_topic #queue名称,可以为空，默认使用表(Table)名称 |
|||
|
|||
#reserve_raw_data: true #保留update之前的数据，针对rocketmq、kafka、rabbitmq有用;默认为false |
|||
@ -0,0 +1,557 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package global |
|||
|
|||
import ( |
|||
"fmt" |
|||
"io/ioutil" |
|||
"log" |
|||
"path/filepath" |
|||
"runtime" |
|||
"strings" |
|||
|
|||
"github.com/juju/errors" |
|||
"gopkg.in/yaml.v2" |
|||
|
|||
"go-mysql-transfer/util/files" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/nets" |
|||
"go-mysql-transfer/util/sys" |
|||
) |
|||
|
|||
const ( |
|||
_targetRedis = "REDIS" |
|||
_targetMongodb = "MONGODB" |
|||
_targetRocketmq = "ROCKETMQ" |
|||
_targetRabbitmq = "RABBITMQ" |
|||
_targetKafka = "KAFKA" |
|||
_targetElasticsearch = "ELASTICSEARCH" |
|||
_targetScript = "SCRIPT" |
|||
|
|||
RedisGroupTypeSentinel = "sentinel" |
|||
RedisGroupTypeCluster = "cluster" |
|||
|
|||
_dataDir = "store" |
|||
|
|||
_zkRootDir = "/transfer" // ZooKeeper and Etcd root
|
|||
|
|||
_flushBulkInterval = 200 |
|||
_flushBulkSize = 100 |
|||
|
|||
// update or insert
|
|||
UpsertAction = "upsert" |
|||
) |
|||
|
|||
var _config *Config |
|||
|
|||
type Config struct { |
|||
Target string `yaml:"target"` // 目标类型，支持redis、mongodb
|
|||
|
|||
Addr string `yaml:"addr"` |
|||
User string `yaml:"user"` |
|||
Password string `yaml:"pass"` |
|||
Charset string `yaml:"charset"` |
|||
|
|||
SlaveID uint32 `yaml:"slave_id"` |
|||
Flavor string `yaml:"flavor"` |
|||
DataDir string `yaml:"data_dir"` |
|||
|
|||
DumpExec string `yaml:"mysqldump"` |
|||
SkipMasterData bool `yaml:"skip_master_data"` |
|||
|
|||
Maxprocs int `yaml:"maxprocs"` // 最大协程数，默认CPU核心数*2
|
|||
BulkSize int64 `yaml:"bulk_size"` |
|||
|
|||
FlushBulkInterval int `yaml:"flush_bulk_interval"` |
|||
|
|||
SkipNoPkTable bool `yaml:"skip_no_pk_table"` |
|||
|
|||
RuleConfigs []*Rule `yaml:"rule"` |
|||
|
|||
LoggerConfig *logs.Config `yaml:"logger"` // 日志配置
|
|||
|
|||
EnableExporter bool `yaml:"enable_exporter"` // 启用prometheus exporter，默认false
|
|||
ExporterPort int `yaml:"exporter_addr"` // prometheus exporter端口
|
|||
|
|||
EnableWebAdmin bool `yaml:"enable_web_admin"` // 启用Web监控，默认false
|
|||
WebAdminPort int `yaml:"web_admin_port"` // web监控端口,默认8060
|
|||
|
|||
Cluster *Cluster `yaml:"cluster"` // 集群配置
|
|||
// ------------------- REDIS -----------------
|
|||
RedisAddr string `yaml:"redis_addrs"` //redis地址
|
|||
RedisGroupType string `yaml:"redis_group_type"` //集群类型 sentinel或者cluster
|
|||
RedisMasterName string `yaml:"redis_master_name"` //Master节点名称
|
|||
RedisPass string `yaml:"redis_pass"` //redis密码
|
|||
RedisDatabase int `yaml:"redis_database"` //redis数据库
|
|||
|
|||
// ------------------- ROCKETMQ -----------------
|
|||
RocketmqNameServers string `yaml:"rocketmq_name_servers"` //rocketmq命名服务地址，多个用逗号分隔
|
|||
RocketmqGroupName string `yaml:"rocketmq_group_name"` //rocketmq group name,默认为空
|
|||
RocketmqInstanceName string `yaml:"rocketmq_instance_name"` //rocketmq instance name,默认为空
|
|||
RocketmqAccessKey string `yaml:"rocketmq_access_key"` //访问控制 accessKey,默认为空
|
|||
RocketmqSecretKey string `yaml:"rocketmq_secret_key"` //访问控制 secretKey,默认为空
|
|||
|
|||
// ------------------- MONGODB -----------------
|
|||
MongodbAddr string `yaml:"mongodb_addrs"` //mongodb地址，多个用逗号分隔
|
|||
MongodbUsername string `yaml:"mongodb_username"` //mongodb用户名，默认为空
|
|||
MongodbPassword string `yaml:"mongodb_password"` //mongodb密码，默认为空
|
|||
|
|||
// ------------------- RABBITMQ -----------------
|
|||
RabbitmqAddr string `yaml:"rabbitmq_addr"` //连接字符串,如: amqp://guest:guest@localhost:5672/
|
|||
|
|||
// ------------------- KAFKA -----------------
|
|||
KafkaAddr string `yaml:"kafka_addrs"` //kafka连接地址，多个用逗号分隔
|
|||
KafkaSASLUser string `yaml:"kafka_sasl_user"` //kafka SASL_PLAINTEXT认证模式 用户名
|
|||
KafkaSASLPassword string `yaml:"kafka_sasl_password"` //kafka SASL_PLAINTEXT认证模式 密码
|
|||
|
|||
// ------------------- ES -----------------
|
|||
ElsAddr string `yaml:"es_addrs"` //Elasticsearch连接地址，多个用逗号分隔
|
|||
ElsUser string `yaml:"es_user"` //Elasticsearch用户名
|
|||
ElsPassword string `yaml:"es_password"` //Elasticsearch密码
|
|||
ElsVersion int `yaml:"es_version"` //Elasticsearch版本，支持6和7、默认为7
|
|||
|
|||
isReserveRawData bool //保留原始数据
|
|||
isMQ bool //是否消息队列
|
|||
} |
|||
|
|||
type Cluster struct { |
|||
Name string `yaml:"name"` |
|||
BindIp string `yaml:"bind_ip"` //绑定IP
|
|||
ZkAddrs string `yaml:"zk_addrs"` |
|||
ZkAuthentication string `yaml:"zk_authentication"` |
|||
EtcdAddrs string `yaml:"etcd_addrs"` |
|||
EtcdUser string `yaml:"etcd_user"` |
|||
EtcdPassword string `yaml:"etcd_password"` |
|||
} |
|||
|
|||
func initConfig(fileName string) error { |
|||
data, err := ioutil.ReadFile(fileName) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
var c Config |
|||
|
|||
if err := yaml.Unmarshal(data, &c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if err := checkConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if err := checkClusterConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
switch strings.ToUpper(c.Target) { |
|||
case _targetRedis: |
|||
if err := checkRedisConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
case _targetRocketmq: |
|||
if err := checkRocketmqConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
case _targetMongodb: |
|||
if err := checkMongodbConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
case _targetRabbitmq: |
|||
if err := checkRabbitmqConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
case _targetKafka: |
|||
if err := checkKafkaConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
case _targetElasticsearch: |
|||
if err := checkElsConfig(&c); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
case _targetScript: |
|||
|
|||
default: |
|||
return errors.Errorf("unsupported target: %s", c.Target) |
|||
} |
|||
|
|||
_config = &c |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func checkConfig(c *Config) error { |
|||
if c.Target == "" { |
|||
return errors.Errorf("empty target not allowed") |
|||
} |
|||
|
|||
if c.Addr == "" { |
|||
return errors.Errorf("empty addr not allowed") |
|||
} |
|||
|
|||
if c.User == "" { |
|||
return errors.Errorf("empty user not allowed") |
|||
} |
|||
|
|||
if c.Password == "" { |
|||
return errors.Errorf("empty pass not allowed") |
|||
} |
|||
|
|||
if c.Charset == "" { |
|||
return errors.Errorf("empty charset not allowed") |
|||
} |
|||
|
|||
if c.SlaveID == 0 { |
|||
return errors.Errorf("empty slave_id not allowed") |
|||
} |
|||
|
|||
if c.Flavor == "" { |
|||
c.Flavor = "mysql" |
|||
} |
|||
|
|||
if c.FlushBulkInterval == 0 { |
|||
c.FlushBulkInterval = _flushBulkInterval |
|||
} |
|||
|
|||
if c.BulkSize == 0 { |
|||
c.BulkSize = _flushBulkSize |
|||
} |
|||
|
|||
if c.DataDir == "" { |
|||
c.DataDir = filepath.Join(sys.CurrentDirectory(), _dataDir) |
|||
} |
|||
|
|||
if err := files.MkdirIfNecessary(c.DataDir); err != nil { |
|||
return err |
|||
} |
|||
|
|||
if c.LoggerConfig == nil { |
|||
c.LoggerConfig = &logs.Config{ |
|||
Store: filepath.Join(c.DataDir, "log"), |
|||
} |
|||
} |
|||
if c.LoggerConfig.Store == "" { |
|||
c.LoggerConfig.Store = filepath.Join(c.DataDir, "log") |
|||
} |
|||
|
|||
if err := files.MkdirIfNecessary(c.LoggerConfig.Store); err != nil { |
|||
return err |
|||
} |
|||
|
|||
if c.ExporterPort == 0 { |
|||
c.ExporterPort = 9595 |
|||
} |
|||
|
|||
if c.WebAdminPort == 0 { |
|||
c.WebAdminPort = 8060 |
|||
} |
|||
|
|||
if c.Maxprocs <= 0 { |
|||
c.Maxprocs = runtime.NumCPU() * 2 |
|||
} |
|||
|
|||
if c.RuleConfigs == nil { |
|||
return errors.Errorf("empty rules not allowed") |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func Cfg() *Config { |
|||
return _config |
|||
} |
|||
|
|||
func checkClusterConfig(c *Config) error { |
|||
if c.Cluster == nil { |
|||
return nil |
|||
} |
|||
|
|||
if c.Cluster.ZkAddrs == "" && c.Cluster.EtcdAddrs == "" { |
|||
return nil |
|||
} |
|||
|
|||
if c.Cluster.Name == "" { |
|||
return errors.Errorf("empty name not allowed in cluster") |
|||
} |
|||
|
|||
if c.Cluster.BindIp != "" && !nets.CheckIp(c.Cluster.BindIp) { |
|||
return errors.New("配置文件错误：配置项'bind_ip' 应为一个IP地址，可以为空") |
|||
} |
|||
if c.Cluster.BindIp == "" { |
|||
ips, err := nets.GetIpList() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if len(ips) > 1 { |
|||
return errors.New(fmt.Sprintf( |
|||
"检测到机器上存在多个IP地址：%v，无法确定向其他集群节点暴露那个IP。请在配置文件'bind_ip'配置项中指定", ips)) |
|||
} |
|||
c.Cluster.BindIp = ips[0] |
|||
} |
|||
|
|||
if c.IsZk() { |
|||
log.Println("cluster by Zookeeper") |
|||
} |
|||
if c.IsEtcd() { |
|||
log.Println("cluster by Etcd") |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func checkRedisConfig(c *Config) error { |
|||
if len(c.RedisAddr) == 0 { |
|||
return errors.Errorf("empty redis_addrs not allowed") |
|||
} |
|||
|
|||
addrList := strings.Split(c.RedisAddr, ",") |
|||
if len(addrList) > 1 { |
|||
if c.RedisGroupType == "" { |
|||
return errors.Errorf("empty group_type not allowed") |
|||
} |
|||
if c.RedisGroupType == RedisGroupTypeSentinel && c.RedisMasterName == "" { |
|||
return errors.Errorf("empty master_name not allowed") |
|||
} |
|||
} |
|||
|
|||
c.isReserveRawData = true |
|||
return nil |
|||
} |
|||
|
|||
func checkRocketmqConfig(c *Config) error { |
|||
if len(c.RocketmqNameServers) == 0 { |
|||
return errors.Errorf("empty rocketmq_name_servers not allowed") |
|||
} |
|||
|
|||
c.isReserveRawData = true |
|||
c.isMQ = true |
|||
return nil |
|||
} |
|||
|
|||
func checkMongodbConfig(c *Config) error { |
|||
if len(c.MongodbAddr) == 0 { |
|||
return errors.Errorf("empty mongodb_addrs not allowed") |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func checkRabbitmqConfig(c *Config) error { |
|||
if len(c.RabbitmqAddr) == 0 { |
|||
return errors.Errorf("empty rabbitmq_addr not allowed") |
|||
} |
|||
|
|||
c.isReserveRawData = true |
|||
c.isMQ = true |
|||
return nil |
|||
} |
|||
|
|||
func checkKafkaConfig(c *Config) error { |
|||
if len(c.KafkaAddr) == 0 { |
|||
return errors.Errorf("empty kafka_addrs not allowed") |
|||
} |
|||
|
|||
c.isReserveRawData = true |
|||
c.isMQ = true |
|||
return nil |
|||
} |
|||
|
|||
func checkElsConfig(c *Config) error { |
|||
if len(c.ElsAddr) == 0 { |
|||
return errors.Errorf("empty es_addrs not allowed") |
|||
} |
|||
|
|||
if !strings.HasPrefix(c.ElsAddr, "http") { |
|||
c.ElsAddr = "http://" + c.ElsAddr |
|||
} |
|||
|
|||
if c.ElsVersion == 0 { |
|||
c.ElsVersion = 7 |
|||
} |
|||
|
|||
if !(c.ElsVersion == 6 || c.ElsVersion == 7) { |
|||
return errors.Errorf("elasticsearch version must 6 or 7") |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (c *Config) IsCluster() bool { |
|||
if !c.IsZk() && !c.IsEtcd() { |
|||
return false |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
func (c *Config) IsZk() bool { |
|||
if c.Cluster == nil { |
|||
return false |
|||
} |
|||
if c.Cluster.ZkAddrs == "" { |
|||
return false |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
func (c *Config) IsEtcd() bool { |
|||
if c.Cluster == nil { |
|||
return false |
|||
} |
|||
if c.Cluster.EtcdAddrs == "" { |
|||
return false |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
func (c *Config) IsRedis() bool { |
|||
return strings.ToUpper(c.Target) == _targetRedis |
|||
} |
|||
|
|||
func (c *Config) IsRocketmq() bool { |
|||
return strings.ToUpper(c.Target) == _targetRocketmq |
|||
} |
|||
|
|||
func (c *Config) IsMongodb() bool { |
|||
return strings.ToUpper(c.Target) == _targetMongodb |
|||
} |
|||
|
|||
func (c *Config) IsRabbitmq() bool { |
|||
return strings.ToUpper(c.Target) == _targetRabbitmq |
|||
} |
|||
|
|||
func (c *Config) IsKafka() bool { |
|||
return strings.ToUpper(c.Target) == _targetKafka |
|||
} |
|||
|
|||
func (c *Config) IsEls() bool { |
|||
return strings.ToUpper(c.Target) == _targetElasticsearch |
|||
} |
|||
|
|||
func (c *Config) IsScript() bool { |
|||
return strings.ToUpper(c.Target) == _targetScript |
|||
} |
|||
|
|||
func (c *Config) IsExporterEnable() bool { |
|||
return c.EnableExporter |
|||
} |
|||
|
|||
func (c *Config) IsReserveRawData() bool { |
|||
return c.isReserveRawData |
|||
} |
|||
|
|||
func (c *Config) IsMQ() bool { |
|||
return c.isMQ |
|||
} |
|||
|
|||
func (c *Config) Destination() string { |
|||
var des string |
|||
switch strings.ToUpper(c.Target) { |
|||
case _targetRedis: |
|||
des += "redis(" |
|||
des += c.RedisAddr |
|||
des += ")" |
|||
case _targetRocketmq: |
|||
des += "rocketmq(" |
|||
des += c.RocketmqNameServers |
|||
des += ")" |
|||
case _targetMongodb: |
|||
des += "mongodb(" |
|||
des += c.MongodbAddr |
|||
des += ")" |
|||
case _targetRabbitmq: |
|||
des += "rabbitmq(" |
|||
des += c.RabbitmqAddr |
|||
des += ")" |
|||
case _targetKafka: |
|||
des += "kafka(" |
|||
des += c.KafkaAddr |
|||
des += ")" |
|||
case _targetElasticsearch: |
|||
des += "elasticsearch(" |
|||
des += c.ElsAddr |
|||
des += ")" |
|||
case _targetScript: |
|||
des += "Lua Script" |
|||
} |
|||
return des |
|||
} |
|||
|
|||
func (c *Config) DestStdName() string { |
|||
switch strings.ToUpper(c.Target) { |
|||
case _targetRedis: |
|||
return "Redis" |
|||
case _targetRocketmq: |
|||
return "RocketMQ" |
|||
case _targetMongodb: |
|||
return "MongoDB" |
|||
case _targetRabbitmq: |
|||
return "RabbitMQ" |
|||
case _targetKafka: |
|||
return "Kafka" |
|||
case _targetElasticsearch: |
|||
return "Elasticsearch" |
|||
} |
|||
|
|||
return "" |
|||
} |
|||
|
|||
func (c *Config) DestAddr() string { |
|||
switch strings.ToUpper(c.Target) { |
|||
case _targetRedis: |
|||
return c.RedisAddr |
|||
case _targetRocketmq: |
|||
return c.RocketmqNameServers |
|||
case _targetMongodb: |
|||
return c.MongodbAddr |
|||
case _targetRabbitmq: |
|||
return c.RabbitmqAddr |
|||
case _targetKafka: |
|||
return c.KafkaAddr |
|||
case _targetElasticsearch: |
|||
return c.ElsAddr |
|||
} |
|||
|
|||
return "" |
|||
} |
|||
|
|||
func (c *Config) ZkRootDir() string { |
|||
return _zkRootDir |
|||
} |
|||
|
|||
func (c *Config) ZkClusterDir() string { |
|||
return _zkRootDir + "/" + c.Cluster.Name |
|||
} |
|||
|
|||
func (c *Config) ZkPositionDir() string { |
|||
return _zkRootDir + "/" + c.Cluster.Name + "/position" |
|||
} |
|||
|
|||
func (c *Config) ZkElectionDir() string { |
|||
return _zkRootDir + "/" + c.Cluster.Name + "/election" |
|||
} |
|||
|
|||
func (c *Config) ZkElectedDir() string { |
|||
return _zkRootDir + "/" + c.Cluster.Name + "/elected" |
|||
} |
|||
|
|||
func (c *Config) ZkNodesDir() string { |
|||
return _zkRootDir + "/" + c.Cluster.Name + "/nodes" |
|||
} |
|||
@ -0,0 +1,87 @@ |
|||
package global |
|||
|
|||
import ( |
|||
"fmt" |
|||
"log" |
|||
"runtime" |
|||
"strconv" |
|||
"syscall" |
|||
"time" |
|||
|
|||
sidlog "github.com/siddontang/go-log/log" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
var ( |
|||
_pid int |
|||
_coordinator int |
|||
_leaderFlag bool |
|||
_leaderNode string |
|||
_currentNode string |
|||
_bootTime time.Time |
|||
) |
|||
|
|||
func SetLeaderFlag(flag bool) { |
|||
_leaderFlag = flag |
|||
} |
|||
|
|||
func IsLeader() bool { |
|||
return _leaderFlag |
|||
} |
|||
|
|||
func SetLeaderNode(leader string) { |
|||
_leaderNode = leader |
|||
} |
|||
|
|||
func LeaderNode() string { |
|||
return _leaderNode |
|||
} |
|||
|
|||
func CurrentNode() string { |
|||
return _currentNode |
|||
} |
|||
|
|||
func IsFollower() bool { |
|||
return !_leaderFlag |
|||
} |
|||
|
|||
func BootTime() time.Time { |
|||
return _bootTime |
|||
} |
|||
|
|||
func Initialize(configPath string) error { |
|||
if err := initConfig(configPath); err != nil { |
|||
return err |
|||
} |
|||
runtime.GOMAXPROCS(_config.Maxprocs) |
|||
|
|||
// 初始化global logger
|
|||
if err := logs.Initialize(_config.LoggerConfig); err != nil { |
|||
return err |
|||
} |
|||
|
|||
streamHandler, err := sidlog.NewStreamHandler(logs.Writer()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
agent := sidlog.New(streamHandler, sidlog.Ltime|sidlog.Lfile|sidlog.Llevel) |
|||
sidlog.SetDefaultLogger(agent) |
|||
|
|||
_bootTime = time.Now() |
|||
_pid = syscall.Getpid() |
|||
|
|||
if _config.IsCluster(){ |
|||
if _config.EnableWebAdmin { |
|||
_currentNode = _config.Cluster.BindIp + ":" + strconv.Itoa(_config.WebAdminPort) |
|||
} else { |
|||
_currentNode = _config.Cluster.BindIp + ":" + strconv.Itoa(_pid) |
|||
} |
|||
} |
|||
|
|||
log.Println(fmt.Sprintf("process id: %d", _pid)) |
|||
log.Println(fmt.Sprintf("GOMAXPROCS :%d", _config.Maxprocs)) |
|||
log.Println(fmt.Sprintf("source %s(%s)", _config.Flavor, _config.Addr)) |
|||
log.Println(fmt.Sprintf("destination %s", _config.Destination())) |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,694 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package global |
|||
|
|||
import ( |
|||
"io/ioutil" |
|||
"path/filepath" |
|||
"strings" |
|||
"sync" |
|||
"text/template" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/schema" |
|||
"github.com/vmihailenco/msgpack" |
|||
"github.com/yuin/gopher-lua" |
|||
"github.com/yuin/gopher-lua/parse" |
|||
|
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/util/dates" |
|||
"go-mysql-transfer/util/files" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
const ( |
|||
RedisStructureString = "String" |
|||
RedisStructureHash = "Hash" |
|||
RedisStructureList = "List" |
|||
RedisStructureSet = "Set" |
|||
RedisStructureSortedSet = "SortedSet" |
|||
|
|||
ValEncoderJson = "json" |
|||
ValEncoderKVCommas = "kv-commas" |
|||
ValEncoderVCommas = "v-commas" |
|||
) |
|||
|
|||
var ( |
|||
_ruleInsMap = make(map[string]*Rule) |
|||
_lockOfRuleInsMap sync.RWMutex |
|||
) |
|||
|
|||
type EsMapping struct { |
|||
Column string `yaml:"column"` // 数据库列名称
|
|||
Field string `yaml:"field"` // 映射后的ES字段名称
|
|||
Type string `yaml:"type"` // ES字段类型
|
|||
Analyzer string `yaml:"analyzer"` // ES分词器
|
|||
Format string `yaml:"format"` // 日期格式
|
|||
} |
|||
|
|||
type Rule struct { |
|||
Schema string `yaml:"schema"` |
|||
Table string `yaml:"table"` |
|||
OrderByColumn string `yaml:"order_by_column"` |
|||
ColumnLowerCase bool `yaml:"column_lower_case"` // 列名称转为小写
|
|||
ColumnUpperCase bool `yaml:"column_upper_case"` // 列名称转为大写
|
|||
ColumnUnderscoreToCamel bool `yaml:"column_underscore_to_camel"` // 列名称下划线转驼峰
|
|||
IncludeColumnConfig string `yaml:"include_columns"` // 包含的列
|
|||
ExcludeColumnConfig string `yaml:"exclude_columns"` // 排除掉的列
|
|||
ColumnMappingConfigs string `yaml:"column_mappings"` // 列名称映射
|
|||
DefaultColumnValueConfig string `yaml:"default_column_values"` // 默认的字段和值
|
|||
// #值编码，支持json、kv-commas、v-commas；默认为json；json形如：{"id":123,"name":"wangjie"} 、kv-commas形如：id=123,name="wangjie"、v-commas形如：123,wangjie
|
|||
ValueEncoder string `yaml:"value_encoder"` |
|||
ValueFormatter string `yaml:"value_formatter"` //格式化定义key,{id}表示字段id的值、{name}表示字段name的值
|
|||
LuaScript string `yaml:"lua_script"` //lua 脚本
|
|||
LuaFilePath string `yaml:"lua_file_path"` //lua 文件地址
|
|||
DateFormatter string `yaml:"date_formatter"` //date类型格式化， 不填写默认2006-01-02
|
|||
DatetimeFormatter string `yaml:"datetime_formatter"` //datetime、timestamp类型格式化，不填写默认RFC3339(2006-01-02T15:04:05Z07:00)
|
|||
|
|||
ReserveRawData bool `yaml:"reserve_raw_data"` // 保留update之前的数据，针对KAFKA、RABBITMQ、ROCKETMQ有效
|
|||
|
|||
// ------------------- REDIS -----------------
|
|||
//对应redis的5种数据类型 String、Hash(字典) 、List(列表) 、Set(集合)、Sorted Set(有序集合)
|
|||
RedisStructure string `yaml:"redis_structure"` |
|||
RedisKeyPrefix string `yaml:"redis_key_prefix"` //key的前缀
|
|||
RedisKeyColumn string `yaml:"redis_key_column"` //使用哪个列的值作为key，不填写默认使用主键
|
|||
// 格式化定义key,如{id}-{name}；{id}表示字段id的值、{name}表示字段name的值
|
|||
RedisKeyFormatter string `yaml:"redis_key_formatter"` |
|||
RedisKeyValue string `yaml:"redis_key_value"` // key的值，固定值
|
|||
// hash的field前缀，仅redis_structure为hash时起作用
|
|||
RedisHashFieldPrefix string `yaml:"redis_hash_field_prefix"` |
|||
// 使用哪个列的值作为hash的field，仅redis_structure为hash时起作用
|
|||
RedisHashFieldColumn string `yaml:"redis_hash_field_column"` |
|||
// Sorted Set(有序集合)的Score
|
|||
RedisSortedSetScoreColumn string `yaml:"redis_sorted_set_score_column"` |
|||
RedisKeyColumnIndex int |
|||
RedisKeyColumnIndexs []int |
|||
RedisHashFieldColumnIndex int |
|||
RedisHashFieldColumnIndexs []int |
|||
RedisSortedSetScoreColumnIndex int |
|||
RedisKeyTmpl *template.Template |
|||
|
|||
// ------------------- ROCKETMQ -----------------
|
|||
RocketmqTopic string `yaml:"rocketmq_topic"` //rocketmq topic名称，可以为空，为空时使用表名称
|
|||
|
|||
// ------------------- MONGODB -----------------
|
|||
MongodbDatabase string `yaml:"mongodb_database"` //mongodb database 不能为空
|
|||
MongodbCollection string `yaml:"mongodb_collection"` //mongodb collection，可以为空，默认使用表(Table)名称
|
|||
|
|||
// ------------------- RABBITMQ -----------------
|
|||
RabbitmqQueue string `yaml:"rabbitmq_queue"` //queue名称,可以为空，默认使用表(Table)名称
|
|||
|
|||
// ------------------- KAFKA -----------------
|
|||
KafkaTopic string `yaml:"kafka_topic"` //TOPIC名称,可以为空，默认使用表(Table)名称
|
|||
|
|||
// ------------------- ES -----------------
|
|||
ElsIndex string `yaml:"es_index"` //Elasticsearch Index,可以为空，默认使用表(Table)名称
|
|||
ElsType string `yaml:"es_type"` //es6.x以后一个Index只能拥有一个Type,可以为空，默认使用_doc; es7.x版本此属性无效
|
|||
EsMappings []*EsMapping `yaml:"es_mappings"` //Elasticsearch mappings映射关系,可以为空，为空时根据数据类型自己推导
|
|||
|
|||
// --------------- no config ----------------
|
|||
TableInfo *schema.Table |
|||
TableColumnSize int |
|||
IsCompositeKey bool //是否联合主键
|
|||
DefaultColumnValueMap map[string]string |
|||
PaddingMap map[string]*model.Padding |
|||
LuaProto *lua.FunctionProto |
|||
LuaFunction *lua.LFunction |
|||
ValueTmpl *template.Template |
|||
} |
|||
|
|||
func RuleDeepClone(res *Rule) (*Rule, error) { |
|||
data, err := msgpack.Marshal(res) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
var r Rule |
|||
err = msgpack.Unmarshal(data, &r) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return &r, nil |
|||
} |
|||
|
|||
func RuleKey(schema string, table string) string { |
|||
return strings.ToLower(schema + ":" + table) |
|||
} |
|||
|
|||
func AddRuleIns(ruleKey string, r *Rule) { |
|||
_lockOfRuleInsMap.Lock() |
|||
defer _lockOfRuleInsMap.Unlock() |
|||
|
|||
_ruleInsMap[ruleKey] = r |
|||
} |
|||
|
|||
func RuleIns(ruleKey string) (*Rule, bool) { |
|||
_lockOfRuleInsMap.RLock() |
|||
defer _lockOfRuleInsMap.RUnlock() |
|||
|
|||
r, ok := _ruleInsMap[ruleKey] |
|||
|
|||
return r, ok |
|||
} |
|||
|
|||
func RuleInsExist(ruleKey string) bool { |
|||
_lockOfRuleInsMap.RLock() |
|||
defer _lockOfRuleInsMap.RUnlock() |
|||
|
|||
_, ok := _ruleInsMap[ruleKey] |
|||
|
|||
return ok |
|||
} |
|||
|
|||
func RuleInsTotal() int { |
|||
_lockOfRuleInsMap.RLock() |
|||
defer _lockOfRuleInsMap.RUnlock() |
|||
|
|||
return len(_ruleInsMap) |
|||
} |
|||
|
|||
func RuleInsList() []*Rule { |
|||
_lockOfRuleInsMap.RLock() |
|||
defer _lockOfRuleInsMap.RUnlock() |
|||
|
|||
list := make([]*Rule, 0, len(_ruleInsMap)) |
|||
for _, rule := range _ruleInsMap { |
|||
list = append(list, rule) |
|||
} |
|||
|
|||
return list |
|||
} |
|||
|
|||
func RuleKeyList() []string { |
|||
_lockOfRuleInsMap.RLock() |
|||
defer _lockOfRuleInsMap.RUnlock() |
|||
|
|||
list := make([]string, 0, len(_ruleInsMap)) |
|||
for k, _ := range _ruleInsMap { |
|||
list = append(list, k) |
|||
} |
|||
|
|||
return list |
|||
} |
|||
|
|||
func (s *Rule) Initialize() error { |
|||
if err := s.buildPaddingMap(); err != nil { |
|||
return err |
|||
} |
|||
|
|||
if s.ValueEncoder == "" { |
|||
s.ValueEncoder = ValEncoderJson |
|||
} |
|||
|
|||
if s.ValueFormatter != "" { |
|||
tmpl, err := template.New(s.TableInfo.Name).Parse(s.ValueFormatter) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
s.ValueTmpl = tmpl |
|||
s.ValueEncoder = "" |
|||
} |
|||
|
|||
if s.DefaultColumnValueConfig != "" { |
|||
dm := make(map[string]string) |
|||
for _, t := range strings.Split(s.DefaultColumnValueConfig, ",") { |
|||
tt := strings.Split(t, "=") |
|||
if len(tt) != 2 { |
|||
return errors.Errorf("default_field_value format error in rule") |
|||
} |
|||
field := tt[0] |
|||
value := tt[1] |
|||
dm[field] = value |
|||
} |
|||
s.DefaultColumnValueMap = dm |
|||
} |
|||
|
|||
if s.DateFormatter != "" { |
|||
s.DateFormatter = dates.ConvertGoFormat(s.DateFormatter) |
|||
} |
|||
|
|||
if s.DatetimeFormatter != "" { |
|||
s.DatetimeFormatter = dates.ConvertGoFormat(s.DatetimeFormatter) |
|||
} |
|||
|
|||
if _config.IsRedis() { |
|||
if err := s.initRedisConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsRocketmq() { |
|||
if err := s.initRocketConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsMongodb() { |
|||
if err := s.initMongoConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsRabbitmq() { |
|||
if err := s.initRabbitmqConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsKafka() { |
|||
if err := s.initKafkaConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsEls() { |
|||
if err := s.initElsConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsScript() { |
|||
if s.LuaScript == "" && s.LuaFilePath == "" { |
|||
return errors.New("empty lua script not allowed") |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) AfterUpdateTableInfo() error { |
|||
if err := s.buildPaddingMap(); err != nil { |
|||
return err |
|||
} |
|||
|
|||
if _config.IsRedis() { |
|||
if err := s.initRedisConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsRocketmq() { |
|||
if err := s.initRocketConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsMongodb() { |
|||
if err := s.initMongoConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsRabbitmq() { |
|||
if err := s.initRabbitmqConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsKafka() { |
|||
if err := s.initKafkaConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsEls() { |
|||
if err := s.initElsConfig(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if _config.IsScript() { |
|||
if s.LuaScript == "" || s.LuaFilePath == "" { |
|||
return errors.New("empty lua script not allowed") |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) buildPaddingMap() error { |
|||
paddingMap := make(map[string]*model.Padding) |
|||
mappings := make(map[string]string) |
|||
|
|||
if s.ColumnMappingConfigs != "" { |
|||
ls := strings.Split(s.ColumnMappingConfigs, ",") |
|||
for _, t := range ls { |
|||
cmc := strings.Split(t, "=") |
|||
if len(cmc) != 2 { |
|||
return errors.Errorf("column_mappings format error in rule") |
|||
} |
|||
column := cmc[0] |
|||
mapped := cmc[1] |
|||
_, index := s.TableColumn(column) |
|||
if index < 0 { |
|||
return errors.Errorf("column_mappings must be table column") |
|||
} |
|||
mappings[strings.ToUpper(column)] = mapped |
|||
} |
|||
} |
|||
|
|||
if len(s.EsMappings) > 0 { |
|||
for _, mapping := range s.EsMappings { |
|||
mappings[strings.ToUpper(mapping.Column)] = mapping.Field |
|||
} |
|||
} |
|||
|
|||
var includes []string |
|||
var excludes []string |
|||
|
|||
if s.IncludeColumnConfig != "" { |
|||
includes = strings.Split(s.IncludeColumnConfig, ",") |
|||
} |
|||
if s.ExcludeColumnConfig != "" { |
|||
excludes = strings.Split(s.ExcludeColumnConfig, ",") |
|||
} |
|||
|
|||
if len(includes) > 0 { |
|||
for _, c := range includes { |
|||
_, index := s.TableColumn(c) |
|||
if index < 0 { |
|||
return errors.New("include_field must be table column") |
|||
} |
|||
paddingMap[c] = s.newPadding(mappings, c) |
|||
} |
|||
} else { |
|||
for _, column := range s.TableInfo.Columns { |
|||
include := true |
|||
for _, exclude := range excludes { |
|||
if column.Name == exclude { |
|||
include = false |
|||
} |
|||
} |
|||
if include { |
|||
paddingMap[column.Name] = s.newPadding(mappings, column.Name) |
|||
} |
|||
} |
|||
} |
|||
|
|||
s.PaddingMap = paddingMap |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) newPadding(mappings map[string]string, columnName string) *model.Padding { |
|||
column, index := s.TableColumn(columnName) |
|||
|
|||
wrapName := s.WrapName(column.Name) |
|||
mapped, exist := mappings[strings.ToUpper(column.Name)] |
|||
if exist { |
|||
wrapName = mapped |
|||
} |
|||
|
|||
return &model.Padding{ |
|||
WrapName: wrapName, |
|||
|
|||
ColumnIndex: index, |
|||
ColumnName: column.Name, |
|||
ColumnType: column.Type, |
|||
ColumnMetadata: column, |
|||
} |
|||
} |
|||
|
|||
func (s *Rule) TableColumn(field string) (*schema.TableColumn, int) { |
|||
for index, c := range s.TableInfo.Columns { |
|||
if strings.ToUpper(c.Name) == strings.ToUpper(field) { |
|||
return &c, index |
|||
} |
|||
} |
|||
return nil, -1 |
|||
} |
|||
|
|||
func (s *Rule) WrapName(fieldName string) string { |
|||
if s.ColumnUnderscoreToCamel { |
|||
return stringutil.Case2Camel(strings.ToLower(fieldName)) |
|||
} |
|||
if s.ColumnLowerCase { |
|||
return strings.ToLower(fieldName) |
|||
} |
|||
if s.ColumnUpperCase { |
|||
return strings.ToUpper(fieldName) |
|||
} |
|||
return fieldName |
|||
} |
|||
|
|||
func (s *Rule) LuaEnable() bool { |
|||
if s.LuaScript == "" && s.LuaFilePath == "" { |
|||
return false |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
func (s *Rule) initRedisConfig() error { |
|||
if s.LuaEnable() { |
|||
return nil |
|||
} |
|||
|
|||
if s.RedisStructure == "" { |
|||
return errors.Errorf("empty redis_structure not allowed in rule") |
|||
} |
|||
|
|||
switch strings.ToUpper(s.RedisStructure) { |
|||
case "STRING": |
|||
s.RedisStructure = RedisStructureString |
|||
if s.RedisKeyColumn == "" && s.RedisKeyFormatter == "" { |
|||
if s.IsCompositeKey { |
|||
for _, v := range s.TableInfo.PKColumns { |
|||
s.RedisKeyColumnIndexs = append(s.RedisKeyColumnIndexs, v) |
|||
} |
|||
s.RedisKeyColumnIndex = -1 |
|||
} else { |
|||
s.RedisKeyColumnIndex = s.TableInfo.PKColumns[0] |
|||
} |
|||
} |
|||
case "HASH": |
|||
s.RedisStructure = RedisStructureHash |
|||
if s.RedisKeyValue == "" { |
|||
return errors.New("empty redis_key_value not allowed") |
|||
} |
|||
// init hash field
|
|||
if s.RedisHashFieldColumn == "" { |
|||
if s.IsCompositeKey { |
|||
for _, v := range s.TableInfo.PKColumns { |
|||
s.RedisHashFieldColumnIndexs = append(s.RedisHashFieldColumnIndexs, v) |
|||
} |
|||
s.RedisHashFieldColumnIndex = -1 |
|||
} else { |
|||
s.RedisHashFieldColumnIndex = s.TableInfo.PKColumns[0] |
|||
} |
|||
} else { |
|||
_, index := s.TableColumn(s.RedisHashFieldColumn) |
|||
if index < 0 { |
|||
return errors.New("redis_hash_field_column must be table column") |
|||
} |
|||
s.RedisHashFieldColumnIndex = index |
|||
} |
|||
case "LIST": |
|||
s.RedisStructure = RedisStructureList |
|||
if s.RedisKeyValue == "" { |
|||
return errors.New("empty redis_key_value not allowed in rule") |
|||
} |
|||
case "SET": |
|||
s.RedisStructure = RedisStructureSet |
|||
if s.RedisKeyValue == "" { |
|||
return errors.New("empty redis_key_value not allowed in rule") |
|||
} |
|||
case "SORTEDSET": |
|||
s.RedisStructure = RedisStructureSortedSet |
|||
if s.RedisKeyValue == "" { |
|||
return errors.New("empty redis_key_value not allowed in rule") |
|||
} |
|||
if s.RedisSortedSetScoreColumn == "" { |
|||
return errors.New("empty redis_sorted_set_score_column not allowed in rule") |
|||
} |
|||
_, index := s.TableColumn(s.RedisSortedSetScoreColumn) |
|||
if index < 0 { |
|||
return errors.New("redis_sorted_set_score_column must be table column") |
|||
} |
|||
s.RedisHashFieldColumnIndex = index |
|||
default: |
|||
return errors.Errorf("redis_structure must be string or hash or list or set") |
|||
} |
|||
|
|||
if s.RedisKeyColumn != "" { |
|||
_, index := s.TableColumn(s.RedisKeyColumn) |
|||
if index < 0 { |
|||
return errors.New("redis_key_column must be table column") |
|||
} |
|||
s.RedisKeyColumnIndex = index |
|||
s.RedisKeyFormatter = "" |
|||
} |
|||
|
|||
if s.RedisKeyFormatter != "" { |
|||
tmpl, err := template.New(s.TableInfo.Name).Parse(s.RedisKeyFormatter) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
s.RedisKeyTmpl = tmpl |
|||
s.RedisKeyColumnIndex = -1 |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) initRocketConfig() error { |
|||
if !s.LuaEnable() { |
|||
if s.RocketmqTopic == "" { |
|||
s.RocketmqTopic = s.Table |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) initMongoConfig() error { |
|||
if !s.LuaEnable() { |
|||
if s.MongodbDatabase == "" { |
|||
return errors.New("empty mongodb_database not allowed in rule") |
|||
} |
|||
|
|||
if s.MongodbCollection == "" { |
|||
s.MongodbCollection = s.Table |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) initRabbitmqConfig() error { |
|||
if !s.LuaEnable() { |
|||
if s.RabbitmqQueue == "" { |
|||
s.RabbitmqQueue = s.Table |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) initElsConfig() error { |
|||
if s.ElsIndex == "" { |
|||
s.ElsIndex = s.Table |
|||
} |
|||
|
|||
if s.ElsType == "" { |
|||
s.ElsType = "_doc" |
|||
} |
|||
|
|||
if len(s.EsMappings) > 0 { |
|||
for _, m := range s.EsMappings { |
|||
if m.Field == "" { |
|||
return errors.New("empty field not allowed in es_mappings") |
|||
} |
|||
if m.Type == "" { |
|||
return errors.New("empty type not allowed in es_mappings") |
|||
} |
|||
if m.Column == "" && !s.LuaEnable() { |
|||
return errors.New("empty column not allowed in es_mappings") |
|||
} |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Rule) initKafkaConfig() error { |
|||
if !s.LuaEnable() { |
|||
if s.KafkaTopic == "" { |
|||
s.KafkaTopic = s.Table |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// 编译Lua
|
|||
func (s *Rule) CompileLuaScript(dataDir string) error { |
|||
script := s.LuaScript |
|||
if s.LuaFilePath != "" { |
|||
var filePath string |
|||
if files.IsExist(s.LuaFilePath) { |
|||
filePath = s.LuaFilePath |
|||
} else { |
|||
filePath = filepath.Join(dataDir, s.LuaFilePath) |
|||
} |
|||
data, err := ioutil.ReadFile(filePath) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
script = string(data) |
|||
} |
|||
|
|||
if script == "" { |
|||
return errors.New("empty lua script not allowed") |
|||
} |
|||
|
|||
s.LuaScript = script |
|||
|
|||
if _config.IsRedis() { |
|||
if !strings.Contains(script, `require("redisOps")`) { |
|||
return errors.New("lua script incorrect format") |
|||
} |
|||
|
|||
if !(strings.Contains(script, `SET(`) || |
|||
strings.Contains(script, `HSET(`) || |
|||
strings.Contains(script, `RPUSH(`) || |
|||
strings.Contains(script, `SADD(`) || |
|||
strings.Contains(script, `ZADD(`) || |
|||
strings.Contains(script, `DEL(`) || |
|||
strings.Contains(script, `HDEL(`) || |
|||
strings.Contains(script, `LREM(`) || |
|||
strings.Contains(script, `ZREM(`) || |
|||
strings.Contains(script, `SREM(`)) { |
|||
|
|||
return errors.New("lua script incorrect format") |
|||
} |
|||
} |
|||
|
|||
if _config.IsRocketmq() { |
|||
if !strings.Contains(script, `require("mqOps")`) { |
|||
return errors.New("lua script incorrect format") |
|||
} |
|||
|
|||
if !(strings.Contains(script, `SEND(`)) { |
|||
return errors.New("lua script incorrect format") |
|||
} |
|||
} |
|||
|
|||
if _config.IsEls() { |
|||
if !strings.Contains(script, `require("esOps")`) { |
|||
return errors.New("lua script incorrect format") |
|||
} |
|||
} |
|||
|
|||
reader := strings.NewReader(script) |
|||
chunk, err := parse.Parse(reader, script) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
var proto *lua.FunctionProto |
|||
proto, err = lua.Compile(chunk, script) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.LuaProto = proto |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
module go-mysql-transfer |
|||
|
|||
go 1.14 |
|||
|
|||
require ( |
|||
github.com/Shopify/sarama v1.27.0 |
|||
github.com/apache/rocketmq-client-go/v2 v2.0.0 |
|||
github.com/gin-gonic/gin v1.6.3 |
|||
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df |
|||
github.com/go-redis/redis v6.15.8+incompatible |
|||
github.com/go-sql-driver/mysql v1.5.0 |
|||
github.com/jmoiron/sqlx v1.2.0 // indirect |
|||
github.com/json-iterator/go v1.1.9 |
|||
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f |
|||
github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd // indirect |
|||
github.com/layeh/gopher-json v0.0.0-20190114024228-97fed8db8427 |
|||
github.com/olivere/elastic v6.2.34+incompatible |
|||
github.com/olivere/elastic/v7 v7.0.19 |
|||
github.com/onsi/ginkgo v1.14.0 // indirect |
|||
github.com/pingcap/errors v0.11.4 |
|||
github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc |
|||
github.com/pkg/errors v0.9.1 |
|||
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 |
|||
github.com/prometheus/client_golang v1.0.0 |
|||
github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e |
|||
github.com/satori/go.uuid v1.2.0 |
|||
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 |
|||
github.com/siddontang/go-mysql v1.1.0 |
|||
github.com/streadway/amqp v1.0.0 |
|||
github.com/vmihailenco/msgpack v4.0.4+incompatible |
|||
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e |
|||
go.etcd.io/bbolt v1.3.3 |
|||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 |
|||
go.mongodb.org/mongo-driver v1.4.0 |
|||
go.uber.org/atomic v1.6.0 |
|||
go.uber.org/zap v1.15.0 |
|||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 |
|||
gopkg.in/yaml.v2 v2.3.0 |
|||
github.com/sony/sonyflake v1.0.0 |
|||
) |
|||
@ -0,0 +1,693 @@ |
|||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= |
|||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= |
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= |
|||
github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= |
|||
github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= |
|||
github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= |
|||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= |
|||
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f h1:5ZfJxyXo8KyX8DgGXC5B7ILL8y51fci/qYz2B4j8iLY= |
|||
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= |
|||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= |
|||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= |
|||
github.com/apache/rocketmq-client-go/v2 v2.0.0 h1:D6jFj3DcNjWyjWn5N/R7Eq8v5kLqlgkFnT/DNQFnWlM= |
|||
github.com/apache/rocketmq-client-go/v2 v2.0.0/go.mod h1:oEZKFDvS7sz/RWU0839+dQBupazyBV7WX5cP6nrio0Q= |
|||
github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs= |
|||
github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= |
|||
github.com/aws/aws-sdk-go v1.33.5 h1:p2fr1ryvNTU6avUWLI+/H7FGv0TBIjzVM5WDgXBBv4U= |
|||
github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= |
|||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= |
|||
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= |
|||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= |
|||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= |
|||
github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= |
|||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= |
|||
github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= |
|||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= |
|||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= |
|||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= |
|||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= |
|||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= |
|||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= |
|||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= |
|||
github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= |
|||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= |
|||
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= |
|||
github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk= |
|||
github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= |
|||
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= |
|||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= |
|||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= |
|||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= |
|||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= |
|||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= |
|||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= |
|||
github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 h1:hxuZop6tSoOi0sxFzoGGYdRqNrPubyaIf9KoBG9tPiE= |
|||
github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= |
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
|||
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= |
|||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= |
|||
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= |
|||
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= |
|||
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= |
|||
github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= |
|||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= |
|||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= |
|||
github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= |
|||
github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= |
|||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= |
|||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= |
|||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= |
|||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= |
|||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= |
|||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= |
|||
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= |
|||
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= |
|||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= |
|||
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= |
|||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= |
|||
github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= |
|||
github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= |
|||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= |
|||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= |
|||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= |
|||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= |
|||
github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= |
|||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= |
|||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= |
|||
github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= |
|||
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= |
|||
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= |
|||
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= |
|||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= |
|||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= |
|||
github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= |
|||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= |
|||
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= |
|||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= |
|||
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= |
|||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= |
|||
github.com/go-playground/overalls v0.0.0-20180201144345-22ec1a223b7c/go.mod h1:UqxAgEOt89sCiXlrc/ycnx00LVvUO/eS8tMUkWX4R7w= |
|||
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= |
|||
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= |
|||
github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= |
|||
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= |
|||
github.com/go-redis/redis v6.15.8+incompatible h1:BKZuG6mCnRj5AOaWJXoCgf6rqTYnYJLe4en2hxT7r9o= |
|||
github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= |
|||
github.com/go-sql-driver/mysql v0.0.0-20170715192408-3955978caca4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= |
|||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= |
|||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= |
|||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= |
|||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= |
|||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= |
|||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= |
|||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= |
|||
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= |
|||
github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= |
|||
github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= |
|||
github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= |
|||
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= |
|||
github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= |
|||
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= |
|||
github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= |
|||
github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= |
|||
github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= |
|||
github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= |
|||
github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= |
|||
github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= |
|||
github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= |
|||
github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= |
|||
github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= |
|||
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= |
|||
github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= |
|||
github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= |
|||
github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= |
|||
github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= |
|||
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= |
|||
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= |
|||
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= |
|||
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= |
|||
github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= |
|||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= |
|||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= |
|||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= |
|||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= |
|||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= |
|||
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= |
|||
github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= |
|||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= |
|||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= |
|||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= |
|||
github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= |
|||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= |
|||
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
|||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
|||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
|||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= |
|||
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= |
|||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= |
|||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= |
|||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= |
|||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= |
|||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= |
|||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= |
|||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= |
|||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= |
|||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
|||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= |
|||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= |
|||
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= |
|||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= |
|||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= |
|||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= |
|||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= |
|||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= |
|||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= |
|||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= |
|||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= |
|||
github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= |
|||
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= |
|||
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= |
|||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= |
|||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= |
|||
github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3 h1:3CYI9xg87xNAD+es02gZxbX/ky4KQeoFBsNOzuoAQZg= |
|||
github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= |
|||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= |
|||
github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= |
|||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= |
|||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= |
|||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= |
|||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= |
|||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= |
|||
github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= |
|||
github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= |
|||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= |
|||
github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= |
|||
github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= |
|||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= |
|||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= |
|||
github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= |
|||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= |
|||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= |
|||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= |
|||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= |
|||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= |
|||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= |
|||
github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= |
|||
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= |
|||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= |
|||
github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= |
|||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= |
|||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= |
|||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= |
|||
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= |
|||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= |
|||
github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf h1:Ut4tTtPNmInWiEWJRernsWm688R0RN6PFO8sZhwI0sk= |
|||
github.com/jeremywohl/flatten v0.0.0-20190921043622-d936035e55cf/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= |
|||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= |
|||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= |
|||
github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= |
|||
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= |
|||
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= |
|||
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= |
|||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= |
|||
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= |
|||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= |
|||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= |
|||
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= |
|||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= |
|||
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= |
|||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= |
|||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= |
|||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= |
|||
github.com/juju/clock v0.0.0-20180524022203-d293bb356ca4/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= |
|||
github.com/juju/errors v0.0.0-20150916125642-1b5e39b83d18/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= |
|||
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f h1:MCOvExGLpaSIzLYB4iQXEHP4jYVU6vmzLNQPdMVrxnM= |
|||
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= |
|||
github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9 h1:Y+lzErDTURqeXqlqYi4YBYbDd7ycU74gW1ADt57/bgY= |
|||
github.com/juju/loggo v0.0.0-20170605014607-8232ab8918d9/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= |
|||
github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= |
|||
github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= |
|||
github.com/juju/retry v0.0.0-20160928201858-1998d01ba1c3/go.mod h1:OohPQGsr4pnxwD5YljhQ+TZnuVRYpa5irjugL1Yuif4= |
|||
github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd h1:4MRI5TGW0cRgovUipCGLF4uF+31Fo8VzkV2753OAfEE= |
|||
github.com/juju/testing v0.0.0-20200706033705-4c23f9c453cd/go.mod h1:hpGvhGHPVbNBraRLZEhoQwFLMrjK8PSlO4D3nDjKYXo= |
|||
github.com/juju/utils v0.0.0-20180808125547-9dfc6dbfb02b/go.mod h1:6/KLg8Wz/y2KVGWEpkK9vMNGkOnu4k/cqs8Z1fKjTOk= |
|||
github.com/juju/version v0.0.0-20161031051906-1f41e27e54f2/go.mod h1:kE8gK5X0CImdr7qpSKl3xB2PmpySSmfj7zVbkZFs81U= |
|||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= |
|||
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= |
|||
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= |
|||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= |
|||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= |
|||
github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= |
|||
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= |
|||
github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= |
|||
github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= |
|||
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= |
|||
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= |
|||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= |
|||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= |
|||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= |
|||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= |
|||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= |
|||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= |
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= |
|||
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= |
|||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= |
|||
github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= |
|||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= |
|||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= |
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= |
|||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= |
|||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= |
|||
github.com/layeh/gopher-json v0.0.0-20190114024228-97fed8db8427 h1:YDEoPyEnynw+2uhIXxeWOS1vwZhYfeP92FpyLmpYmwo= |
|||
github.com/layeh/gopher-json v0.0.0-20190114024228-97fed8db8427/go.mod h1:E/q28EyUVBgBQnONAVPIdwvEsv4Ve0vaCA9JWim4+3I= |
|||
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= |
|||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= |
|||
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= |
|||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= |
|||
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= |
|||
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= |
|||
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= |
|||
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= |
|||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= |
|||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= |
|||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= |
|||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= |
|||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= |
|||
github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= |
|||
github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= |
|||
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= |
|||
github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= |
|||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= |
|||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= |
|||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= |
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= |
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= |
|||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= |
|||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= |
|||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= |
|||
github.com/montanaflynn/stats v0.0.0-20151014174947-eeaced052adb/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= |
|||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= |
|||
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808 h1:pmpDGKLw4n82EtrNiLqB+xSz/JQwFOaZuMALYUHwX5s= |
|||
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= |
|||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= |
|||
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= |
|||
github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= |
|||
github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= |
|||
github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= |
|||
github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= |
|||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= |
|||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= |
|||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= |
|||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= |
|||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= |
|||
github.com/olivere/elastic v6.2.34+incompatible h1:GdvWBAqyIyEEUd+J2sSj6EnIaBywz7zZtN+Ps4JCv0g= |
|||
github.com/olivere/elastic v6.2.34+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= |
|||
github.com/olivere/elastic/v7 v7.0.19 h1:w4F6JpqOISadhYf/n0NR1cNj73xHqh4pzPwD1Gkidts= |
|||
github.com/olivere/elastic/v7 v7.0.19/go.mod h1:4Jqt5xvjqpjCqgnTcHwl3j8TLs8mvoOK8NYgo/qEOu4= |
|||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= |
|||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= |
|||
github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= |
|||
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= |
|||
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= |
|||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= |
|||
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= |
|||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= |
|||
github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= |
|||
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= |
|||
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= |
|||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= |
|||
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= |
|||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= |
|||
github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= |
|||
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= |
|||
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= |
|||
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= |
|||
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= |
|||
github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4 h1:iRtOAQ6FXkY/BGvst3CDfTva4nTqh6CL8WXvanLdbu0= |
|||
github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= |
|||
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9 h1:KH4f4Si9XK6/IW50HtoaiLIFHGkapOM6w83za47UYik= |
|||
github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= |
|||
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= |
|||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= |
|||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= |
|||
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8= |
|||
github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= |
|||
github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d h1:rCmRK0lCRrHMUbS99BKFYhK9YxJDNw0xB033cQbYo0s= |
|||
github.com/pingcap/fn v0.0.0-20191016082858-07623b84a47d/go.mod h1:fMRU1BA1y+r89AxUoaAar4JjrhUkVDt0o0Np6V8XbDQ= |
|||
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= |
|||
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= |
|||
github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY= |
|||
github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1 h1:J5oimSv+0emw5e/D1ZX/zh2WcMv0pOVT9QKruXfvJbg= |
|||
github.com/pingcap/kvproto v0.0.0-20191104103048-40f562012fb1/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= |
|||
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw= |
|||
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww9rkZ1DnWfl0pXV3BOWlkYIjA= |
|||
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= |
|||
github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= |
|||
github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5 h1:gJUprtXZ95CeFSb3FsDvQ3cSLB9c0W/lPYdqsaRSOng= |
|||
github.com/pingcap/parser v0.0.0-20191112053614-3b43b46331d5/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= |
|||
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0 h1:GIEq+wZfrl2bcJxpuSrEH4H7/nlf5YdmpS+dU9lNIt8= |
|||
github.com/pingcap/pd v1.1.0-beta.0.20190923032047-5c648dc365e0/go.mod h1:G/6rJpnYwM0LKMec2rI82/5Kg6GaZMvlfB+e6/tvYmI= |
|||
github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc h1:RtwQ6Kb8QWkUcInh2vzSc8HtQ2VHNNBV6datScfyF3c= |
|||
github.com/pingcap/tidb v1.1.0-beta.0.20191115021711-b274eb2079dc/go.mod h1:8DSQhELIQr/UqWaolPciDK1Nj6ARvD0NjBmQLCRJpxY= |
|||
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible h1:H1jg0aDWz2SLRh3hNBo2HFtnuHtudIUvBumU7syRkic= |
|||
github.com/pingcap/tidb-tools v3.0.6-0.20191106033616-90632dda3863+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= |
|||
github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= |
|||
github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e h1:TWwzCfLrj9GH5uaT0VcvdSnrHuwEntUfoHDTYdOzNNI= |
|||
github.com/pingcap/tipb v0.0.0-20191112054303-0b0ad0d4a92e/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= |
|||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
|||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= |
|||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
|||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= |
|||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
|||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
|||
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= |
|||
github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= |
|||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= |
|||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= |
|||
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= |
|||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= |
|||
github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= |
|||
github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= |
|||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= |
|||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= |
|||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= |
|||
github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= |
|||
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= |
|||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= |
|||
github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= |
|||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= |
|||
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= |
|||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= |
|||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= |
|||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= |
|||
github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7 h1:FUL3b97ZY2EPqg2NbXKuMHs5pXJB9hjj1fDHnF2vl28= |
|||
github.com/remyoudompheng/bigfft v0.0.0-20190512091148-babf20351dd7/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= |
|||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= |
|||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= |
|||
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= |
|||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= |
|||
github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e h1:CGjiMQ0wMH4wtNWrlj6kiTbkPt2F3rbYnhGX6TWLfco= |
|||
github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= |
|||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= |
|||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= |
|||
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= |
|||
github.com/shirou/gopsutil v2.18.10+incompatible h1:cy84jW6EVRPa5g9HAHrlbxMSIjBhDSX0OFYyMYminYs= |
|||
github.com/shirou/gopsutil v2.18.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= |
|||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= |
|||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= |
|||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= |
|||
github.com/shurcooL/vfsgen v0.0.0-20181020040650-a97a25d856ca/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= |
|||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= |
|||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= |
|||
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= |
|||
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= |
|||
github.com/siddontang/go-mysql v1.1.0 h1:NfkS1skrPwUd3hsUqhc6jrv24dKTNMANxKRmDsf1fMc= |
|||
github.com/siddontang/go-mysql v1.1.0/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= |
|||
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= |
|||
github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= |
|||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= |
|||
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= |
|||
github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= |
|||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= |
|||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= |
|||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= |
|||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= |
|||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= |
|||
github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck= |
|||
github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= |
|||
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= |
|||
github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 h1:N8Bg45zpk/UcpNGnfJt2y/3lRWASHNTUET8owPYCgYI= |
|||
github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= |
|||
github.com/smartystreets/gunit v1.3.4/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= |
|||
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= |
|||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= |
|||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= |
|||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= |
|||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= |
|||
github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= |
|||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= |
|||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= |
|||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= |
|||
github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= |
|||
github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= |
|||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
|||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
|||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= |
|||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= |
|||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= |
|||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= |
|||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= |
|||
github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= |
|||
github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= |
|||
github.com/struCoder/pidusage v0.1.2/go.mod h1:pWBlW3YuSwRl6h7R5KbvA4N8oOqe9LjaKW5CwT1SPjI= |
|||
github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d h1:4J9HCZVpvDmj2tiKGSTUnb3Ok/9CEQb9oqu9LHKQQpc= |
|||
github.com/syndtr/goleveldb v0.0.0-20180815032940-ae2bd5eed72d/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= |
|||
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= |
|||
github.com/tidwall/gjson v1.2.1 h1:j0efZLrZUvNerEf6xqoi0NjWMK5YlLrR7Guo/dxY174= |
|||
github.com/tidwall/gjson v1.2.1/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA= |
|||
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= |
|||
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= |
|||
github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g= |
|||
github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= |
|||
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= |
|||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= |
|||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= |
|||
github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= |
|||
github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= |
|||
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= |
|||
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= |
|||
github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= |
|||
github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= |
|||
github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= |
|||
github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= |
|||
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= |
|||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= |
|||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= |
|||
github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= |
|||
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= |
|||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= |
|||
github.com/unrolled/render v0.0.0-20171102162132-65450fb6b2d3/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= |
|||
github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d h1:ggUgChAeyge4NZ4QUw6lhHsVymzwSDJOZcE0s2X8S20= |
|||
github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= |
|||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= |
|||
github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= |
|||
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= |
|||
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= |
|||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= |
|||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= |
|||
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= |
|||
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= |
|||
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= |
|||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= |
|||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= |
|||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= |
|||
github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= |
|||
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e h1:oIpIX9VKxSCFrfjsKpluGbNPBGq9iNnT9crH781j9wY= |
|||
github.com/yuin/gopher-lua v0.0.0-20200603152657-dc2b0ca8b37e/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= |
|||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= |
|||
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= |
|||
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= |
|||
go.etcd.io/etcd v0.0.0-20190320044326-77d4b742cdbf/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= |
|||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= |
|||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= |
|||
go.mongodb.org/mongo-driver v1.4.0 h1:C8rFn1VF4GVEM/rG+dSoMmlm2pyQ9cs2/oRtUATejRU= |
|||
go.mongodb.org/mongo-driver v1.4.0/go.mod h1:llVBH2pkj9HywK0Dtdt6lDikOjFLbceHVu/Rc0iMKLs= |
|||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= |
|||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= |
|||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= |
|||
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= |
|||
go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= |
|||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= |
|||
go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= |
|||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= |
|||
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= |
|||
go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= |
|||
go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= |
|||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= |
|||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= |
|||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= |
|||
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= |
|||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= |
|||
go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= |
|||
go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= |
|||
go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= |
|||
golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= |
|||
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= |
|||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= |
|||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= |
|||
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= |
|||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= |
|||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= |
|||
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= |
|||
golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= |
|||
golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf h1:fnPsqIDRbCSgumaMCRpoIoF2s4qxv0xSSS0BVZUE/ss= |
|||
golang.org/x/crypto v0.0.0-20191029031824-8986dd9e96cf/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= |
|||
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= |
|||
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= |
|||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= |
|||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= |
|||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= |
|||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= |
|||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= |
|||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= |
|||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= |
|||
golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= |
|||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
|||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= |
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
|||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
|||
golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
|||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= |
|||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= |
|||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= |
|||
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs= |
|||
golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= |
|||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= |
|||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= |
|||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= |
|||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= |
|||
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= |
|||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
|||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= |
|||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= |
|||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= |
|||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= |
|||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= |
|||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= |
|||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= |
|||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= |
|||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
|||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
|||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= |
|||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= |
|||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
|||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
|||
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
|||
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
|||
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= |
|||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= |
|||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= |
|||
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= |
|||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= |
|||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= |
|||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/tools v0.0.0-20191107010934-f79515f33823 h1:akkRBeitX2EZP59KdtKw310CI4WGPCNPyrLbE7WZA8Y= |
|||
golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= |
|||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= |
|||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |
|||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= |
|||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= |
|||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= |
|||
google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= |
|||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= |
|||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= |
|||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= |
|||
google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 h1:oFSK4421fpCKRrpzIpybyBVWyht05NegY9+L/3TLAZs= |
|||
google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= |
|||
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= |
|||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= |
|||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= |
|||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= |
|||
google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= |
|||
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= |
|||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= |
|||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= |
|||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= |
|||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= |
|||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= |
|||
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= |
|||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= |
|||
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= |
|||
gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= |
|||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= |
|||
gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= |
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= |
|||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= |
|||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= |
|||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
|||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= |
|||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= |
|||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= |
|||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= |
|||
gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= |
|||
gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= |
|||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= |
|||
gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= |
|||
gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= |
|||
gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= |
|||
gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= |
|||
gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= |
|||
gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= |
|||
gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= |
|||
gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4 h1:hILp2hNrRnYjZpmIbx70psAHbBSEcQ1NIzDcUbJ1b6g= |
|||
gopkg.in/mgo.v2 v2.0.0-20160818015218-f2b6f6c918c4/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= |
|||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= |
|||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= |
|||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= |
|||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= |
|||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= |
|||
gopkg.in/yaml.v2 v2.0.0-20170712054546-1be3d31502d6/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= |
|||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= |
|||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= |
|||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= |
|||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
|||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
|||
gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YCZPcD3lo9AoczZpucwc= |
|||
gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
|||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= |
|||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= |
|||
honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= |
|||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= |
|||
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= |
|||
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= |
|||
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4 h1:VO9oZbbkvTwqLimlQt15QNdOOBArT2dw/bvzsMZBiqQ= |
|||
sourcegraph.com/sourcegraph/appdash v0.0.0-20180531100431-4c381bd170b4/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= |
|||
sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= |
|||
stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= |
|||
stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= |
|||
@ -0,0 +1,198 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package main |
|||
|
|||
import ( |
|||
"flag" |
|||
"fmt" |
|||
"log" |
|||
"os" |
|||
"os/signal" |
|||
"regexp" |
|||
"syscall" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/service" |
|||
"go-mysql-transfer/storage" |
|||
"go-mysql-transfer/util/stringutil" |
|||
"go-mysql-transfer/web" |
|||
) |
|||
|
|||
var ( |
|||
helpFlag bool |
|||
cfgPath string |
|||
stockFlag bool |
|||
positionFlag bool |
|||
statusFlag bool |
|||
) |
|||
|
|||
func init() { |
|||
flag.BoolVar(&helpFlag, "help", false, "this help") |
|||
flag.StringVar(&cfgPath, "config", "app.yml", "application config file") |
|||
flag.BoolVar(&stockFlag, "stock", false, "stock data import") |
|||
flag.BoolVar(&positionFlag, "position", false, "set dump position") |
|||
flag.BoolVar(&statusFlag, "status", false, "display application status") |
|||
flag.Usage = usage |
|||
} |
|||
|
|||
func main() { |
|||
|
|||
//stockFlag = true
|
|||
|
|||
//cfgPath = "D:\\transfer\\app.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\release_test_luascript.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\rabbitmq_release_test_lua.yml"
|
|||
//cfgPath = "D:\\transfer\\rabbitmq_release_test_rule.yml"
|
|||
|
|||
// cfgPath = "D:\\transfer\\kafka_release_test_lua.yml"
|
|||
// cfgPath = "D:\\transfer\\kafka_release_test_rule.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\rocketmq_release_test_lua.yml"
|
|||
//cfgPath = "D:\\transfer\\rocketmq_release_test_rule.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\es7_release_test_lua.yml"
|
|||
//cfgPath = "D:\\transfer\\es7_release_test_rule.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\es6_release_test_lua.yml"
|
|||
//cfgPath = "D:\\transfer\\es6_release_test_rule.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\redis_release_test_lua.yml"
|
|||
//cfgPath = "D:\\transfer\\redis_release_test_rule.yml"
|
|||
|
|||
//cfgPath = "D:\\transfer\\mongo_release_test_rule.yml"
|
|||
//cfgPath = "D:\\transfer\\mongo_release_test_lua.yml"
|
|||
|
|||
flag.Parse() |
|||
if helpFlag { |
|||
flag.Usage() |
|||
return |
|||
} |
|||
|
|||
// 初始化global
|
|||
err := global.Initialize(cfgPath) |
|||
if err != nil { |
|||
println(errors.ErrorStack(err)) |
|||
return |
|||
} |
|||
|
|||
if stockFlag { |
|||
doStock() |
|||
return |
|||
} |
|||
|
|||
// 初始化Storage
|
|||
err = storage.Initialize() |
|||
if err != nil { |
|||
println(errors.ErrorStack(err)) |
|||
return |
|||
} |
|||
|
|||
if statusFlag { |
|||
doStatus() |
|||
return |
|||
} |
|||
|
|||
if positionFlag { |
|||
doPosition() |
|||
return |
|||
} |
|||
|
|||
err = service.Initialize() |
|||
if err != nil { |
|||
println(errors.ErrorStack(err)) |
|||
return |
|||
} |
|||
|
|||
if err := metrics.Initialize(); err != nil { |
|||
println(errors.ErrorStack(err)) |
|||
return |
|||
} |
|||
|
|||
if err := web.Start(); err != nil { |
|||
println(errors.ErrorStack(err)) |
|||
return |
|||
} |
|||
service.StartUp() // start application
|
|||
|
|||
s := make(chan os.Signal, 1) |
|||
signal.Notify(s, os.Kill, os.Interrupt, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) |
|||
sin := <-s |
|||
log.Printf("application stoped，signal: %s \n", sin.String()) |
|||
|
|||
web.Close() |
|||
service.Close() |
|||
storage.Close() |
|||
} |
|||
|
|||
func doStock() { |
|||
stock := service.NewStockService() |
|||
if err := stock.Run(); err != nil { |
|||
println(errors.ErrorStack(err)) |
|||
} |
|||
stock.Close() |
|||
} |
|||
|
|||
func doStatus() { |
|||
ps := storage.NewPositionStorage() |
|||
pos, _ := ps.Get() |
|||
fmt.Printf("The current dump position is : %s %d \n", pos.Name, pos.Pos) |
|||
} |
|||
|
|||
func doPosition() { |
|||
others := flag.Args() |
|||
if len(others) != 2 { |
|||
println("error: please input the binlog's File and Position") |
|||
return |
|||
} |
|||
f := others[0] |
|||
p := others[1] |
|||
|
|||
matched, _ := regexp.MatchString(".+\\.\\d+$", f) |
|||
if !matched { |
|||
println("error: The parameter File must be like: mysql-bin.000001") |
|||
return |
|||
} |
|||
|
|||
pp, err := stringutil.ToUint32(p) |
|||
if nil != err { |
|||
println("error: The parameter Position must be number") |
|||
return |
|||
} |
|||
ps := storage.NewPositionStorage() |
|||
pos := mysql.Position{ |
|||
Name: f, |
|||
Pos: pp, |
|||
} |
|||
ps.Save(pos) |
|||
fmt.Printf("The current dump position is : %s %d \n", f, pp) |
|||
} |
|||
|
|||
func usage() { |
|||
fmt.Fprintf(os.Stderr, `version: 1.0.0 |
|||
Usage: transfer [-c filename] [-s stock] |
|||
|
|||
Options: |
|||
`) |
|||
flag.PrintDefaults() |
|||
} |
|||
@ -0,0 +1,227 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package metrics |
|||
|
|||
import ( |
|||
"fmt" |
|||
"net/http" |
|||
|
|||
"github.com/prometheus/client_golang/prometheus" |
|||
"github.com/prometheus/client_golang/prometheus/promauto" |
|||
"github.com/prometheus/client_golang/prometheus/promhttp" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"go.uber.org/atomic" |
|||
|
|||
"go-mysql-transfer/global" |
|||
) |
|||
|
|||
const ( |
|||
DestStateOK = 1 |
|||
DestStateFail = 0 |
|||
|
|||
LeaderState = 1 |
|||
FollowerState = 0 |
|||
) |
|||
|
|||
var ( |
|||
leaderState atomic.Bool |
|||
destState atomic.Bool |
|||
delay atomic.Uint32 |
|||
insertRecord map[string]*atomic.Uint64 |
|||
updateRecord map[string]*atomic.Uint64 |
|||
deleteRecord map[string]*atomic.Uint64 |
|||
) |
|||
|
|||
var ( |
|||
leaderStateGauge = promauto.NewGauge( |
|||
prometheus.GaugeOpts{ |
|||
Name: "transfer_leader_state", |
|||
Help: "The cluster leader state: 0=false, 1=true", |
|||
}, |
|||
) |
|||
|
|||
destStateGauge = promauto.NewGauge( |
|||
prometheus.GaugeOpts{ |
|||
Name: "transfer_destination_state", |
|||
Help: "The destination running state: 0=stopped, 1=ok", |
|||
}, |
|||
) |
|||
|
|||
delayGauge = promauto.NewGauge( |
|||
prometheus.GaugeOpts{ |
|||
Name: "transfer_delay", |
|||
Help: "The transfer slave lag", |
|||
}, |
|||
) |
|||
|
|||
insertCounter = promauto.NewCounterVec( |
|||
prometheus.CounterOpts{ |
|||
Name: "transfer_inserted_num", |
|||
Help: "The number of data inserted to destination", |
|||
}, []string{"table"}, |
|||
) |
|||
|
|||
updateCounter = promauto.NewCounterVec( |
|||
prometheus.CounterOpts{ |
|||
Name: "transfer_updated_num", |
|||
Help: "The number of data updated to destination", |
|||
}, []string{"table"}, |
|||
) |
|||
|
|||
deleteCounter = promauto.NewCounterVec( |
|||
prometheus.CounterOpts{ |
|||
Name: "transfer_deleted_num", |
|||
Help: "The number of data deleted from destination", |
|||
}, []string{"table"}, |
|||
) |
|||
) |
|||
|
|||
func Initialize() error { |
|||
if global.Cfg().EnableExporter { |
|||
go func() { |
|||
http.Handle("/", promhttp.Handler()) |
|||
http.ListenAndServe(fmt.Sprintf(":%d", global.Cfg().ExporterPort), nil) |
|||
}() |
|||
} |
|||
if global.Cfg().EnableWebAdmin { |
|||
insertRecord = make(map[string]*atomic.Uint64) |
|||
updateRecord = make(map[string]*atomic.Uint64) |
|||
deleteRecord = make(map[string]*atomic.Uint64) |
|||
for _, k := range global.RuleKeyList() { |
|||
insertRecord[k] = &atomic.Uint64{} |
|||
updateRecord[k] = &atomic.Uint64{} |
|||
deleteRecord[k] = &atomic.Uint64{} |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func SetLeaderState(state int) { |
|||
if global.Cfg().EnableExporter { |
|||
leaderStateGauge.Set(float64(state)) |
|||
} |
|||
if global.Cfg().EnableWebAdmin { |
|||
leaderState.Store(LeaderState == state) |
|||
} |
|||
} |
|||
|
|||
func SetDestState(state int) { |
|||
if global.Cfg().EnableExporter { |
|||
destStateGauge.Set(float64(state)) |
|||
} |
|||
if global.Cfg().EnableWebAdmin { |
|||
destState.Store(DestStateOK == state) |
|||
} |
|||
} |
|||
|
|||
func DestState() bool { |
|||
return destState.Load() |
|||
} |
|||
|
|||
func SetTransferDelay(d uint32) { |
|||
if global.Cfg().EnableExporter { |
|||
delayGauge.Set(float64(d)) |
|||
} |
|||
if global.Cfg().EnableWebAdmin { |
|||
delay.Store(d) |
|||
} |
|||
} |
|||
|
|||
func UpdateActionNum(action, lab string) { |
|||
if global.Cfg().EnableExporter { |
|||
switch action { |
|||
case canal.InsertAction: |
|||
insertCounter.WithLabelValues(lab).Inc() |
|||
case canal.UpdateAction: |
|||
updateCounter.WithLabelValues(lab).Inc() |
|||
case canal.DeleteAction: |
|||
deleteCounter.WithLabelValues(lab).Inc() |
|||
} |
|||
} |
|||
if global.Cfg().EnableWebAdmin { |
|||
switch action { |
|||
case canal.InsertAction: |
|||
if v, ok := insertRecord[lab]; ok { |
|||
v.Inc() |
|||
} |
|||
case canal.UpdateAction: |
|||
if v, ok := updateRecord[lab]; ok { |
|||
v.Inc() |
|||
} |
|||
case canal.DeleteAction: |
|||
if v, ok := deleteRecord[lab]; ok { |
|||
v.Inc() |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
func InsertAmount() uint64 { |
|||
var amount uint64 |
|||
for _, v := range insertRecord { |
|||
amount += v.Load() |
|||
} |
|||
return amount |
|||
} |
|||
|
|||
func UpdateAmount() uint64 { |
|||
var amount uint64 |
|||
for _, v := range updateRecord { |
|||
amount += v.Load() |
|||
} |
|||
return amount |
|||
} |
|||
|
|||
func DeleteAmount() uint64 { |
|||
var amount uint64 |
|||
for _, v := range deleteRecord { |
|||
amount += v.Load() |
|||
} |
|||
return amount |
|||
} |
|||
|
|||
func LabInsertAmount(lab string) uint64 { |
|||
var nn uint64 |
|||
n, ok := insertRecord[lab] |
|||
if ok { |
|||
nn = n.Load() |
|||
} |
|||
return nn |
|||
} |
|||
|
|||
func LabUpdateRecord(lab string) uint64 { |
|||
var nn uint64 |
|||
n, ok := updateRecord[lab] |
|||
if ok { |
|||
nn = n.Load() |
|||
} |
|||
return nn |
|||
} |
|||
|
|||
func LabDeleteRecord(lab string) uint64 { |
|||
var nn uint64 |
|||
n, ok := deleteRecord[lab] |
|||
if ok { |
|||
nn = n.Load() |
|||
} |
|||
return nn |
|||
} |
|||
|
|||
func LeaderFlag() bool { |
|||
return leaderState.Load() |
|||
} |
|||
@ -0,0 +1,12 @@ |
|||
package model |
|||
|
|||
import "github.com/siddontang/go-mysql/schema" |
|||
|
|||
type Padding struct { |
|||
WrapName string |
|||
|
|||
ColumnName string |
|||
ColumnIndex int |
|||
ColumnType int |
|||
ColumnMetadata *schema.TableColumn |
|||
} |
|||
@ -0,0 +1,17 @@ |
|||
package model |
|||
|
|||
const ( |
|||
PipelineInfoNormal = 1 |
|||
PipelineInfoDisable = 2 |
|||
) |
|||
|
|||
// PipelineInfo 管道
|
|||
type PipelineInfo struct { |
|||
Id uint64 |
|||
Name string |
|||
SourceId uint64 |
|||
TargetId uint64 |
|||
Status uint8 |
|||
CreateTime int64 |
|||
UpdateTime int64 |
|||
} |
|||
@ -0,0 +1,11 @@ |
|||
package model |
|||
|
|||
// Replog = Replicated Log
|
|||
type Replog struct { |
|||
Id uint64 |
|||
Type uint8 |
|||
Target uint8 |
|||
TargetId uint8 |
|||
timestamp int64 |
|||
Context []byte |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
package model |
|||
|
|||
import "sync" |
|||
|
|||
var RowRequestPool = sync.Pool{ |
|||
New: func() interface{} { |
|||
return new(RowRequest) |
|||
}, |
|||
} |
|||
|
|||
type RowRequest struct { |
|||
RuleKey string |
|||
Action string |
|||
Timestamp uint32 |
|||
Old []interface{} |
|||
Row []interface{} |
|||
} |
|||
|
|||
type PosRequest struct { |
|||
Name string |
|||
Pos uint32 |
|||
Force bool |
|||
} |
|||
|
|||
func BuildRowRequest() *RowRequest { |
|||
return RowRequestPool.Get().(*RowRequest) |
|||
} |
|||
|
|||
func ReleaseRowRequest(t *RowRequest) { |
|||
RowRequestPool.Put(t) |
|||
} |
|||
@ -0,0 +1,93 @@ |
|||
package model |
|||
|
|||
import "sync" |
|||
|
|||
var mqRespondPool = sync.Pool{ |
|||
New: func() interface{} { |
|||
return new(MQRespond) |
|||
}, |
|||
} |
|||
|
|||
var esRespondPool = sync.Pool{ |
|||
New: func() interface{} { |
|||
return new(ESRespond) |
|||
}, |
|||
} |
|||
|
|||
var mongoRespondPool = sync.Pool{ |
|||
New: func() interface{} { |
|||
return new(MongoRespond) |
|||
}, |
|||
} |
|||
|
|||
var redisRespondPool = sync.Pool{ |
|||
New: func() interface{} { |
|||
return new(RedisRespond) |
|||
}, |
|||
} |
|||
|
|||
type MQRespond struct { |
|||
Topic string `json:"-"` |
|||
Action string `json:"action"` |
|||
Timestamp uint32 `json:"timestamp"` |
|||
Raw interface{} `json:"raw,omitempty"` |
|||
Date interface{} `json:"date"` |
|||
ByteArray []byte `json:"-"` |
|||
} |
|||
|
|||
type ESRespond struct { |
|||
Index string |
|||
Id string |
|||
Action string |
|||
Date string |
|||
} |
|||
|
|||
type MongoRespond struct { |
|||
RuleKey string |
|||
Collection string |
|||
Action string |
|||
Id interface{} |
|||
Table map[string]interface{} |
|||
} |
|||
|
|||
type RedisRespond struct { |
|||
Action string |
|||
Structure string |
|||
Key string |
|||
Field string |
|||
Score float64 |
|||
OldVal interface{} |
|||
Val interface{} |
|||
} |
|||
|
|||
func BuildMQRespond() *MQRespond { |
|||
return mqRespondPool.Get().(*MQRespond) |
|||
} |
|||
|
|||
func ReleaseMQRespond(t *MQRespond) { |
|||
mqRespondPool.Put(t) |
|||
} |
|||
|
|||
func BuildESRespond() *ESRespond { |
|||
return esRespondPool.Get().(*ESRespond) |
|||
} |
|||
|
|||
func ReleaseESRespond(t *ESRespond) { |
|||
esRespondPool.Put(t) |
|||
} |
|||
|
|||
func BuildMongoRespond() *MongoRespond { |
|||
return mongoRespondPool.Get().(*MongoRespond) |
|||
} |
|||
|
|||
func ReleaseMongoRespond(t *MongoRespond) { |
|||
mongoRespondPool.Put(t) |
|||
} |
|||
|
|||
func BuildRedisRespond() *RedisRespond { |
|||
return redisRespondPool.Get().(*RedisRespond) |
|||
} |
|||
|
|||
func ReleaseRedisRespond(t *RedisRespond) { |
|||
redisRespondPool.Put(t) |
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
package model |
|||
|
|||
//Type
|
|||
// 1: mongodb Addr Username Password
|
|||
// 2: elasticsearch Addr User Password Version
|
|||
// 3: redis Addr GroupType MasterName Pass Database
|
|||
// 4: rocketmq NameServers GroupName InstanceName AccessKey SecretKey
|
|||
// 5: kafka Addr SASLUser SASLPassword
|
|||
// 6: rabbitmq Addr
|
|||
// 7: script
|
|||
|
|||
const ( |
|||
TargetTypeMongodb = 1 |
|||
TargetTypeElasticsearch = 2 |
|||
TargetTypeRedis = 3 |
|||
TargetTypeRocketmq = 4 |
|||
TargetTypeKafka = 5 |
|||
TargetTypeRabbitmq = 6 |
|||
TargetTypeScript = 7 |
|||
) |
|||
|
|||
// TargetInfo 目标
|
|||
type TargetInfo struct { |
|||
Id uint64 |
|||
Type uint8 |
|||
Name string |
|||
Addr string //地址
|
|||
Username string //用户名
|
|||
Password string //密码
|
|||
|
|||
GroupType string `yaml:"group_type"` //集群类型 sentinel或者cluster
|
|||
MasterName string `yaml:"master_name"` //Master节点名称
|
|||
Database int `yaml:"database"` //数据库
|
|||
|
|||
Version int //版本
|
|||
|
|||
NameServers string //命名服务地址，多个用逗号分隔
|
|||
GroupName string //group name,默认为空
|
|||
InstanceName string //instance name,默认为空
|
|||
AccessKey string //访问控制 accessKey,默认为空
|
|||
SecretKey string //访问控制 secretKey,默认为空
|
|||
|
|||
SASLUser string `yaml:"sasl_user"` //SASL_PLAINTEXT认证模式 用户名
|
|||
SASLPassword string `yaml:"sasl_password"` //SASL_PLAINTEXT认证模式 密码
|
|||
|
|||
} |
|||
@ -0,0 +1,20 @@ |
|||
syntax = "proto3"; |
|||
// 定义包名 |
|||
option go_package ="/transport"; |
|||
|
|||
service Transport { |
|||
// Ping |
|||
rpc Ping (PingRequest) returns (PongResponse); |
|||
} |
|||
|
|||
// 定义Ping请求消息结构 |
|||
message PingRequest { |
|||
// 类型 字段 = 标识号 |
|||
string token = 1; |
|||
string context = 2; |
|||
} |
|||
|
|||
// 定义 Pong响应消息结构 |
|||
message PongResponse { |
|||
string context = 1; |
|||
} |
|||
@ -0,0 +1,65 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package service |
|||
|
|||
import ( |
|||
"log" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
) |
|||
|
|||
type ClusterService struct { |
|||
electionSignal chan bool //选举信号
|
|||
} |
|||
|
|||
func (s *ClusterService) boot() error { |
|||
log.Println("start master election") |
|||
err := _electionService.Elect() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.startElectListener() |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *ClusterService) startElectListener() { |
|||
go func() { |
|||
for { |
|||
select { |
|||
case selected := <-s.electionSignal: |
|||
global.SetLeaderNode(_electionService.Leader()) |
|||
global.SetLeaderFlag(selected) |
|||
if selected { |
|||
metrics.SetLeaderState(metrics.LeaderState) |
|||
_transferService.StartUp() |
|||
} else { |
|||
metrics.SetLeaderState(metrics.FollowerState) |
|||
_transferService.stopDump() |
|||
} |
|||
} |
|||
} |
|||
|
|||
}() |
|||
} |
|||
|
|||
func (s *ClusterService) Nodes() []string { |
|||
return _electionService.Nodes() |
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package election |
|||
|
|||
import ( |
|||
"go-mysql-transfer/global" |
|||
) |
|||
|
|||
type Service interface { |
|||
Elect() error |
|||
IsLeader() bool |
|||
Leader() string |
|||
Nodes() []string |
|||
} |
|||
|
|||
func NewElection(_informCh chan bool) Service { |
|||
if global.Cfg().IsZk() { |
|||
return newZkElection(_informCh) |
|||
} else { |
|||
return newEtcdElection(_informCh) |
|||
} |
|||
} |
|||
@ -0,0 +1,170 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package election |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"log" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/juju/errors" |
|||
"go.etcd.io/etcd/clientv3/concurrency" |
|||
"go.uber.org/atomic" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/storage" |
|||
"go-mysql-transfer/util/etcds" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
const _electionNodeTTL = 2 //秒
|
|||
|
|||
type etcdElection struct { |
|||
once sync.Once |
|||
|
|||
informCh chan bool |
|||
|
|||
selected atomic.Bool |
|||
ensured atomic.Bool |
|||
leader atomic.String |
|||
} |
|||
|
|||
func newEtcdElection(_informCh chan bool) *etcdElection { |
|||
return &etcdElection{ |
|||
informCh: _informCh, |
|||
} |
|||
} |
|||
|
|||
func (s *etcdElection) Elect() error { |
|||
s.doElect() |
|||
s.ensureFollower() |
|||
return nil |
|||
} |
|||
|
|||
func (s *etcdElection) doElect() { |
|||
go func() { |
|||
|
|||
for { |
|||
session, err := concurrency.NewSession(storage.EtcdConn(), concurrency.WithTTL(_electionNodeTTL)) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
return |
|||
} |
|||
elc := concurrency.NewElection(session, global.Cfg().ZkElectionDir()) |
|||
ctx := context.Background() |
|||
if err = elc.Campaign(ctx, global.CurrentNode()); err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
session.Close() |
|||
s.beFollower("") |
|||
continue |
|||
} |
|||
|
|||
select { |
|||
case <-session.Done(): |
|||
s.beFollower("") |
|||
continue |
|||
default: |
|||
s.beLeader() |
|||
err = etcds.UpdateOrCreate(global.Cfg().ZkElectedDir(), elc.Key(), storage.EtcdOps()) |
|||
if err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
return |
|||
} |
|||
} |
|||
|
|||
shouldBreak := false |
|||
for !shouldBreak { |
|||
select { |
|||
case <-session.Done(): |
|||
logs.Warn("etcd session has done") |
|||
shouldBreak = true |
|||
s.beFollower("") |
|||
break |
|||
case <-ctx.Done(): |
|||
ctxTmp, _ := context.WithTimeout(context.Background(), time.Second*_electionNodeTTL) |
|||
elc.Resign(ctxTmp) |
|||
session.Close() |
|||
s.beFollower("") |
|||
return |
|||
} |
|||
} |
|||
} |
|||
}() |
|||
} |
|||
|
|||
func (s *etcdElection) IsLeader() bool { |
|||
return s.selected.Load() |
|||
} |
|||
|
|||
func (s *etcdElection) Leader() string { |
|||
return s.leader.Load() |
|||
} |
|||
|
|||
func (s *etcdElection) ensureFollower() { |
|||
go func() { |
|||
for { |
|||
if s.selected.Load() { |
|||
break |
|||
} |
|||
|
|||
k, _, err := etcds.Get(global.Cfg().ZkElectedDir(), storage.EtcdOps()) |
|||
if err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
continue |
|||
} |
|||
|
|||
var l []byte |
|||
l, _, err = etcds.Get(string(k), storage.EtcdOps()) |
|||
if err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
continue |
|||
} |
|||
|
|||
s.ensured.Store(true) |
|||
s.beFollower(string(l)) |
|||
break |
|||
} |
|||
}() |
|||
} |
|||
|
|||
func (s *etcdElection) Nodes() []string { |
|||
var nodes []string |
|||
ls, err := etcds.List("/transfer/myTransfer/election", storage.EtcdOps()) |
|||
if err == nil { |
|||
for _, v := range ls { |
|||
nodes = append(nodes, string(v.Value)) |
|||
} |
|||
} |
|||
return nodes |
|||
} |
|||
|
|||
func (s *etcdElection) beLeader() { |
|||
s.selected.Store(true) |
|||
s.leader.Store(global.CurrentNode()) |
|||
s.informCh <- s.selected.Load() |
|||
log.Println("the current node is the master") |
|||
} |
|||
|
|||
func (s *etcdElection) beFollower(leader string) { |
|||
s.selected.Store(false) |
|||
s.informCh <- s.selected.Load() |
|||
s.leader.Store(leader) |
|||
log.Println(fmt.Sprintf("The current node is the follower, master node is : %s", s.leader.Load())) |
|||
} |
|||
@ -0,0 +1,166 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package election |
|||
|
|||
import ( |
|||
"fmt" |
|||
"log" |
|||
"sync" |
|||
|
|||
"github.com/samuel/go-zookeeper/zk" |
|||
"go.uber.org/atomic" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/storage" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
type zkElection struct { |
|||
once sync.Once |
|||
informCh chan bool |
|||
|
|||
selected atomic.Bool |
|||
leader atomic.String |
|||
|
|||
connectingAmount atomic.Int64 |
|||
downgraded atomic.Bool |
|||
} |
|||
|
|||
func newZkElection(_informCh chan bool) *zkElection { |
|||
return &zkElection{ |
|||
informCh: _informCh, |
|||
} |
|||
} |
|||
|
|||
func (s *zkElection) Elect() error { |
|||
data := []byte(global.CurrentNode()) |
|||
acl := zk.WorldACL(zk.PermAll) |
|||
_, err := storage.ZKConn().Create(global.Cfg().ZkElectionDir(), data, zk.FlagEphemeral, acl) |
|||
if err == nil { |
|||
s.beLeader() |
|||
} else { |
|||
v, _, err := storage.ZKConn().Get(global.Cfg().ZkElectionDir()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
leader := string(v) |
|||
if leader == global.CurrentNode() && s.downgraded.Load() { |
|||
s.beLeader() |
|||
} else { |
|||
s.beFollower(leader) |
|||
} |
|||
} |
|||
|
|||
// register
|
|||
dir := global.Cfg().ZkNodesDir() + "/" + global.CurrentNode() |
|||
storage.ZKConn().Create(dir, data, zk.FlagEphemeral, acl) |
|||
|
|||
s.once.Do(func() { |
|||
s.startConnectionWatchTask() |
|||
s.startNodeWatchTask() |
|||
}) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *zkElection) IsLeader() bool { |
|||
return s.selected.Load() |
|||
} |
|||
|
|||
func (s *zkElection) Leader() string { |
|||
return s.leader.Load() |
|||
} |
|||
|
|||
func (s *zkElection) Nodes() []string { |
|||
v, _, err := storage.ZKConn().Children(global.Cfg().ZkNodesDir()) |
|||
if err != nil { |
|||
return nil |
|||
} |
|||
return v |
|||
} |
|||
|
|||
func (s *zkElection) beLeader() { |
|||
s.selected.Store(true) |
|||
s.leader.Store(global.CurrentNode()) |
|||
s.informCh <- s.selected.Load() |
|||
log.Println("the current node is the master") |
|||
} |
|||
|
|||
func (s *zkElection) beFollower(leader string) { |
|||
s.selected.Store(false) |
|||
s.leader.Store(leader) |
|||
s.informCh <- s.selected.Load() |
|||
log.Println(fmt.Sprintf("The current node is the follower, master node is : %s", leader)) |
|||
} |
|||
|
|||
func (s *zkElection) startConnectionWatchTask() { |
|||
logs.Info("Start zookeeper connection Status watch task") |
|||
go func() { |
|||
for { |
|||
select { |
|||
case event := <-storage.ZKStatusSignal(): |
|||
logs.Infof("ZK ConnStatus: %v", event) |
|||
if s.selected.Load() { |
|||
if zk.StateConnecting == event.State { |
|||
s.connectingAmount.Inc() |
|||
} |
|||
if s.connectingAmount.Load() > int64(len(storage.ZKAddresses())) { |
|||
s.downgrading() |
|||
} |
|||
} |
|||
if zk.StateHasSession == event.State { |
|||
s.connectingAmount.Store(0) |
|||
if s.downgraded.Load() { |
|||
logs.Info("zookeeper HasSession restart elect") |
|||
s.Elect() |
|||
s.downgraded.Store(false) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
}() |
|||
} |
|||
|
|||
func (s *zkElection) startNodeWatchTask() { |
|||
go func() { |
|||
|
|||
logs.Info("Start zookeeper election node watch task") |
|||
_, _, ch, _ := storage.ZKConn().ChildrenW(global.Cfg().ZkElectionDir()) |
|||
for { |
|||
select { |
|||
case childEvent := <-ch: |
|||
if childEvent.Type == zk.EventNodeDeleted { |
|||
logs.Info("Start elect new master ...") |
|||
err := s.Elect() |
|||
if err != nil { |
|||
logs.Errorf("elect new master error %s ", err.Error()) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
}() |
|||
} |
|||
|
|||
func (s *zkElection) downgrading() { |
|||
if !s.downgraded.Load() { |
|||
log.Println("Lost contact with zookeeper, The current node degraded to Follower") |
|||
s.downgraded.Store(true) |
|||
s.beFollower("") |
|||
} |
|||
} |
|||
@ -0,0 +1,310 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"context" |
|||
"log" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/olivere/elastic" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logagent" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
type Elastic6Endpoint struct { |
|||
first string |
|||
hosts []string |
|||
client *elastic.Client |
|||
|
|||
retryLock sync.Mutex |
|||
} |
|||
|
|||
func newElastic6Endpoint() *Elastic6Endpoint { |
|||
r := &Elastic6Endpoint{} |
|||
r.hosts = strings.Split(global.Cfg().ElsAddr, ",") |
|||
r.first = r.hosts[0] |
|||
return r |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) Connect() error { |
|||
var options []elastic.ClientOptionFunc |
|||
options = append(options, elastic.SetErrorLog(logagent.NewElsLoggerAgent())) |
|||
options = append(options, elastic.SetURL(s.hosts...)) |
|||
if global.Cfg().ElsUser != "" && global.Cfg().ElsPassword != "" { |
|||
options = append(options, elastic.SetBasicAuth(global.Cfg().ElsUser, global.Cfg().Password)) |
|||
} |
|||
|
|||
client, err := elastic.NewClient(options...) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.client = client |
|||
return s.indexMapping() |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) indexMapping() error { |
|||
for _, rule := range global.RuleInsList() { |
|||
exists, err := s.client.IndexExists(rule.ElsIndex).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if exists { |
|||
err = s.updateIndexMapping(rule) |
|||
} else { |
|||
err = s.insertIndexMapping(rule) |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) insertIndexMapping(rule *global.Rule) error { |
|||
var properties map[string]interface{} |
|||
if rule.LuaEnable() { |
|||
properties = buildPropertiesByMappings(rule) |
|||
} else { |
|||
properties = buildPropertiesByRule(rule) |
|||
} |
|||
|
|||
mapping := map[string]interface{}{ |
|||
"mappings": map[string]interface{}{ |
|||
rule.ElsType: map[string]interface{}{ |
|||
"properties": properties, |
|||
}, |
|||
}, |
|||
} |
|||
body := stringutil.ToJsonString(mapping) |
|||
|
|||
ret, err := s.client.CreateIndex(rule.ElsIndex).Body(body).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if !ret.Acknowledged { |
|||
return errors.Errorf("create index %s err", rule.ElsIndex) |
|||
} |
|||
logs.Infof("create index succeed, index: %s", body) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) updateIndexMapping(rule *global.Rule) error { |
|||
ret, err := s.client.GetMapping().Index(rule.ElsIndex).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
if ret[rule.ElsIndex]==nil{ |
|||
return nil |
|||
} |
|||
retIndex := ret[rule.ElsIndex].(map[string]interface{}) |
|||
|
|||
if retIndex["mappings"] == nil { |
|||
return nil |
|||
} |
|||
retMaps := retIndex["mappings"].(map[string]interface{}) |
|||
|
|||
if retMaps["_doc"]==nil{ |
|||
return nil |
|||
} |
|||
retDoc := retMaps["_doc"].(map[string]interface{}) |
|||
|
|||
if retDoc["properties"]==nil{ |
|||
return nil |
|||
} |
|||
retPros := retDoc["properties"].(map[string]interface{}) |
|||
|
|||
var currents map[string]interface{} |
|||
if rule.LuaEnable() { |
|||
currents = buildPropertiesByMappings(rule) |
|||
} else { |
|||
currents = buildPropertiesByRule(rule) |
|||
} |
|||
|
|||
if len(retPros) < len(currents) { |
|||
properties := make(map[string]interface{}) |
|||
mapping := map[string]interface{}{ |
|||
"properties": properties, |
|||
} |
|||
for field, current := range currents { |
|||
if _, exist := retPros[field]; !exist { |
|||
properties[field] = current |
|||
} |
|||
} |
|||
doc := stringutil.ToJsonString(mapping) |
|||
ret, err := s.client.PutMapping().Index(rule.ElsIndex).Type(rule.ElsType).BodyString(doc).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if !ret.Acknowledged { |
|||
return errors.Errorf("update index %s err", rule.ElsIndex) |
|||
} |
|||
logs.Infof("update index succeed, index: %s", doc) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) Ping() error { |
|||
if _, _, err := s.client.Ping(s.first).Do(context.Background()); err == nil { |
|||
return nil |
|||
} |
|||
|
|||
for _, host := range s.hosts { |
|||
if _, _, err := s.client.Ping(host).Do(context.Background()); err == nil { |
|||
return nil |
|||
} |
|||
} |
|||
|
|||
return errors.New("ssx") |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
bulk := s.client.Bulk() |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoESOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
for _, resp := range ls { |
|||
logs.Infof("action: %s, Index: %s , Id:%s, value: %v", resp.Action, resp.Index, resp.Id, resp.Date) |
|||
s.prepareBulk(resp.Action, resp.Index, rule.ElsType, resp.Id, resp.Date, bulk) |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
body := encodeValue(rule, kvm) |
|||
logs.Infof("action: %s, Index: %s , Id:%s, value: %v", row.Action, rule.ElsIndex, id, body) |
|||
s.prepareBulk(row.Action, rule.ElsIndex, rule.ElsType, stringutil.ToString(id), body, bulk) |
|||
} |
|||
} |
|||
|
|||
if bulk.NumberOfActions()==0{ |
|||
return nil |
|||
} |
|||
|
|||
r, err := bulk.Do(context.Background()) |
|||
if err != nil { |
|||
log.Println(err.Error()) |
|||
return err |
|||
} |
|||
|
|||
if len(r.Failed()) > 0 { |
|||
for _, f := range r.Failed() { |
|||
reason := f.Index + " " + f.Type + " " + f.Result |
|||
if f.Error == nil && "not_found" == f.Result { |
|||
return nil |
|||
} |
|||
|
|||
if f.Error != nil { |
|||
reason = f.Error.Reason |
|||
} |
|||
log.Println(reason) |
|||
return errors.New(reason) |
|||
} |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) Stock(rows []*model.RowRequest) int64 { |
|||
if len(rows) == 0 { |
|||
return 0 |
|||
} |
|||
|
|||
bulk := s.client.Bulk() |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoESOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
for _, resp := range ls { |
|||
s.prepareBulk(resp.Action, resp.Index, rule.ElsType, resp.Id, resp.Date, bulk) |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
body := encodeValue(rule, kvm) |
|||
s.prepareBulk(row.Action, rule.ElsIndex, rule.ElsType, stringutil.ToString(id), body, bulk) |
|||
} |
|||
} |
|||
|
|||
r, err := bulk.Do(context.Background()) |
|||
if err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
return 0 |
|||
} |
|||
|
|||
return int64(len(r.Succeeded())) |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) prepareBulk(action, index, _type, id, doc string, bulk *elastic.BulkService) { |
|||
switch action { |
|||
case canal.InsertAction: |
|||
req := elastic.NewBulkIndexRequest().Index(index).Type(_type).Id(id).Doc(doc) |
|||
bulk.Add(req) |
|||
case canal.UpdateAction: |
|||
req := elastic.NewBulkUpdateRequest().Index(index).Type(_type).Id(id).Doc(doc) |
|||
bulk.Add(req) |
|||
case canal.DeleteAction: |
|||
req := elastic.NewBulkDeleteRequest().Index(index).Type(_type).Id(id) |
|||
bulk.Add(req) |
|||
} |
|||
logs.Infof("index: %s, type:%s, action:%s, doc: %s", index, _type, action, doc) |
|||
} |
|||
|
|||
func (s *Elastic6Endpoint) Close() { |
|||
if s.client != nil { |
|||
s.client.Stop() |
|||
} |
|||
} |
|||
@ -0,0 +1,312 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"context" |
|||
"log" |
|||
"sync" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/olivere/elastic/v7" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logagent" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
type Elastic7Endpoint struct { |
|||
first string |
|||
hosts []string |
|||
client *elastic.Client |
|||
|
|||
retryLock sync.Mutex |
|||
} |
|||
|
|||
func newElastic7Endpoint() *Elastic7Endpoint { |
|||
hosts := elsHosts(global.Cfg().ElsAddr) |
|||
r := &Elastic7Endpoint{} |
|||
r.hosts = hosts |
|||
r.first = hosts[0] |
|||
return r |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) Connect() error { |
|||
var options []elastic.ClientOptionFunc |
|||
options = append(options, elastic.SetErrorLog(logagent.NewElsLoggerAgent())) |
|||
options = append(options, elastic.SetURL(s.hosts...)) |
|||
if global.Cfg().ElsUser != "" && global.Cfg().ElsPassword != "" { |
|||
options = append(options, elastic.SetBasicAuth(global.Cfg().ElsUser, global.Cfg().Password)) |
|||
} |
|||
|
|||
client, err := elastic.NewClient(options...) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.client = client |
|||
return s.indexMapping() |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) indexMapping() error { |
|||
for _, rule := range global.RuleInsList() { |
|||
exists, err := s.client.IndexExists(rule.ElsIndex).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if exists { |
|||
err = s.updateIndexMapping(rule) |
|||
} else { |
|||
err = s.insertIndexMapping(rule) |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) insertIndexMapping(rule *global.Rule) error { |
|||
var properties map[string]interface{} |
|||
if rule.LuaEnable() { |
|||
properties = buildPropertiesByMappings(rule) |
|||
} else { |
|||
properties = buildPropertiesByRule(rule) |
|||
} |
|||
|
|||
mapping := map[string]interface{}{ |
|||
"mappings": map[string]interface{}{ |
|||
"properties": properties, |
|||
}, |
|||
} |
|||
body := stringutil.ToJsonString(mapping) |
|||
|
|||
ret, err := s.client.CreateIndex(rule.ElsIndex).Body(body).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if !ret.Acknowledged { |
|||
return errors.Errorf("create index %s err", rule.ElsIndex) |
|||
} |
|||
|
|||
logs.Infof("create index: %s ,mappings: %s", rule.ElsIndex, body) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) updateIndexMapping(rule *global.Rule) error { |
|||
ret, err := s.client.GetMapping().Index(rule.ElsIndex).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
if ret[rule.ElsIndex]==nil{ |
|||
return nil |
|||
} |
|||
retIndex := ret[rule.ElsIndex].(map[string]interface{}) |
|||
|
|||
if retIndex["mappings"] == nil { |
|||
return nil |
|||
} |
|||
retMaps := retIndex["mappings"].(map[string]interface{}) |
|||
|
|||
if retMaps["properties"] == nil { |
|||
return nil |
|||
} |
|||
retPros := retMaps["properties"].(map[string]interface{}) |
|||
|
|||
var currents map[string]interface{} |
|||
if rule.LuaEnable() { |
|||
currents = buildPropertiesByMappings(rule) |
|||
} else { |
|||
currents = buildPropertiesByRule(rule) |
|||
} |
|||
|
|||
if len(retPros) < len(currents) { |
|||
properties := make(map[string]interface{}) |
|||
mapping := map[string]interface{}{ |
|||
"properties": properties, |
|||
} |
|||
for field, current := range currents { |
|||
if _, exist := retPros[field]; !exist { |
|||
properties[field] = current |
|||
} |
|||
} |
|||
|
|||
doc := stringutil.ToJsonString(mapping) |
|||
ret, err := s.client.PutMapping().Index(rule.ElsIndex).BodyString(doc).Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if !ret.Acknowledged { |
|||
return errors.Errorf("update index %s err", rule.ElsIndex) |
|||
} |
|||
|
|||
logs.Infof("update index: %s ,properties: %s", rule.ElsIndex, doc) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) Ping() error { |
|||
if _, _, err := s.client.Ping(s.first).Do(context.Background()); err == nil { |
|||
return nil |
|||
} |
|||
|
|||
for _, host := range s.hosts { |
|||
if _, _, err := s.client.Ping(host).Do(context.Background()); err == nil { |
|||
return nil |
|||
} |
|||
} |
|||
|
|||
return errors.New("ssx") |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
bulk := s.client.Bulk() |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoESOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
for _, resp := range ls { |
|||
logs.Infof("action: %s, Index: %s , Id:%s, value: %v", resp.Action, resp.Index, resp.Id, resp.Date) |
|||
s.prepareBulk(resp.Action, resp.Index, resp.Id, resp.Date, bulk) |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
body := encodeValue(rule, kvm) |
|||
logs.Infof("action: %s, Index: %s , Id:%s, value: %v", row.Action, rule.ElsIndex, id, body) |
|||
s.prepareBulk(row.Action, rule.ElsIndex, stringutil.ToString(id), body, bulk) |
|||
} |
|||
} |
|||
|
|||
if bulk.NumberOfActions() == 0 { |
|||
return nil |
|||
} |
|||
|
|||
r, err := bulk.Do(context.Background()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
if len(r.Failed()) > 0 { |
|||
for _, f := range r.Failed() { |
|||
reason := f.Index + " " + f.Type + " " + f.Result |
|||
if f.Error == nil && "not_found" == f.Result { |
|||
return nil |
|||
} |
|||
|
|||
if f.Error != nil { |
|||
reason = f.Error.Reason |
|||
} |
|||
log.Println(reason) |
|||
return errors.New(reason) |
|||
} |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) Stock(rows []*model.RowRequest) int64 { |
|||
if len(rows) == 0 { |
|||
return 0 |
|||
} |
|||
|
|||
bulk := s.client.Bulk() |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoESOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
for _, resp := range ls { |
|||
s.prepareBulk(resp.Action, resp.Index, resp.Id, resp.Date, bulk) |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
body := encodeValue(rule, kvm) |
|||
s.prepareBulk(row.Action, rule.ElsIndex, stringutil.ToString(id), body, bulk) |
|||
} |
|||
} |
|||
|
|||
r, err := bulk.Do(context.Background()) |
|||
if err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
return 0 |
|||
} |
|||
|
|||
if len(r.Failed()) > 0 { |
|||
for _, f := range r.Failed() { |
|||
logs.Error(f.Error.Reason) |
|||
} |
|||
} |
|||
|
|||
return int64(len(r.Succeeded())) |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) prepareBulk(action, index, id, doc string, bulk *elastic.BulkService) { |
|||
switch action { |
|||
case canal.InsertAction: |
|||
req := elastic.NewBulkIndexRequest().Index(index).Id(id).Doc(doc) |
|||
bulk.Add(req) |
|||
case canal.UpdateAction: |
|||
req := elastic.NewBulkUpdateRequest().Index(index).Id(id).Doc(doc) |
|||
bulk.Add(req) |
|||
case canal.DeleteAction: |
|||
req := elastic.NewBulkDeleteRequest().Index(index).Id(id) |
|||
bulk.Add(req) |
|||
} |
|||
|
|||
logs.Infof("index: %s, doc: %s", index, doc) |
|||
} |
|||
|
|||
func (s *Elastic7Endpoint) Close() { |
|||
if s.client != nil { |
|||
s.client.Stop() |
|||
} |
|||
} |
|||
@ -0,0 +1,400 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"bytes" |
|||
"strconv" |
|||
"strings" |
|||
"time" |
|||
|
|||
jsoniter "github.com/json-iterator/go" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
"github.com/siddontang/go-mysql/schema" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
var json = jsoniter.ConfigCompatibleWithStandardLibrary |
|||
|
|||
const defaultDateFormatter = "2006-01-02" |
|||
|
|||
type Endpoint interface { |
|||
Connect() error |
|||
Ping() error |
|||
Consume(mysql.Position, []*model.RowRequest) error |
|||
Stock([]*model.RowRequest) int64 |
|||
Close() |
|||
} |
|||
|
|||
func NewEndpoint(ds *canal.Canal) Endpoint { |
|||
cfg := global.Cfg() |
|||
luaengine.InitActuator(ds) |
|||
|
|||
if cfg.IsRedis() { |
|||
return newRedisEndpoint() |
|||
} |
|||
|
|||
if cfg.IsMongodb() { |
|||
return newMongoEndpoint() |
|||
} |
|||
|
|||
if cfg.IsRocketmq() { |
|||
return newRocketEndpoint() |
|||
} |
|||
|
|||
if cfg.IsRabbitmq() { |
|||
return newRabbitEndpoint() |
|||
} |
|||
|
|||
if cfg.IsKafka() { |
|||
return newKafkaEndpoint() |
|||
} |
|||
|
|||
if cfg.IsEls() { |
|||
if cfg.ElsVersion == 6 { |
|||
return newElastic6Endpoint() |
|||
} |
|||
if cfg.ElsVersion == 7 { |
|||
return newElastic7Endpoint() |
|||
} |
|||
} |
|||
|
|||
if cfg.IsScript() { |
|||
return newScriptEndpoint() |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func convertColumnData(value interface{}, col *schema.TableColumn, rule *global.Rule) interface{} { |
|||
if value == nil { |
|||
return nil |
|||
} |
|||
|
|||
switch col.Type { |
|||
case schema.TYPE_ENUM: |
|||
switch value := value.(type) { |
|||
case int64: |
|||
eNum := value - 1 |
|||
if eNum < 0 || eNum >= int64(len(col.EnumValues)) { |
|||
// we insert invalid enum value before, so return empty
|
|||
logs.Warnf("invalid binlog enum index %d, for enum %v", eNum, col.EnumValues) |
|||
return "" |
|||
} |
|||
return col.EnumValues[eNum] |
|||
case string: |
|||
return value |
|||
case []byte: |
|||
return string(value) |
|||
} |
|||
case schema.TYPE_SET: |
|||
switch value := value.(type) { |
|||
case int64: |
|||
bitmask := value |
|||
sets := make([]string, 0, len(col.SetValues)) |
|||
for i, s := range col.SetValues { |
|||
if bitmask&int64(1<<uint(i)) > 0 { |
|||
sets = append(sets, s) |
|||
} |
|||
} |
|||
return strings.Join(sets, ",") |
|||
} |
|||
case schema.TYPE_BIT: |
|||
switch value := value.(type) { |
|||
case string: |
|||
if value == "\x01" { |
|||
return int64(1) |
|||
} |
|||
return int64(0) |
|||
} |
|||
case schema.TYPE_STRING: |
|||
switch value := value.(type) { |
|||
case []byte: |
|||
return string(value[:]) |
|||
} |
|||
case schema.TYPE_JSON: |
|||
var f interface{} |
|||
var err error |
|||
switch v := value.(type) { |
|||
case string: |
|||
err = json.Unmarshal([]byte(v), &f) |
|||
case []byte: |
|||
err = json.Unmarshal(v, &f) |
|||
} |
|||
if err == nil && f != nil { |
|||
return f |
|||
} |
|||
case schema.TYPE_DATETIME, schema.TYPE_TIMESTAMP: |
|||
var vv string |
|||
switch v := value.(type) { |
|||
case string: |
|||
vv = v |
|||
case []byte: |
|||
vv = string(v) |
|||
} |
|||
if rule.DatetimeFormatter != "" { |
|||
vt, err := time.Parse(mysql.TimeFormat, vv) |
|||
if err != nil || vt.IsZero() { // failed to parse date or zero date
|
|||
return nil |
|||
} |
|||
return vt.Format(rule.DatetimeFormatter) |
|||
} |
|||
return vv |
|||
case schema.TYPE_DATE: |
|||
var vv string |
|||
switch v := value.(type) { |
|||
case string: |
|||
vv = v |
|||
case []byte: |
|||
vv = string(v) |
|||
} |
|||
if rule.DateFormatter != "" { |
|||
vt, err := time.Parse(defaultDateFormatter, vv) |
|||
if err != nil || vt.IsZero() { // failed to parse date or zero date
|
|||
return nil |
|||
} |
|||
return vt.Format(rule.DateFormatter) |
|||
} |
|||
return vv |
|||
case schema.TYPE_NUMBER: |
|||
switch v := value.(type) { |
|||
case string: |
|||
vv, err := strconv.ParseInt(v, 10, 64) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
return nil |
|||
} |
|||
return vv |
|||
case []byte: |
|||
str := string(v) |
|||
vv, err := strconv.ParseInt(str, 10, 64) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
return nil |
|||
} |
|||
return vv |
|||
} |
|||
case schema.TYPE_DECIMAL, schema.TYPE_FLOAT: |
|||
switch v := value.(type) { |
|||
case string: |
|||
vv, err := strconv.ParseFloat(v, 64) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
return nil |
|||
} |
|||
return vv |
|||
case []byte: |
|||
str := string(v) |
|||
vv, err := strconv.ParseFloat(str, 64) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
return nil |
|||
} |
|||
return vv |
|||
} |
|||
} |
|||
|
|||
return value |
|||
} |
|||
|
|||
func encodeValue(rule *global.Rule, kv map[string]interface{}) string { |
|||
if rule.ValueTmpl != nil { |
|||
var tmplBytes bytes.Buffer |
|||
err := rule.ValueTmpl.Execute(&tmplBytes, kv) |
|||
if err != nil { |
|||
return "" |
|||
} |
|||
return tmplBytes.String() |
|||
} |
|||
|
|||
var val string |
|||
switch rule.ValueEncoder { |
|||
case global.ValEncoderJson: |
|||
data, err := json.Marshal(kv) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
val = "" |
|||
} else { |
|||
val = string(data) |
|||
} |
|||
case global.ValEncoderKVCommas: |
|||
var ls []string |
|||
for k, v := range kv { |
|||
str := stringutil.ToString(k) + "=" + stringutil.ToString(v) |
|||
ls = append(ls, str) |
|||
} |
|||
val = strings.Join(ls, ",") |
|||
case global.ValEncoderVCommas: |
|||
var ls []string |
|||
for _, v := range kv { |
|||
ls = append(ls, stringutil.ToString(v)) |
|||
} |
|||
val = strings.Join(ls, ",") |
|||
} |
|||
|
|||
return val |
|||
} |
|||
|
|||
func rowMap(req *model.RowRequest, rule *global.Rule, primitive bool) map[string]interface{} { |
|||
kv := make(map[string]interface{}, len(rule.PaddingMap)) |
|||
|
|||
if rule.DefaultColumnValueConfig != "" { |
|||
for k, v := range rule.DefaultColumnValueMap { |
|||
if primitive { |
|||
kv[k] = v |
|||
} else { |
|||
kv[rule.WrapName(k)] = v |
|||
} |
|||
} |
|||
} |
|||
|
|||
if primitive { |
|||
for _, padding := range rule.PaddingMap { |
|||
kv[padding.ColumnName] = convertColumnData(req.Row[padding.ColumnIndex], padding.ColumnMetadata, rule) |
|||
} |
|||
} else { |
|||
for _, padding := range rule.PaddingMap { |
|||
kv[padding.WrapName] = convertColumnData(req.Row[padding.ColumnIndex], padding.ColumnMetadata, rule) |
|||
} |
|||
} |
|||
return kv |
|||
} |
|||
|
|||
func oldRowMap(req *model.RowRequest, rule *global.Rule, primitive bool) map[string]interface{} { |
|||
kv := make(map[string]interface{}, len(rule.PaddingMap)) |
|||
|
|||
if rule.DefaultColumnValueConfig != "" { |
|||
for k, v := range rule.DefaultColumnValueMap { |
|||
if primitive { |
|||
kv[k] = v |
|||
} else { |
|||
kv[rule.WrapName(k)] = v |
|||
} |
|||
} |
|||
} |
|||
|
|||
if primitive { |
|||
for _, padding := range rule.PaddingMap { |
|||
kv[padding.ColumnName] = convertColumnData(req.Old[padding.ColumnIndex], padding.ColumnMetadata, rule) |
|||
} |
|||
} else { |
|||
for _, padding := range rule.PaddingMap { |
|||
kv[padding.WrapName] = convertColumnData(req.Old[padding.ColumnIndex], padding.ColumnMetadata, rule) |
|||
} |
|||
} |
|||
return kv |
|||
} |
|||
|
|||
func primaryKey(re *model.RowRequest, rule *global.Rule) interface{} { |
|||
if rule.IsCompositeKey { // 组合ID
|
|||
var key string |
|||
for _, index := range rule.TableInfo.PKColumns { |
|||
key += stringutil.ToString(re.Row[index]) |
|||
} |
|||
return key |
|||
} else { |
|||
index := rule.TableInfo.PKColumns[0] |
|||
data := re.Row[index] |
|||
column := rule.TableInfo.Columns[index] |
|||
return convertColumnData(data, &column, rule) |
|||
} |
|||
} |
|||
|
|||
func elsHosts(addr string) []string { |
|||
var hosts []string |
|||
splits := strings.Split(addr, ",") |
|||
for _, split := range splits { |
|||
if !strings.HasPrefix(split, "http:") { |
|||
hosts = append(hosts, "http://"+split) |
|||
} else { |
|||
hosts = append(hosts, split) |
|||
} |
|||
} |
|||
|
|||
return hosts |
|||
} |
|||
|
|||
func buildPropertiesByRule(rule *global.Rule) map[string]interface{} { |
|||
properties := make(map[string]interface{}) |
|||
for _, padding := range rule.PaddingMap { |
|||
property := make(map[string]interface{}) |
|||
switch padding.ColumnType { |
|||
case schema.TYPE_BINARY: |
|||
property["type"] = "binary" |
|||
case schema.TYPE_NUMBER: |
|||
property["type"] = "long" |
|||
case schema.TYPE_DECIMAL: |
|||
property["type"] = "double" |
|||
case schema.TYPE_FLOAT: |
|||
property["type"] = "float" |
|||
case schema.TYPE_DATE: |
|||
property["type"] = "date" |
|||
property["format"] = "yyyy-MM-dd" |
|||
case schema.TYPE_DATETIME, schema.TYPE_TIMESTAMP: |
|||
property["type"] = "date" |
|||
property["format"] = "yyyy-MM-dd HH:mm:ss" |
|||
default: |
|||
property["type"] = "keyword" |
|||
} |
|||
properties[padding.WrapName] = property |
|||
} |
|||
|
|||
if len(rule.DefaultColumnValueMap) > 0 { |
|||
for key, _ := range rule.DefaultColumnValueMap { |
|||
property := make(map[string]interface{}) |
|||
property["type"] = "keyword" |
|||
properties[key] = property |
|||
} |
|||
} |
|||
|
|||
for _, mapping := range rule.EsMappings { |
|||
property := make(map[string]interface{}) |
|||
property["type"] = mapping.Type |
|||
if mapping.Format != "" { |
|||
property["format"] = mapping.Format |
|||
} |
|||
if mapping.Analyzer != "" { |
|||
property["analyzer"] = mapping.Analyzer |
|||
} |
|||
properties[mapping.Field] = property |
|||
} |
|||
|
|||
return properties |
|||
} |
|||
|
|||
func buildPropertiesByMappings(rule *global.Rule) map[string]interface{} { |
|||
properties := make(map[string]interface{}) |
|||
for _, mapping := range rule.EsMappings { |
|||
property := make(map[string]interface{}) |
|||
property["type"] = mapping.Type |
|||
if mapping.Format != "" { |
|||
property["format"] = mapping.Format |
|||
} |
|||
if mapping.Analyzer != "" { |
|||
property["analyzer"] = mapping.Analyzer |
|||
} |
|||
properties[mapping.Field] = property |
|||
} |
|||
return properties |
|||
} |
|||
@ -0,0 +1,239 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"log" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"github.com/Shopify/sarama" |
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
type KafkaEndpoint struct { |
|||
client sarama.Client |
|||
producer sarama.AsyncProducer |
|||
|
|||
retryLock sync.Mutex |
|||
} |
|||
|
|||
func newKafkaEndpoint() *KafkaEndpoint { |
|||
r := &KafkaEndpoint{} |
|||
return r |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) Connect() error { |
|||
cfg := sarama.NewConfig() |
|||
cfg.Producer.Partitioner = sarama.NewRandomPartitioner |
|||
|
|||
if global.Cfg().KafkaSASLUser != "" && global.Cfg().KafkaSASLPassword != "" { |
|||
cfg.Net.SASL.Enable = true |
|||
cfg.Net.SASL.User = global.Cfg().KafkaSASLUser |
|||
cfg.Net.SASL.Password = global.Cfg().KafkaSASLPassword |
|||
} |
|||
|
|||
var err error |
|||
var client sarama.Client |
|||
ls := strings.Split(global.Cfg().KafkaAddr, ",") |
|||
client, err = sarama.NewClient(ls, cfg) |
|||
if err != nil { |
|||
return errors.Errorf("unable to create kafka client: %q", err) |
|||
} |
|||
|
|||
var producer sarama.AsyncProducer |
|||
producer, err = sarama.NewAsyncProducerFromClient(client) |
|||
if err != nil { |
|||
return errors.Errorf("unable to create kafka producer: %q", err) |
|||
} |
|||
|
|||
s.producer = producer |
|||
s.client = client |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) Ping() error { |
|||
return s.client.RefreshMetadata() |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
var ms []*sarama.ProducerMessage |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
ls, err := s.buildMessages(row, rule) |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
ms = append(ms, ls...) |
|||
} else { |
|||
m, err := s.buildMessage(row, rule) |
|||
if err != nil { |
|||
return errors.Errorf(errors.ErrorStack(err)) |
|||
} |
|||
ms = append(ms, m) |
|||
} |
|||
} |
|||
|
|||
for _, m := range ms { |
|||
s.producer.Input() <- m |
|||
select { |
|||
case err := <-s.producer.Errors(): |
|||
return err |
|||
default: |
|||
} |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) Stock(rows []*model.RowRequest) int64 { |
|||
expect := true |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
ls, err := s.buildMessages(row, rule) |
|||
if err != nil { |
|||
logs.Errorf(errors.ErrorStack(err)) |
|||
expect = false |
|||
break |
|||
} |
|||
for _, m := range ls { |
|||
s.producer.Input() <- m |
|||
select { |
|||
case err := <-s.producer.Errors(): |
|||
logs.Error(err.Error()) |
|||
expect = false |
|||
break |
|||
default: |
|||
} |
|||
} |
|||
if !expect { |
|||
break |
|||
} |
|||
} else { |
|||
m, err := s.buildMessage(row, rule) |
|||
if err != nil { |
|||
logs.Errorf(errors.ErrorStack(err)) |
|||
expect = false |
|||
break |
|||
} |
|||
s.producer.Input() <- m |
|||
select { |
|||
case err := <-s.producer.Errors(): |
|||
logs.Error(err.Error()) |
|||
expect = false |
|||
break |
|||
default: |
|||
|
|||
} |
|||
} |
|||
} |
|||
|
|||
if !expect { |
|||
return 0 |
|||
} |
|||
|
|||
return int64(len(rows)) |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) buildMessages(row *model.RowRequest, rule *global.Rule) ([]*sarama.ProducerMessage, error) { |
|||
var err error |
|||
var ls []*model.MQRespond |
|||
kvm := rowMap(row, rule, true) |
|||
if row.Action == canal.UpdateAction { |
|||
previous := oldRowMap(row, rule, true) |
|||
ls, err = luaengine.DoMQOps(kvm, previous, row.Action, rule) |
|||
} else { |
|||
ls, err = luaengine.DoMQOps(kvm, nil, row.Action, rule) |
|||
} |
|||
if err != nil { |
|||
return nil, errors.Errorf("lua 脚本执行失败 : %s ", err) |
|||
} |
|||
|
|||
var ms []*sarama.ProducerMessage |
|||
for _, resp := range ls { |
|||
m := &sarama.ProducerMessage{ |
|||
Topic: resp.Topic, |
|||
Value: sarama.ByteEncoder(resp.ByteArray), |
|||
} |
|||
logs.Infof("topic: %s, message: %s", resp.Topic, string(resp.ByteArray)) |
|||
ms = append(ms, m) |
|||
} |
|||
|
|||
return ms, nil |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) buildMessage(row *model.RowRequest, rule *global.Rule) (*sarama.ProducerMessage, error) { |
|||
kvm := rowMap(row, rule, false) |
|||
resp := new(model.MQRespond) |
|||
resp.Action = row.Action |
|||
resp.Timestamp = row.Timestamp |
|||
if rule.ValueEncoder == global.ValEncoderJson { |
|||
resp.Date = kvm |
|||
} else { |
|||
resp.Date = encodeValue(rule, kvm) |
|||
} |
|||
|
|||
if rule.ReserveRawData && canal.UpdateAction == row.Action { |
|||
resp.Raw = oldRowMap(row, rule, false) |
|||
} |
|||
|
|||
body, err := json.Marshal(resp) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
m := &sarama.ProducerMessage{ |
|||
Topic: rule.KafkaTopic, |
|||
Value: sarama.ByteEncoder(body), |
|||
} |
|||
logs.Infof("topic: %s, message: %s", rule.KafkaTopic, string(body)) |
|||
return m, nil |
|||
} |
|||
|
|||
func (s *KafkaEndpoint) Close() { |
|||
if s.producer != nil { |
|||
s.producer.Close() |
|||
} |
|||
if s.client != nil { |
|||
s.client.Close() |
|||
} |
|||
} |
|||
@ -0,0 +1,377 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"context" |
|||
"log" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
"go.mongodb.org/mongo-driver/bson" |
|||
"go.mongodb.org/mongo-driver/mongo" |
|||
"go.mongodb.org/mongo-driver/mongo/options" |
|||
"go.mongodb.org/mongo-driver/mongo/readpref" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
type cKey struct { |
|||
database string |
|||
collection string |
|||
} |
|||
|
|||
type MongoEndpoint struct { |
|||
options *options.ClientOptions |
|||
client *mongo.Client |
|||
lock sync.Mutex |
|||
collections map[cKey]*mongo.Collection |
|||
collLock sync.RWMutex |
|||
|
|||
retryLock sync.Mutex |
|||
} |
|||
|
|||
func newMongoEndpoint() *MongoEndpoint { |
|||
addrList := strings.Split(global.Cfg().MongodbAddr, ",") |
|||
opts := &options.ClientOptions{ |
|||
Hosts: addrList, |
|||
} |
|||
|
|||
if global.Cfg().MongodbUsername != "" && global.Cfg().MongodbPassword != "" { |
|||
opts.Auth = &options.Credential{ |
|||
Username: global.Cfg().MongodbUsername, |
|||
Password: global.Cfg().MongodbPassword, |
|||
} |
|||
} |
|||
|
|||
r := &MongoEndpoint{} |
|||
r.options = opts |
|||
r.collections = make(map[cKey]*mongo.Collection) |
|||
return r |
|||
} |
|||
|
|||
func (s *MongoEndpoint) Connect() error { |
|||
client, err := mongo.Connect(context.Background(), s.options) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.client = client |
|||
|
|||
s.collLock.Lock() |
|||
for _, rule := range global.RuleInsList() { |
|||
cc := s.client.Database(rule.MongodbDatabase).Collection(rule.MongodbCollection) |
|||
s.collections[s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection)] = cc |
|||
} |
|||
s.collLock.Unlock() |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *MongoEndpoint) Ping() error { |
|||
return s.client.Ping(context.Background(), readpref.Primary()) |
|||
} |
|||
|
|||
func (s *MongoEndpoint) isDuplicateKeyError(stack string) bool { |
|||
return strings.Contains(stack, "E11000 duplicate key error") |
|||
} |
|||
|
|||
func (s *MongoEndpoint) collectionKey(database, collection string) cKey { |
|||
return cKey{ |
|||
database: database, |
|||
collection: collection, |
|||
} |
|||
} |
|||
|
|||
func (s *MongoEndpoint) collection(key cKey) *mongo.Collection { |
|||
s.collLock.RLock() |
|||
c, ok := s.collections[key] |
|||
s.collLock.RUnlock() |
|||
if ok { |
|||
return c |
|||
} |
|||
|
|||
s.collLock.Lock() |
|||
c = s.client.Database(key.database).Collection(key.collection) |
|||
s.collections[key] = c |
|||
s.collLock.Unlock() |
|||
|
|||
return c |
|||
} |
|||
|
|||
func (s *MongoEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
models := make(map[cKey][]mongo.WriteModel, 0) |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoMongoOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
for _, resp := range ls { |
|||
var model mongo.WriteModel |
|||
switch resp.Action { |
|||
case canal.InsertAction: |
|||
model = mongo.NewInsertOneModel().SetDocument(resp.Table) |
|||
case canal.UpdateAction: |
|||
model = mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": resp.Id}).SetUpdate(bson.M{"$set": resp.Table}) |
|||
case global.UpsertAction: |
|||
model = mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": resp.Id}).SetUpsert(true).SetUpdate(bson.M{"$set": resp.Table}) |
|||
case canal.DeleteAction: |
|||
model = mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": resp.Id}) |
|||
} |
|||
|
|||
key := s.collectionKey(rule.MongodbDatabase, resp.Collection) |
|||
array, ok := models[key] |
|||
if !ok { |
|||
array = make([]mongo.WriteModel, 0) |
|||
} |
|||
|
|||
logs.Infof("action:%s, collection:%s, id:%v, data:%v", resp.Action, resp.Collection, resp.Id, resp.Table) |
|||
|
|||
array = append(array, model) |
|||
models[key] = array |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
kvm["_id"] = id |
|||
var model mongo.WriteModel |
|||
switch row.Action { |
|||
case canal.InsertAction: |
|||
model = mongo.NewInsertOneModel().SetDocument(kvm) |
|||
case canal.UpdateAction: |
|||
model = mongo.NewUpdateOneModel().SetFilter(bson.M{"_id": id}).SetUpdate(bson.M{"$set": kvm}) |
|||
case canal.DeleteAction: |
|||
model = mongo.NewDeleteOneModel().SetFilter(bson.M{"_id": id}) |
|||
} |
|||
|
|||
ccKey := s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection) |
|||
array, ok := models[ccKey] |
|||
if !ok { |
|||
array = make([]mongo.WriteModel, 0) |
|||
} |
|||
|
|||
logs.Infof("action:%s, collection:%s, id:%v, data:%v", row.Action, rule.MongodbCollection, id, kvm) |
|||
|
|||
array = append(array, model) |
|||
models[ccKey] = array |
|||
} |
|||
} |
|||
|
|||
var slowly bool |
|||
for key, model := range models { |
|||
collection := s.collection(key) |
|||
_, err := collection.BulkWrite(context.Background(), model) |
|||
if err != nil { |
|||
if s.isDuplicateKeyError(err.Error()) { |
|||
slowly = true |
|||
} else { |
|||
return err |
|||
} |
|||
logs.Error(errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
} |
|||
if slowly { |
|||
_, err := s.doConsumeSlowly(rows) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *MongoEndpoint) Stock(rows []*model.RowRequest) int64 { |
|||
expect := true |
|||
models := make(map[cKey][]mongo.WriteModel, 0) |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoMongoOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
expect = false |
|||
break |
|||
} |
|||
|
|||
for _, resp := range ls { |
|||
ccKey := s.collectionKey(rule.MongodbDatabase, resp.Collection) |
|||
model := mongo.NewInsertOneModel().SetDocument(resp.Table) |
|||
array, ok := models[ccKey] |
|||
if !ok { |
|||
array = make([]mongo.WriteModel, 0) |
|||
} |
|||
array = append(array, model) |
|||
models[ccKey] = array |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
kvm["_id"] = id |
|||
|
|||
ccKey := s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection) |
|||
model := mongo.NewInsertOneModel().SetDocument(kvm) |
|||
array, ok := models[ccKey] |
|||
if !ok { |
|||
array = make([]mongo.WriteModel, 0) |
|||
} |
|||
array = append(array, model) |
|||
models[ccKey] = array |
|||
} |
|||
} |
|||
|
|||
if !expect { |
|||
return 0 |
|||
} |
|||
|
|||
var slowly bool |
|||
var sum int64 |
|||
for key, vs := range models { |
|||
collection := s.collection(key) |
|||
rr, err := collection.BulkWrite(context.Background(), vs) |
|||
if err != nil { |
|||
if s.isDuplicateKeyError(err.Error()) { |
|||
slowly = true |
|||
} |
|||
logs.Error(errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
sum += rr.InsertedCount |
|||
} |
|||
|
|||
if slowly { |
|||
logs.Info("do consume slowly ... ... ") |
|||
slowlySum, err := s.doConsumeSlowly(rows) |
|||
if err != nil { |
|||
logs.Warnf(err.Error()) |
|||
} |
|||
return slowlySum |
|||
} |
|||
|
|||
return sum |
|||
} |
|||
|
|||
func (s *MongoEndpoint) doConsumeSlowly(rows []*model.RowRequest) (int64, error) { |
|||
var sum int64 |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoMongoOps(kvm, row.Action, rule) |
|||
if err != nil { |
|||
logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
return sum, err |
|||
} |
|||
for _, resp := range ls { |
|||
collection := s.collection(s.collectionKey(rule.MongodbDatabase, resp.Collection)) |
|||
switch resp.Action { |
|||
case canal.InsertAction: |
|||
_, err := collection.InsertOne(context.Background(), resp.Table) |
|||
if err != nil { |
|||
if s.isDuplicateKeyError(err.Error()) { |
|||
logs.Warnf("duplicate key [ %v ]", stringutil.ToJsonString(resp.Table)) |
|||
} else { |
|||
return sum, err |
|||
} |
|||
} |
|||
case canal.UpdateAction: |
|||
_, err := collection.UpdateOne(context.Background(), bson.M{"_id": resp.Id}, bson.M{"$set": resp.Table}) |
|||
if err != nil { |
|||
return sum, err |
|||
} |
|||
case canal.DeleteAction: |
|||
_, err := collection.DeleteOne(context.Background(), bson.M{"_id": resp.Id}) |
|||
if err != nil { |
|||
return sum, err |
|||
} |
|||
} |
|||
logs.Infof("action:%s, collection:%s, id:%v, data:%v", |
|||
row.Action, collection.Name(), resp.Id, resp.Table) |
|||
} |
|||
} else { |
|||
kvm := rowMap(row, rule, false) |
|||
id := primaryKey(row, rule) |
|||
kvm["_id"] = id |
|||
|
|||
collection := s.collection(s.collectionKey(rule.MongodbDatabase, rule.MongodbCollection)) |
|||
|
|||
switch row.Action { |
|||
case canal.InsertAction: |
|||
_, err := collection.InsertOne(context.Background(), kvm) |
|||
if err != nil { |
|||
if s.isDuplicateKeyError(err.Error()) { |
|||
logs.Warnf("duplicate key [ %v ]", stringutil.ToJsonString(kvm)) |
|||
} else { |
|||
return sum, err |
|||
} |
|||
} |
|||
case canal.UpdateAction: |
|||
_, err := collection.UpdateOne(context.Background(), bson.M{"_id": id}, bson.M{"$set": kvm}) |
|||
if err != nil { |
|||
return sum, err |
|||
} |
|||
case canal.DeleteAction: |
|||
_, err := collection.DeleteOne(context.Background(), bson.M{"_id": id}) |
|||
if err != nil { |
|||
return sum, err |
|||
} |
|||
} |
|||
|
|||
logs.Infof("action:%s, collection:%s, id:%v, data:%v", row.Action, collection.Name(), id, kvm) |
|||
} |
|||
sum++ |
|||
} |
|||
return sum, nil |
|||
} |
|||
|
|||
func (s *MongoEndpoint) Close() { |
|||
if s.client != nil { |
|||
s.client.Disconnect(context.Background()) |
|||
} |
|||
} |
|||
@ -0,0 +1,54 @@ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"context" |
|||
"go.mongodb.org/mongo-driver/bson" |
|||
"go.mongodb.org/mongo-driver/mongo" |
|||
"go.mongodb.org/mongo-driver/mongo/options" |
|||
"go.mongodb.org/mongo-driver/mongo/readpref" |
|||
"testing" |
|||
) |
|||
|
|||
func TestMongoPing(t *testing.T) { |
|||
opts := &options.ClientOptions{ |
|||
Hosts: []string{"127.0.0.1:27018"}, |
|||
} |
|||
|
|||
// 连接数据库
|
|||
client, err := mongo.Connect(context.Background(), opts) |
|||
if err != nil { |
|||
t.Error(err.Error()) |
|||
} |
|||
|
|||
err = client.Ping(context.Background(), readpref.Primary()) |
|||
if err != nil { |
|||
t.Error(err.Error()) |
|||
} |
|||
} |
|||
|
|||
func TestCheckData(t *testing.T) { |
|||
opts := &options.ClientOptions{ |
|||
Hosts: []string{"127.0.0.1:27018"}, |
|||
} |
|||
opts.Auth = &options.Credential{ |
|||
Username: "test", |
|||
Password: "test", |
|||
} |
|||
|
|||
// 连接数据库
|
|||
client, err := mongo.Connect(context.Background(), opts) |
|||
if err != nil { |
|||
t.Error(err.Error()) |
|||
} |
|||
|
|||
db := client.Database("test") |
|||
cc := db.Collection("ss") |
|||
|
|||
for i := 0; i < 22032; i++ { |
|||
r := cc.FindOne(context.Background(), bson.M{"_id": i}) |
|||
if r.Err() != nil { |
|||
t.Error(r.Err()) |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,232 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"log" |
|||
"strconv" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
"github.com/streadway/amqp" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/nets" |
|||
) |
|||
|
|||
type RabbitEndpoint struct { |
|||
rabCon *amqp.Connection |
|||
rabChl *amqp.Channel |
|||
queues map[string]bool |
|||
serverUrl string |
|||
} |
|||
|
|||
func newRabbitEndpoint() *RabbitEndpoint { |
|||
r := &RabbitEndpoint{} |
|||
r.queues = make(map[string]bool) |
|||
return r |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) Connect() error { |
|||
if s.rabChl != nil { |
|||
s.rabChl.Close() |
|||
s.rabChl = nil |
|||
} |
|||
if s.rabCon != nil { |
|||
s.rabCon.Close() |
|||
s.rabCon = nil |
|||
} |
|||
|
|||
con, err := amqp.Dial(global.Cfg().RabbitmqAddr) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
uri, _ := amqp.ParseURI(global.Cfg().RabbitmqAddr) |
|||
s.serverUrl = uri.Host + ":" + strconv.Itoa(uri.Port) |
|||
|
|||
var chl *amqp.Channel |
|||
chl, err = con.Channel() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.rabCon = con |
|||
s.rabChl = chl |
|||
|
|||
if len(s.queues) == 0 { |
|||
for _, rule := range global.RuleInsList() { |
|||
_, err := s.rabChl.QueueDeclare( |
|||
rule.RabbitmqQueue, false, false, false, false, nil, |
|||
) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
s.queues[rule.RabbitmqQueue] = true |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) Ping() error { |
|||
_, err := nets.IsActiveTCPAddr(s.serverUrl) |
|||
return err |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) mergeQueue(name string) { |
|||
_, ok := s.queues[name] |
|||
if ok { |
|||
return |
|||
} |
|||
|
|||
s.rabChl.QueueDeclare(name, false, false, false, false, nil) |
|||
s.queues[name] = true |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
err := s.doLuaConsume(row, rule) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} else { |
|||
err := s.doRuleConsume(row, rule) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) Stock(rows []*model.RowRequest) int64 { |
|||
var sum int64 |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
err := s.doLuaConsume(row, rule) |
|||
if err != nil { |
|||
logs.Errorf(errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
} else { |
|||
err := s.doRuleConsume(row, rule) |
|||
if err != nil { |
|||
logs.Errorf(errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
} |
|||
sum++ |
|||
} |
|||
|
|||
return sum |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) doLuaConsume(req *model.RowRequest, rule *global.Rule) error { |
|||
var err error |
|||
var ls []*model.MQRespond |
|||
kvm := rowMap(req, rule, true) |
|||
if req.Action == canal.UpdateAction { |
|||
previous := oldRowMap(req, rule, true) |
|||
ls, err = luaengine.DoMQOps(kvm, previous, req.Action, rule) |
|||
} else { |
|||
ls, err = luaengine.DoMQOps(kvm, nil, req.Action, rule) |
|||
} |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("lua 脚本执行失败 : %s ", err) |
|||
} |
|||
|
|||
for _, resp := range ls { |
|||
s.mergeQueue(resp.Topic) |
|||
err := s.rabChl.Publish("", resp.Topic, false, false, |
|||
amqp.Publishing{ |
|||
ContentType: "text/plain", |
|||
Body: resp.ByteArray, |
|||
}) |
|||
|
|||
logs.Infof("topic: %s, message: %s", resp.Topic, string(resp.ByteArray)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) doRuleConsume(req *model.RowRequest, rule *global.Rule) error { |
|||
kvm := rowMap(req, rule, false) |
|||
|
|||
resp := new(model.MQRespond) |
|||
resp.Action = req.Action |
|||
resp.Timestamp = req.Timestamp |
|||
if rule.ValueEncoder == global.ValEncoderJson { |
|||
resp.Date = kvm |
|||
} else { |
|||
resp.Date = encodeValue(rule, kvm) |
|||
} |
|||
|
|||
if rule.ReserveRawData && canal.UpdateAction == req.Action { |
|||
resp.Raw = oldRowMap(req, rule, false) |
|||
} |
|||
|
|||
body, err := json.Marshal(resp) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
err = s.rabChl.Publish("", rule.RabbitmqQueue, false, false, |
|||
amqp.Publishing{ |
|||
ContentType: "text/plain", |
|||
Body: body, |
|||
}) |
|||
|
|||
logs.Infof("topic: %s, message: %s", rule.RabbitmqQueue, string(body)) |
|||
|
|||
return err |
|||
} |
|||
|
|||
func (s *RabbitEndpoint) Close() { |
|||
if s.rabChl != nil { |
|||
s.rabChl.Close() |
|||
} |
|||
if s.rabCon != nil { |
|||
s.rabCon.Close() |
|||
} |
|||
} |
|||
@ -0,0 +1,333 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"bytes" |
|||
"log" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"github.com/go-redis/redis" |
|||
"github.com/pingcap/errors" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logs" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
type RedisEndpoint struct { |
|||
isCluster bool |
|||
client *redis.Client |
|||
cluster *redis.ClusterClient |
|||
retryLock sync.Mutex |
|||
} |
|||
|
|||
func newRedisEndpoint() *RedisEndpoint { |
|||
cfg := global.Cfg() |
|||
r := &RedisEndpoint{} |
|||
|
|||
list := strings.Split(cfg.RedisAddr, ",") |
|||
if len(list) == 1 { |
|||
r.client = redis.NewClient(&redis.Options{ |
|||
Addr: cfg.RedisAddr, |
|||
Password: cfg.RedisPass, |
|||
DB: cfg.RedisDatabase, |
|||
}) |
|||
} else { |
|||
if cfg.RedisGroupType == global.RedisGroupTypeSentinel { |
|||
r.client = redis.NewFailoverClient(&redis.FailoverOptions{ |
|||
MasterName: cfg.RedisMasterName, |
|||
SentinelAddrs: list, |
|||
Password: cfg.RedisPass, |
|||
DB: cfg.RedisDatabase, |
|||
}) |
|||
} |
|||
if cfg.RedisGroupType == global.RedisGroupTypeCluster { |
|||
r.isCluster = true |
|||
r.cluster = redis.NewClusterClient(&redis.ClusterOptions{ |
|||
Addrs: list, |
|||
Password: cfg.RedisPass, |
|||
}) |
|||
} |
|||
} |
|||
|
|||
return r |
|||
} |
|||
|
|||
func (s *RedisEndpoint) Connect() error { |
|||
return s.Ping() |
|||
} |
|||
|
|||
func (s *RedisEndpoint) Ping() error { |
|||
var err error |
|||
if s.isCluster { |
|||
_, err = s.cluster.Ping().Result() |
|||
} else { |
|||
_, err = s.client.Ping().Result() |
|||
} |
|||
return err |
|||
} |
|||
|
|||
func (s *RedisEndpoint) pipe() redis.Pipeliner { |
|||
var pipe redis.Pipeliner |
|||
if s.isCluster { |
|||
pipe = s.cluster.Pipeline() |
|||
} else { |
|||
pipe = s.client.Pipeline() |
|||
} |
|||
return pipe |
|||
} |
|||
|
|||
func (s *RedisEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
pipe := s.pipe() |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
var err error |
|||
var ls []*model.RedisRespond |
|||
kvm := rowMap(row, rule, true) |
|||
if row.Action == canal.UpdateAction { |
|||
previous := oldRowMap(row, rule, true) |
|||
ls, err = luaengine.DoRedisOps(kvm, previous, row.Action, rule) |
|||
} else { |
|||
ls, err = luaengine.DoRedisOps(kvm, nil, row.Action, rule) |
|||
} |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("Lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
for _, resp := range ls { |
|||
s.preparePipe(resp, pipe) |
|||
logs.Infof("action: %s, structure: %s ,key: %s ,field: %s, value: %v", resp.Action, resp.Structure, resp.Key, resp.Field, resp.Val) |
|||
} |
|||
kvm = nil |
|||
} else { |
|||
resp := s.ruleRespond(row, rule) |
|||
s.preparePipe(resp, pipe) |
|||
logs.Infof("action: %s, structure: %s ,key: %s ,field: %s, value: %v", resp.Action, resp.Structure, resp.Key, resp.Field, resp.Val) |
|||
} |
|||
} |
|||
|
|||
_, err := pipe.Exec() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *RedisEndpoint) Stock(rows []*model.RowRequest) int64 { |
|||
pipe := s.pipe() |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
kvm := rowMap(row, rule, true) |
|||
ls, err := luaengine.DoRedisOps(kvm, nil, row.Action, rule) |
|||
if err != nil { |
|||
logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
for _, resp := range ls { |
|||
s.preparePipe(resp, pipe) |
|||
} |
|||
} else { |
|||
resp := s.ruleRespond(row, rule) |
|||
resp.Action = row.Action |
|||
resp.Structure = rule.RedisStructure |
|||
s.preparePipe(resp, pipe) |
|||
} |
|||
} |
|||
|
|||
var counter int64 |
|||
res, err := pipe.Exec() |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
} |
|||
|
|||
for _, re := range res { |
|||
if re.Err() == nil { |
|||
counter++ |
|||
} |
|||
} |
|||
|
|||
return counter |
|||
} |
|||
|
|||
func (s *RedisEndpoint) ruleRespond(row *model.RowRequest, rule *global.Rule) *model.RedisRespond { |
|||
resp := new(model.RedisRespond) |
|||
resp.Action = row.Action |
|||
resp.Structure = rule.RedisStructure |
|||
|
|||
kvm := rowMap(row, rule, false) |
|||
resp.Key = s.encodeKey(row, rule) |
|||
if resp.Structure == global.RedisStructureHash { |
|||
resp.Field = s.encodeHashField(row, rule) |
|||
} |
|||
if resp.Structure == global.RedisStructureSortedSet { |
|||
resp.Score = s.encodeSortedSetScoreField(row, rule) |
|||
} |
|||
|
|||
if resp.Action == canal.InsertAction { |
|||
resp.Val = encodeValue(rule, kvm) |
|||
} else if resp.Action == canal.UpdateAction { |
|||
if rule.RedisStructure == global.RedisStructureList || |
|||
rule.RedisStructure == global.RedisStructureSet || |
|||
rule.RedisStructure == global.RedisStructureSortedSet { |
|||
oldKvm := oldRowMap(row, rule, false) |
|||
resp.OldVal = encodeValue(rule, oldKvm) |
|||
} |
|||
resp.Val = encodeValue(rule, kvm) |
|||
} else { |
|||
if rule.RedisStructure == global.RedisStructureList || |
|||
rule.RedisStructure == global.RedisStructureSet || |
|||
rule.RedisStructure == global.RedisStructureSortedSet { |
|||
resp.Val = encodeValue(rule, kvm) |
|||
} |
|||
} |
|||
|
|||
return resp |
|||
} |
|||
|
|||
func (s *RedisEndpoint) preparePipe(resp *model.RedisRespond, pipe redis.Cmdable) { |
|||
switch resp.Structure { |
|||
case global.RedisStructureString: |
|||
if resp.Action == canal.DeleteAction { |
|||
pipe.Del(resp.Key) |
|||
} else { |
|||
pipe.Set(resp.Key, resp.Val, 0) |
|||
} |
|||
case global.RedisStructureHash: |
|||
if resp.Action == canal.DeleteAction { |
|||
pipe.HDel(resp.Key, resp.Field) |
|||
} else { |
|||
pipe.HSet(resp.Key, resp.Field, resp.Val) |
|||
} |
|||
case global.RedisStructureList: |
|||
if resp.Action == canal.DeleteAction { |
|||
pipe.LRem(resp.Key, 0, resp.Val) |
|||
} else if resp.Action == canal.UpdateAction { |
|||
pipe.LRem(resp.Key, 0, resp.OldVal) |
|||
pipe.RPush(resp.Key, resp.Val) |
|||
} else { |
|||
pipe.RPush(resp.Key, resp.Val) |
|||
} |
|||
case global.RedisStructureSet: |
|||
if resp.Action == canal.DeleteAction { |
|||
pipe.SRem(resp.Key, resp.Val) |
|||
} else if resp.Action == canal.UpdateAction { |
|||
pipe.SRem(resp.Key, 0, resp.OldVal) |
|||
pipe.SAdd(resp.Key, resp.Val) |
|||
} else { |
|||
pipe.SAdd(resp.Key, resp.Val) |
|||
} |
|||
case global.RedisStructureSortedSet: |
|||
if resp.Action == canal.DeleteAction { |
|||
pipe.ZRem(resp.Key, resp.Val) |
|||
} else if resp.Action == canal.UpdateAction { |
|||
pipe.ZRem(resp.Key, 0, resp.OldVal) |
|||
val := redis.Z{Score: resp.Score, Member: resp.Val} |
|||
pipe.ZAdd(resp.Key, val) |
|||
} else { |
|||
val := redis.Z{Score: resp.Score, Member: resp.Val} |
|||
pipe.ZAdd(resp.Key, val) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func (s *RedisEndpoint) encodeKey(req *model.RowRequest, rule *global.Rule) string { |
|||
if rule.RedisKeyValue != "" { |
|||
return rule.RedisKeyValue |
|||
} |
|||
|
|||
if rule.RedisKeyFormatter != "" { |
|||
kv := rowMap(req, rule, true) |
|||
var tmplBytes bytes.Buffer |
|||
err := rule.RedisKeyTmpl.Execute(&tmplBytes, kv) |
|||
if err != nil { |
|||
return "" |
|||
} |
|||
return tmplBytes.String() |
|||
} |
|||
|
|||
var key string |
|||
if rule.RedisKeyColumnIndex < 0 { |
|||
for _, v := range rule.RedisKeyColumnIndexs { |
|||
key += stringutil.ToString(req.Row[v]) |
|||
} |
|||
} else { |
|||
key = stringutil.ToString(req.Row[rule.RedisKeyColumnIndex]) |
|||
} |
|||
if rule.RedisKeyPrefix != "" { |
|||
key = rule.RedisKeyPrefix + key |
|||
} |
|||
|
|||
return key |
|||
} |
|||
|
|||
func (s *RedisEndpoint) encodeHashField(req *model.RowRequest, rule *global.Rule) string { |
|||
var field string |
|||
|
|||
if rule.RedisHashFieldColumnIndex < 0 { |
|||
for _, v := range rule.RedisHashFieldColumnIndexs { |
|||
field += stringutil.ToString(req.Row[v]) |
|||
} |
|||
} else { |
|||
field = stringutil.ToString(req.Row[rule.RedisHashFieldColumnIndex]) |
|||
} |
|||
|
|||
if rule.RedisHashFieldPrefix != "" { |
|||
field = rule.RedisHashFieldPrefix + field |
|||
} |
|||
|
|||
return field |
|||
} |
|||
|
|||
func (s *RedisEndpoint) encodeSortedSetScoreField(req *model.RowRequest, rule *global.Rule) float64 { |
|||
obj := req.Row[rule.RedisHashFieldColumnIndex] |
|||
if obj == nil { |
|||
return 0 |
|||
} |
|||
|
|||
str := stringutil.ToString(obj) |
|||
return stringutil.ToFloat64Safe(str) |
|||
} |
|||
|
|||
func (s *RedisEndpoint) Close() { |
|||
if s.client != nil { |
|||
s.client.Close() |
|||
} |
|||
} |
|||
@ -0,0 +1,266 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"context" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"log" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"github.com/apache/rocketmq-client-go/v2" |
|||
"github.com/apache/rocketmq-client-go/v2/primitive" |
|||
"github.com/apache/rocketmq-client-go/v2/producer" |
|||
"github.com/apache/rocketmq-client-go/v2/rlog" |
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logagent" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
const _rocketRetry = 2 |
|||
|
|||
type RocketEndpoint struct { |
|||
client rocketmq.Producer |
|||
retryLock sync.Mutex |
|||
} |
|||
|
|||
func newRocketEndpoint() *RocketEndpoint { |
|||
rlog.SetLogger(logagent.NewRocketmqLoggerAgent()) |
|||
cfg := global.Cfg() |
|||
|
|||
options := make([]producer.Option, 0) |
|||
serverList := strings.Split(cfg.RocketmqNameServers, ",") |
|||
options = append(options, producer.WithNameServer(serverList)) |
|||
options = append(options, producer.WithRetry(_rocketRetry)) |
|||
if cfg.RocketmqGroupName != "" { |
|||
options = append(options, producer.WithGroupName(cfg.RocketmqGroupName)) |
|||
} |
|||
if cfg.RocketmqInstanceName != "" { |
|||
options = append(options, producer.WithInstanceName(cfg.RocketmqInstanceName)) |
|||
} |
|||
if cfg.RocketmqAccessKey != "" && cfg.RocketmqSecretKey != "" { |
|||
options = append(options, producer.WithCredentials(primitive.Credentials{ |
|||
AccessKey: cfg.RocketmqAccessKey, |
|||
SecretKey: cfg.RocketmqSecretKey, |
|||
})) |
|||
} |
|||
|
|||
producer, _ := rocketmq.NewProducer(options...) |
|||
r := &RocketEndpoint{} |
|||
r.client = producer |
|||
return r |
|||
} |
|||
|
|||
func (s *RocketEndpoint) Connect() error { |
|||
return s.client.Start() |
|||
} |
|||
|
|||
func (s *RocketEndpoint) Ping() error { |
|||
ping := &primitive.Message{ |
|||
Topic: "BenchmarkTest", |
|||
Body: []byte("ping"), |
|||
} |
|||
_, err := s.client.SendSync(context.Background(), ping) |
|||
return err |
|||
} |
|||
|
|||
func (s *RocketEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
var ms []*primitive.Message |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
|
|||
if rule.LuaEnable() { |
|||
ls, err := s.buildMessages(row, rule) |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
ms = append(ms, ls...) |
|||
} else { |
|||
m, err := s.buildMessage(row, rule) |
|||
if err != nil { |
|||
return errors.New(errors.ErrorStack(err)) |
|||
} |
|||
ms = append(ms, m) |
|||
} |
|||
} |
|||
|
|||
if len(ms) ==0{ |
|||
return nil |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
wg.Add(1) |
|||
var callbackErr error |
|||
err := s.client.SendAsync(context.Background(), |
|||
func(ctx context.Context, result *primitive.SendResult, e error) { |
|||
if e != nil { |
|||
callbackErr = e |
|||
} |
|||
wg.Done() |
|||
}, ms...) |
|||
|
|||
if err != nil { |
|||
return err |
|||
} |
|||
wg.Wait() |
|||
|
|||
if callbackErr != nil { |
|||
return err |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *RocketEndpoint) Stock(rows []*model.RowRequest) int64 { |
|||
expect := true |
|||
var ms []*primitive.Message |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
ls, err := s.buildMessages(row, rule) |
|||
if err != nil { |
|||
logs.Errorf(errors.ErrorStack(err)) |
|||
expect = false |
|||
break |
|||
} |
|||
ms = append(ms, ls...) |
|||
} else { |
|||
m, err := s.buildMessage(row, rule) |
|||
if err != nil { |
|||
logs.Errorf(errors.ErrorStack(err)) |
|||
expect = false |
|||
break |
|||
} |
|||
ms = append(ms, m) |
|||
} |
|||
} |
|||
|
|||
if !expect { |
|||
return 0 |
|||
} |
|||
|
|||
if len(ms) == 0 { |
|||
return 0 |
|||
} |
|||
|
|||
var wg sync.WaitGroup |
|||
wg.Add(1) |
|||
err := s.client.SendAsync(context.Background(), |
|||
func(ctx context.Context, result *primitive.SendResult, e error) { |
|||
if e != nil { |
|||
logs.Error(errors.ErrorStack(e)) |
|||
expect = false |
|||
} |
|||
wg.Done() |
|||
}, ms...) |
|||
|
|||
if err != nil { |
|||
logs.Error(errors.ErrorStack(err)) |
|||
return 0 |
|||
} |
|||
|
|||
wg.Wait() |
|||
|
|||
if expect { |
|||
return int64(len(ms)) |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (s *RocketEndpoint) buildMessages(req *model.RowRequest, rule *global.Rule) ([]*primitive.Message, error) { |
|||
var err error |
|||
var ls []*model.MQRespond |
|||
kvm := rowMap(req, rule, true) |
|||
if req.Action == canal.UpdateAction { |
|||
previous := oldRowMap(req, rule, true) |
|||
ls, err = luaengine.DoMQOps(kvm, previous, req.Action, rule) |
|||
} else { |
|||
ls, err = luaengine.DoMQOps(kvm, nil, req.Action, rule) |
|||
} |
|||
if err != nil { |
|||
return nil, errors.Errorf("lua 脚本执行失败 : %s ", err) |
|||
} |
|||
|
|||
var ms []*primitive.Message |
|||
for _, resp := range ls { |
|||
m := &primitive.Message{ |
|||
Topic: resp.Topic, |
|||
Body: resp.ByteArray, |
|||
} |
|||
logs.Infof("topic: %s, message: %s", m.Topic, string(m.Body)) |
|||
ms = append(ms, m) |
|||
} |
|||
|
|||
return ms, nil |
|||
} |
|||
|
|||
func (s *RocketEndpoint) buildMessage(req *model.RowRequest, rule *global.Rule) (*primitive.Message, error) { |
|||
kvm := rowMap(req, rule, false) |
|||
resp := new(model.MQRespond) |
|||
resp.Action = req.Action |
|||
resp.Timestamp = req.Timestamp |
|||
if rule.ValueEncoder == global.ValEncoderJson { |
|||
resp.Date = kvm |
|||
} else { |
|||
resp.Date = encodeValue(rule, kvm) |
|||
} |
|||
|
|||
if rule.ReserveRawData && canal.UpdateAction == req.Action { |
|||
resp.Raw = oldRowMap(req, rule, false) |
|||
} |
|||
|
|||
body, err := json.Marshal(resp) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
m := &primitive.Message{ |
|||
Topic: rule.RocketmqTopic, |
|||
Body: body, |
|||
} |
|||
|
|||
logs.Infof("topic: %s, message: %s", m.Topic, string(m.Body)) |
|||
|
|||
return m, nil |
|||
} |
|||
|
|||
func (s *RocketEndpoint) Close() { |
|||
if s.client != nil { |
|||
s.client.Shutdown() |
|||
} |
|||
} |
|||
@ -0,0 +1,93 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package endpoint |
|||
|
|||
import ( |
|||
"log" |
|||
|
|||
"github.com/pingcap/errors" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/luaengine" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
type ScriptEndpoint struct { |
|||
} |
|||
|
|||
func newScriptEndpoint() *ScriptEndpoint { |
|||
return &ScriptEndpoint{} |
|||
} |
|||
|
|||
func (s *ScriptEndpoint) Connect() error { |
|||
return nil |
|||
} |
|||
|
|||
func (s *ScriptEndpoint) Ping() error { |
|||
return nil |
|||
} |
|||
|
|||
func (s *ScriptEndpoint) Consume(from mysql.Position, rows []*model.RowRequest) error { |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
metrics.UpdateActionNum(row.Action, row.RuleKey) |
|||
kvm := rowMap(row, rule, true) |
|||
err := luaengine.DoScript(kvm, row.Action, rule) |
|||
if err != nil { |
|||
log.Println("Lua 脚本执行失败!!! ,详情请参见日志") |
|||
return errors.Errorf("Lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
} |
|||
kvm = nil |
|||
} |
|||
|
|||
logs.Infof("处理完成 %d 条数据", len(rows)) |
|||
return nil |
|||
} |
|||
|
|||
func (s *ScriptEndpoint) Stock(rows []*model.RowRequest) int64 { |
|||
var counter int64 |
|||
for _, row := range rows { |
|||
rule, _ := global.RuleIns(row.RuleKey) |
|||
if rule.TableColumnSize != len(row.Row) { |
|||
logs.Warnf("%s schema mismatching", row.RuleKey) |
|||
continue |
|||
} |
|||
|
|||
kvm := rowMap(row, rule, true) |
|||
err := luaengine.DoScript(kvm, row.Action, rule) |
|||
if err != nil { |
|||
logs.Errorf("lua 脚本执行失败 : %s ", errors.ErrorStack(err)) |
|||
break |
|||
} |
|||
counter++ |
|||
} |
|||
|
|||
return counter |
|||
} |
|||
|
|||
func (s *ScriptEndpoint) Close() { |
|||
|
|||
} |
|||
@ -0,0 +1,199 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package service |
|||
|
|||
import ( |
|||
"go-mysql-transfer/metrics" |
|||
"log" |
|||
"time" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
"github.com/siddontang/go-mysql/replication" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
type handler struct { |
|||
queue chan interface{} |
|||
stop chan struct{} |
|||
} |
|||
|
|||
func newHandler() *handler { |
|||
return &handler{ |
|||
queue: make(chan interface{}, 4096), |
|||
stop: make(chan struct{}, 1), |
|||
} |
|||
} |
|||
|
|||
func (s *handler) OnRotate(e *replication.RotateEvent) error { |
|||
s.queue <- model.PosRequest{ |
|||
Name: string(e.NextLogName), |
|||
Pos: uint32(e.Position), |
|||
Force: true, |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) OnTableChanged(schema, table string) error { |
|||
err := _transferService.updateRule(schema, table) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) OnDDL(nextPos mysql.Position, _ *replication.QueryEvent) error { |
|||
s.queue <- model.PosRequest{ |
|||
Name: nextPos.Name, |
|||
Pos: nextPos.Pos, |
|||
Force: true, |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) OnXID(nextPos mysql.Position) error { |
|||
s.queue <- model.PosRequest{ |
|||
Name: nextPos.Name, |
|||
Pos: nextPos.Pos, |
|||
Force: false, |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) OnRow(e *canal.RowsEvent) error { |
|||
ruleKey := global.RuleKey(e.Table.Schema, e.Table.Name) |
|||
if !global.RuleInsExist(ruleKey) { |
|||
return nil |
|||
} |
|||
|
|||
var requests []*model.RowRequest |
|||
if e.Action != canal.UpdateAction { |
|||
// 定长分配
|
|||
requests = make([]*model.RowRequest, 0, len(e.Rows)) |
|||
} |
|||
|
|||
if e.Action == canal.UpdateAction { |
|||
for i := 0; i < len(e.Rows); i++ { |
|||
if (i+1)%2 == 0 { |
|||
v := new(model.RowRequest) |
|||
v.RuleKey = ruleKey |
|||
v.Action = e.Action |
|||
v.Timestamp = e.Header.Timestamp |
|||
if global.Cfg().IsReserveRawData() { |
|||
v.Old = e.Rows[i-1] |
|||
} |
|||
v.Row = e.Rows[i] |
|||
requests = append(requests, v) |
|||
} |
|||
} |
|||
} else { |
|||
for _, row := range e.Rows { |
|||
v := new(model.RowRequest) |
|||
v.RuleKey = ruleKey |
|||
v.Action = e.Action |
|||
v.Timestamp = e.Header.Timestamp |
|||
v.Row = row |
|||
requests = append(requests, v) |
|||
} |
|||
} |
|||
s.queue <- requests |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) OnGTID(gtid mysql.GTIDSet) error { |
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) OnPosSynced(pos mysql.Position, set mysql.GTIDSet, force bool) error { |
|||
return nil |
|||
} |
|||
|
|||
func (s *handler) String() string { |
|||
return "TransferHandler" |
|||
} |
|||
|
|||
func (s *handler) startListener() { |
|||
go func() { |
|||
interval := time.Duration(global.Cfg().FlushBulkInterval) |
|||
bulkSize := global.Cfg().BulkSize |
|||
ticker := time.NewTicker(time.Millisecond * interval) |
|||
defer ticker.Stop() |
|||
|
|||
lastSavedTime := time.Now() |
|||
requests := make([]*model.RowRequest, 0, bulkSize) |
|||
var current mysql.Position |
|||
from, _ := _transferService.positionDao.Get() |
|||
for { |
|||
needFlush := false |
|||
needSavePos := false |
|||
select { |
|||
case v := <-s.queue: |
|||
switch v := v.(type) { |
|||
case model.PosRequest: |
|||
now := time.Now() |
|||
if v.Force || now.Sub(lastSavedTime) > 3*time.Second { |
|||
lastSavedTime = now |
|||
needFlush = true |
|||
needSavePos = true |
|||
current = mysql.Position{ |
|||
Name: v.Name, |
|||
Pos: v.Pos, |
|||
} |
|||
} |
|||
case []*model.RowRequest: |
|||
requests = append(requests, v...) |
|||
needFlush = int64(len(requests)) >= global.Cfg().BulkSize |
|||
} |
|||
case <-ticker.C: |
|||
needFlush = true |
|||
case <-s.stop: |
|||
return |
|||
} |
|||
|
|||
if needFlush && len(requests) > 0 && _transferService.endpointEnable.Load() { |
|||
err := _transferService.endpoint.Consume(from, requests) |
|||
if err != nil { |
|||
_transferService.endpointEnable.Store(false) |
|||
metrics.SetDestState(metrics.DestStateFail) |
|||
logs.Error(err.Error()) |
|||
go _transferService.stopDump() |
|||
} |
|||
requests = requests[0:0] |
|||
} |
|||
if needSavePos && _transferService.endpointEnable.Load() { |
|||
logs.Infof("save position %s %d", current.Name, current.Pos) |
|||
if err := _transferService.positionDao.Save(current); err != nil { |
|||
logs.Errorf("save sync position %s err %v, close sync", current, err) |
|||
_transferService.Close() |
|||
return |
|||
} |
|||
from = current |
|||
} |
|||
} |
|||
}() |
|||
} |
|||
|
|||
func (s *handler) stopListener() { |
|||
log.Println("transfer stop") |
|||
s.stop <- struct{}{} |
|||
} |
|||
@ -0,0 +1,294 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"sync" |
|||
|
|||
luaJson "github.com/layeh/gopher-json" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/util/byteutil" |
|||
"go-mysql-transfer/util/httpclient" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
const ( |
|||
_globalRET = "___RET___" |
|||
_globalROW = "___ROW___" |
|||
_globalACT = "___ACT___" |
|||
) |
|||
|
|||
var ( |
|||
_pool *luaStatePool |
|||
_ds *canal.Canal |
|||
|
|||
_httpClient *httpclient.HttpClient |
|||
) |
|||
|
|||
type luaStatePool struct { |
|||
lock sync.Mutex |
|||
saved []*lua.LState |
|||
} |
|||
|
|||
func InitActuator(ds *canal.Canal) { |
|||
_ds = ds |
|||
_pool = &luaStatePool{ |
|||
saved: make([]*lua.LState, 0, 3), |
|||
} |
|||
} |
|||
|
|||
func (p *luaStatePool) Get() *lua.LState { |
|||
p.lock.Lock() |
|||
defer p.lock.Unlock() |
|||
|
|||
n := len(p.saved) |
|||
if n == 0 { |
|||
return p.New() |
|||
} |
|||
x := p.saved[n-1] |
|||
p.saved = p.saved[0 : n-1] |
|||
return x |
|||
} |
|||
|
|||
func (p *luaStatePool) New() *lua.LState { |
|||
_httpClient = httpclient.NewClient() |
|||
|
|||
L := lua.NewState() |
|||
|
|||
luaJson.Preload(L) |
|||
|
|||
L.PreloadModule("scriptOps", scriptModule) |
|||
L.PreloadModule("dbOps", dbModule) |
|||
L.PreloadModule("httpOps", httpModule) |
|||
|
|||
L.PreloadModule("redisOps", redisModule) |
|||
L.PreloadModule("mqOps", mqModule) |
|||
L.PreloadModule("mongodbOps", mongoModule) |
|||
L.PreloadModule("esOps", esModule) |
|||
|
|||
return L |
|||
} |
|||
|
|||
func (p *luaStatePool) Put(L *lua.LState) { |
|||
p.lock.Lock() |
|||
defer p.lock.Unlock() |
|||
|
|||
p.saved = append(p.saved, L) |
|||
} |
|||
|
|||
func (p *luaStatePool) Shutdown() { |
|||
for _, L := range p.saved { |
|||
L.Close() |
|||
} |
|||
} |
|||
|
|||
func rawRow(L *lua.LState) int { |
|||
row := L.GetGlobal(_globalROW) |
|||
L.Push(row) |
|||
return 1 |
|||
} |
|||
|
|||
func rawAction(L *lua.LState) int { |
|||
act := L.GetGlobal(_globalACT) |
|||
L.Push(act) |
|||
return 1 |
|||
} |
|||
|
|||
func paddingTable(l *lua.LState, table *lua.LTable, kv map[string]interface{}) { |
|||
for k, v := range kv { |
|||
switch v.(type) { |
|||
case float64: |
|||
ft := v.(float64) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case float32: |
|||
ft := v.(float32) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case int: |
|||
ft := v.(int) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case uint: |
|||
ft := v.(uint) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case int8: |
|||
ft := v.(int8) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case uint8: |
|||
ft := v.(uint8) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case int16: |
|||
ft := v.(int16) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case uint16: |
|||
ft := v.(uint16) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case int32: |
|||
ft := v.(int32) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case uint32: |
|||
ft := v.(uint32) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case int64: |
|||
ft := v.(int64) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case uint64: |
|||
ft := v.(uint64) |
|||
l.SetTable(table, lua.LString(k), lua.LNumber(ft)) |
|||
case string: |
|||
ft := v.(string) |
|||
l.SetTable(table, lua.LString(k), lua.LString(ft)) |
|||
case []byte: |
|||
ft := string(v.([]byte)) |
|||
l.SetTable(table, lua.LString(k), lua.LString(ft)) |
|||
case nil: |
|||
l.SetTable(table, lua.LString(k), lua.LNil) |
|||
default: |
|||
jsonValue, _ := json.Marshal(v) |
|||
l.SetTable(table, lua.LString(k), lua.LString(jsonValue)) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func lvToString(lv lua.LValue) string { |
|||
if lua.LVCanConvToString(lv) { |
|||
return lua.LVAsString(lv) |
|||
} |
|||
|
|||
return lv.String() |
|||
} |
|||
|
|||
func lvToByteArray(lv lua.LValue) []byte { |
|||
switch lv.Type() { |
|||
case lua.LTNil: |
|||
return nil |
|||
case lua.LTBool: |
|||
return byteutil.JsonBytes(lua.LVAsBool(lv)) |
|||
case lua.LTNumber: |
|||
return []byte(lv.String()) |
|||
case lua.LTString: |
|||
return []byte(lua.LVAsString(lv)) |
|||
case lua.LTTable: |
|||
ret := lvToInterface(lv, false) |
|||
return byteutil.JsonBytes(ret) |
|||
default: |
|||
return byteutil.JsonBytes(lv) |
|||
} |
|||
} |
|||
|
|||
func lvToInterface(lv lua.LValue, tableToJson bool) interface{} { |
|||
switch lv.Type() { |
|||
case lua.LTNil: |
|||
return nil |
|||
case lua.LTBool: |
|||
return lua.LVAsBool(lv) |
|||
case lua.LTNumber: |
|||
return float64(lua.LVAsNumber(lv)) |
|||
case lua.LTString: |
|||
return lua.LVAsString(lv) |
|||
case lua.LTTable: |
|||
t, _ := lv.(*lua.LTable) |
|||
len := t.MaxN() |
|||
if len == 0 { // table
|
|||
ret := make(map[string]interface{}) |
|||
t.ForEach(func(key, value lua.LValue) { |
|||
ret[lvToString(key)] = lvToInterface(value, false) |
|||
}) |
|||
if tableToJson { |
|||
return stringutil.ToJsonString(ret) |
|||
} |
|||
return ret |
|||
} else { // array
|
|||
ret := make([]interface{}, 0, len) |
|||
for i := 1; i <= len; i++ { |
|||
ret = append(ret, lvToInterface(t.RawGetInt(i), false)) |
|||
} |
|||
if tableToJson { |
|||
return stringutil.ToJsonString(ret) |
|||
} |
|||
return ret |
|||
} |
|||
default: |
|||
return lv |
|||
} |
|||
} |
|||
|
|||
func lvToMap(lv lua.LValue) (map[string]interface{}, bool) { |
|||
switch lv.Type() { |
|||
case lua.LTTable: |
|||
t := lvToInterface(lv, false) |
|||
ret := t.(map[string]interface{}) |
|||
return ret, true |
|||
default: |
|||
return nil, false |
|||
} |
|||
} |
|||
|
|||
func interfaceToLv(v interface{}) lua.LValue { |
|||
switch v.(type) { |
|||
case float64: |
|||
ft := v.(float64) |
|||
return lua.LNumber(ft) |
|||
case float32: |
|||
ft := v.(float32) |
|||
return lua.LNumber(ft) |
|||
case int: |
|||
ft := v.(int) |
|||
return lua.LNumber(ft) |
|||
case uint: |
|||
ft := v.(uint) |
|||
return lua.LNumber(ft) |
|||
case int8: |
|||
ft := v.(int8) |
|||
return lua.LNumber(ft) |
|||
case uint8: |
|||
ft := v.(uint8) |
|||
return lua.LNumber(ft) |
|||
case int16: |
|||
ft := v.(int16) |
|||
return lua.LNumber(ft) |
|||
case uint16: |
|||
ft := v.(uint16) |
|||
return lua.LNumber(ft) |
|||
case int32: |
|||
ft := v.(int32) |
|||
return lua.LNumber(ft) |
|||
case uint32: |
|||
ft := v.(uint32) |
|||
return lua.LNumber(ft) |
|||
case int64: |
|||
ft := v.(int64) |
|||
return lua.LNumber(ft) |
|||
case uint64: |
|||
ft := v.(uint64) |
|||
return lua.LNumber(ft) |
|||
case string: |
|||
ft := v.(string) |
|||
return lua.LString(ft) |
|||
case []byte: |
|||
ft := string(v.([]byte)) |
|||
return lua.LString(ft) |
|||
case nil: |
|||
return lua.LNil |
|||
default: |
|||
jsonValue, _ := json.Marshal(v) |
|||
return lua.LString(jsonValue) |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,124 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"github.com/layeh/gopher-json" |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
func dbModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _dbModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _dbModuleApi = map[string]lua.LGFunction{ |
|||
"selectOne": selectOne, |
|||
"select": selectList, |
|||
} |
|||
|
|||
func selectOne(L *lua.LState) int { |
|||
sql := L.CheckString(1) |
|||
|
|||
logs.Infof("lua db module execute sql: %s", sql) |
|||
|
|||
rs, err := _ds.Execute(sql) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
rowNumber := rs.RowNumber() |
|||
|
|||
table := L.NewTable() |
|||
if rowNumber > 1 { |
|||
logs.Error("return more than 1 row") |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString("return more than 1 row")) |
|||
return 2 |
|||
} |
|||
|
|||
if rowNumber == 1 { |
|||
for field, index := range rs.FieldNames { |
|||
v, err := rs.GetValue(0, index) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
val := interfaceToLv(v) |
|||
L.SetTable(table, lua.LString(field), val) |
|||
} |
|||
} |
|||
|
|||
if data, err := json.Encode(table); err == nil { |
|||
logs.Infof("lua db module result: %s", string(data)) |
|||
} |
|||
|
|||
L.Push(table) |
|||
return 1 |
|||
} |
|||
|
|||
func selectList(L *lua.LState) int { |
|||
sql := L.CheckString(1) |
|||
|
|||
logs.Infof("lua db module execute sql: %s", sql) |
|||
|
|||
rs, err := _ds.Execute(sql) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
rowNumber := rs.RowNumber() |
|||
|
|||
ret := L.NewTable() |
|||
if rowNumber > 0 { |
|||
for i := 0; i < rowNumber; i++ { |
|||
table := L.NewTable() |
|||
for field, index := range rs.FieldNames { |
|||
v, err := rs.GetValue(i, index) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
val := interfaceToLv(v) |
|||
L.SetTable(table, lua.LString(field), val) |
|||
} |
|||
L.SetTable(ret, lua.LNumber(i+1), table) |
|||
} |
|||
} |
|||
|
|||
if data, err := json.Encode(ret); err == nil { |
|||
logs.Infof("lua db module result: %s", string(data)) |
|||
} else { |
|||
logs.Error(err.Error()) |
|||
} |
|||
|
|||
L.Push(ret) |
|||
return 1 |
|||
} |
|||
@ -0,0 +1,134 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
func esModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _esModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _esModuleApi = map[string]lua.LGFunction{ |
|||
"rawRow": rawRow, |
|||
"rawAction": rawAction, |
|||
|
|||
"INSERT": esInsert, |
|||
"UPDATE": esUpdate, |
|||
"DELETE": esDelete, |
|||
} |
|||
|
|||
func esInsert(L *lua.LState) int { |
|||
index := L.CheckAny(1) |
|||
id := L.CheckAny(2) |
|||
body := L.CheckAny(3) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("index"), index) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(canal.InsertAction)) |
|||
L.SetTable(data, lua.LString("id"), id) |
|||
L.SetTable(data, lua.LString("body"), body) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func esUpdate(L *lua.LState) int { |
|||
index := L.CheckAny(1) |
|||
id := L.CheckAny(2) |
|||
body := L.CheckAny(3) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("index"), index) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(canal.UpdateAction)) |
|||
L.SetTable(data, lua.LString("id"), id) |
|||
L.SetTable(data, lua.LString("body"), body) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func esDelete(L *lua.LState) int { |
|||
index := L.CheckAny(1) |
|||
id := L.CheckAny(2) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("index"), index) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(canal.DeleteAction)) |
|||
L.SetTable(data, lua.LString("id"), id) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func DoESOps(input map[string]interface{}, action string, rule *global.Rule) ([]*model.ESRespond, error) { |
|||
L := _pool.Get() |
|||
defer _pool.Put(L) |
|||
|
|||
row := L.NewTable() |
|||
paddingTable(L, row, input) |
|||
ret := L.NewTable() |
|||
L.SetGlobal(_globalRET, ret) |
|||
L.SetGlobal(_globalROW, row) |
|||
L.SetGlobal(_globalACT, lua.LString(action)) |
|||
|
|||
funcFromProto := L.NewFunctionFromProto(rule.LuaProto) |
|||
L.Push(funcFromProto) |
|||
err := L.PCall(0, lua.MultRet, nil) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
responds := make([]*model.ESRespond, 0, ret.Len()) |
|||
ret.ForEach(func(k lua.LValue, v lua.LValue) { |
|||
resp := new(model.ESRespond) |
|||
resp.Index = lvToString(L.GetTable(v, lua.LString("index"))) |
|||
resp.Id = lvToString(L.GetTable(v, lua.LString("id"))) |
|||
resp.Action = lvToString(L.GetTable(v, lua.LString("action"))) |
|||
|
|||
var data string |
|||
body := L.GetTable(v, lua.LString("body")) |
|||
switch body.Type() { |
|||
case lua.LTNumber: |
|||
data = lua.LVAsString(body) |
|||
case lua.LTString: |
|||
data = lua.LVAsString(body) |
|||
case lua.LTTable: |
|||
mm, _ := lvToMap(body) |
|||
data = stringutil.ToJsonString(mm) |
|||
default: |
|||
data = stringutil.ToJsonString(body) |
|||
} |
|||
resp.Date = data |
|||
responds = append(responds, resp) |
|||
}) |
|||
|
|||
return responds, nil |
|||
} |
|||
@ -0,0 +1,156 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
func httpModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _httpModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _httpModuleApi = map[string]lua.LGFunction{ |
|||
"get": doGet, |
|||
"delete": doDelete, |
|||
"post": doPost, |
|||
"put": doPut, |
|||
} |
|||
|
|||
func doGet(L *lua.LState) int { |
|||
ret := L.NewTable() |
|||
paramUrl := L.CheckString(1) |
|||
paramOps := L.CheckTable(2) |
|||
|
|||
cli := _httpClient.GET(paramUrl) |
|||
if headers, ok := lvToMap(paramOps); ok { |
|||
cli.SetHeaders(headers) |
|||
} |
|||
|
|||
entity, err := cli.DoForEntity() |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
|
|||
ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) |
|||
ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) |
|||
|
|||
L.Push(ret) |
|||
return 1 |
|||
} |
|||
|
|||
func doDelete(L *lua.LState) int { |
|||
ret := L.NewTable() |
|||
paramUrl := L.CheckString(1) |
|||
paramOps := L.CheckTable(2) |
|||
|
|||
cli := _httpClient.DELETE(paramUrl) |
|||
if headers, ok := lvToMap(paramOps); ok { |
|||
cli.SetHeaders(headers) |
|||
} |
|||
|
|||
entity, err := cli.DoForEntity() |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
|
|||
ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) |
|||
ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) |
|||
|
|||
L.Push(ret) |
|||
return 1 |
|||
} |
|||
|
|||
func doPost(L *lua.LState) int { |
|||
ret := L.NewTable() |
|||
paramUrl := L.CheckString(1) |
|||
paramHeaders := L.CheckTable(2) |
|||
paramContents := L.CheckTable(3) |
|||
|
|||
cli := _httpClient.POST(paramUrl) |
|||
if headers, ok := lvToMap(paramHeaders); ok { |
|||
cli.SetHeaders(headers) |
|||
} |
|||
|
|||
contents, ok := lvToMap(paramContents) |
|||
if !ok { |
|||
logs.Error("The argument must Table") |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString("The argument must Table")) |
|||
return 2 |
|||
} |
|||
|
|||
entity, err := cli.SetBodyAsForm(contents).DoForEntity() |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
|
|||
ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) |
|||
ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) |
|||
|
|||
L.Push(ret) |
|||
return 1 |
|||
} |
|||
|
|||
func doPut(L *lua.LState) int { |
|||
ret := L.NewTable() |
|||
paramUrl := L.CheckString(1) |
|||
paramHeaders := L.CheckTable(2) |
|||
paramContents := L.CheckTable(3) |
|||
|
|||
cli := _httpClient.PUT(paramUrl) |
|||
if headers, ok := lvToMap(paramHeaders); ok { |
|||
cli.SetHeaders(headers) |
|||
} |
|||
|
|||
contents, ok := lvToMap(paramContents) |
|||
if !ok { |
|||
logs.Error("The argument must Table") |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString("The argument must Table")) |
|||
return 2 |
|||
} |
|||
|
|||
entity, err := cli.SetBodyAsForm(contents).DoForEntity() |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
L.Push(lua.LNil) |
|||
L.Push(lua.LString(err.Error())) |
|||
return 2 |
|||
} |
|||
|
|||
ret.RawSet(lua.LString("status_code"), lua.LNumber(entity.StatusCode())) |
|||
ret.RawSet(lua.LString("body"), lua.LString(string(entity.Data()))) |
|||
|
|||
L.Push(ret) |
|||
return 1 |
|||
} |
|||
@ -0,0 +1,161 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
func mongoModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _mongoModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _mongoModuleApi = map[string]lua.LGFunction{ |
|||
"rawRow": rawRow, |
|||
"rawAction": rawAction, |
|||
|
|||
"INSERT": mongoInsert, |
|||
"UPDATE": mongoUpdate, |
|||
"DELETE": mongoDelete, |
|||
"UPSERT": mongoUpsert, |
|||
} |
|||
|
|||
func mongoInsert(L *lua.LState) int { |
|||
collection := L.CheckAny(1) |
|||
table := L.CheckAny(2) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("collection"), collection) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(canal.InsertAction)) |
|||
L.SetTable(data, lua.LString("table"), table) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func mongoUpdate(L *lua.LState) int { |
|||
collection := L.CheckAny(1) |
|||
id := L.CheckAny(2) |
|||
table := L.CheckAny(3) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("collection"), collection) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(canal.UpdateAction)) |
|||
L.SetTable(data, lua.LString("id"), id) |
|||
L.SetTable(data, lua.LString("table"), table) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func mongoUpsert(L *lua.LState) int { |
|||
collection := L.CheckAny(1) |
|||
id := L.CheckAny(2) |
|||
table := L.CheckAny(3) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("collection"), collection) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(global.UpsertAction)) |
|||
L.SetTable(data, lua.LString("id"), id) |
|||
L.SetTable(data, lua.LString("table"), table) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func mongoDelete(L *lua.LState) int { |
|||
collection := L.CheckAny(1) |
|||
id := L.CheckAny(2) |
|||
|
|||
data := L.NewTable() |
|||
L.SetTable(data, lua.LString("collection"), collection) |
|||
L.SetTable(data, lua.LString("action"), lua.LString(canal.DeleteAction)) |
|||
L.SetTable(data, lua.LString("id"), id) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString(stringutil.UUID()), data) |
|||
return 0 |
|||
} |
|||
|
|||
func DoMongoOps(input map[string]interface{}, action string, rule *global.Rule) ([]*model.MongoRespond, error) { |
|||
L := _pool.Get() |
|||
defer _pool.Put(L) |
|||
|
|||
row := L.NewTable() |
|||
paddingTable(L, row, input) |
|||
ret := L.NewTable() |
|||
L.SetGlobal(_globalRET, ret) |
|||
L.SetGlobal(_globalROW, row) |
|||
L.SetGlobal(_globalACT, lua.LString(action)) |
|||
|
|||
funcFromProto := L.NewFunctionFromProto(rule.LuaProto) |
|||
L.Push(funcFromProto) |
|||
err := L.PCall(0, lua.MultRet, nil) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
asserted := true |
|||
responds := make([]*model.MongoRespond, 0, ret.Len()) |
|||
ret.ForEach(func(k lua.LValue, v lua.LValue) { |
|||
resp := new(model.MongoRespond) |
|||
resp.Collection = lvToString(L.GetTable(v, lua.LString("collection"))) |
|||
resp.Action = lvToString(L.GetTable(v, lua.LString("action"))) |
|||
resp.Id = lvToInterface(L.GetTable(v, lua.LString("id")), true) |
|||
lvTable := L.GetTable(v, lua.LString("table")) |
|||
|
|||
var table map[string]interface{} |
|||
if action != canal.DeleteAction { |
|||
table, asserted = lvToMap(lvTable) |
|||
if !asserted { |
|||
return |
|||
} |
|||
resp.Table = table |
|||
} |
|||
|
|||
if action == canal.InsertAction { |
|||
_id, ok := table["_id"] |
|||
if !ok { |
|||
resp.Id = stringutil.UUID() |
|||
table["_id"] = resp.Id |
|||
} else { |
|||
resp.Id = _id |
|||
} |
|||
} |
|||
|
|||
responds = append(responds, resp) |
|||
}) |
|||
|
|||
if !asserted { |
|||
return nil, errors.New("The parameter must be of table type") |
|||
} |
|||
|
|||
return responds, nil |
|||
} |
|||
@ -0,0 +1,85 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
) |
|||
|
|||
func mqModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _mqModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _mqModuleApi = map[string]lua.LGFunction{ |
|||
"rawRow": rawRow, |
|||
"rawOldRow": rawOldRow, |
|||
"rawAction": rawAction, |
|||
|
|||
"SEND": msgSend, |
|||
} |
|||
|
|||
func msgSend(L *lua.LState) int { |
|||
topic := L.CheckAny(1) |
|||
msg := L.CheckAny(2) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, msg, topic) |
|||
return 0 |
|||
} |
|||
|
|||
func DoMQOps(input map[string]interface{}, previous map[string]interface{}, action string, rule *global.Rule) ([]*model.MQRespond, error) { |
|||
L := _pool.Get() |
|||
defer _pool.Put(L) |
|||
|
|||
row := L.NewTable() |
|||
paddingTable(L, row, input) |
|||
ret := L.NewTable() |
|||
L.SetGlobal(_globalRET, ret) |
|||
L.SetGlobal(_globalROW, row) |
|||
L.SetGlobal(_globalACT, lua.LString(action)) |
|||
|
|||
if action == canal.UpdateAction { |
|||
oldRow := L.NewTable() |
|||
paddingTable(L, oldRow, previous) |
|||
L.SetGlobal(_globalOLDROW, oldRow) |
|||
} |
|||
|
|||
funcFromProto := L.NewFunctionFromProto(rule.LuaProto) |
|||
L.Push(funcFromProto) |
|||
err := L.PCall(0, lua.MultRet, nil) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
list := make([]*model.MQRespond, 0, ret.Len()) |
|||
ret.ForEach(func(k lua.LValue, v lua.LValue) { |
|||
resp := new(model.MQRespond) |
|||
resp.ByteArray = lvToByteArray(k) |
|||
resp.Topic = lvToString(v) |
|||
list = append(list, resp) |
|||
}) |
|||
|
|||
return list, nil |
|||
} |
|||
@ -0,0 +1,245 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package luaengine |
|||
|
|||
import ( |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
const _globalOLDROW = "___OLDROW___" |
|||
|
|||
func redisModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _redisModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _redisModuleApi = map[string]lua.LGFunction{ |
|||
"rawRow": rawRow, |
|||
"rawOldRow": rawOldRow, |
|||
"rawAction": rawAction, |
|||
|
|||
"SET": redisSet, |
|||
"DEL": redisDel, |
|||
|
|||
"HSET": redisHSet, |
|||
"HDEL": redisHDel, |
|||
|
|||
"RPUSH": redisRPush, |
|||
"LREM": redisLRem, |
|||
|
|||
"SADD": redisSAdd, |
|||
"SREM": redisSRem, |
|||
|
|||
"ZADD": redisZAdd, |
|||
"ZREM": redisZRem, |
|||
} |
|||
|
|||
func rawOldRow(L *lua.LState) int { |
|||
row := L.GetGlobal(_globalOLDROW) |
|||
L.Push(row) |
|||
return 1 |
|||
} |
|||
|
|||
func redisSet(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
val := L.CheckAny(2) |
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("insert_1_"+key), val) |
|||
return 0 |
|||
} |
|||
|
|||
func redisDel(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("delete_1_"+key), lua.LBool(true)) |
|||
return 0 |
|||
} |
|||
|
|||
func redisHSet(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
field := L.CheckAny(2) |
|||
val := L.CheckAny(3) |
|||
|
|||
hash := L.NewTable() |
|||
L.SetTable(hash, lua.LString("key"), lua.LString(key)) |
|||
L.SetTable(hash, lua.LString("field"), field) |
|||
L.SetTable(hash, lua.LString("val"), val) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("insert_2_"+stringutil.UUID()), hash) |
|||
return 0 |
|||
} |
|||
|
|||
func redisHDel(L *lua.LState) int { |
|||
key := L.CheckAny(1) |
|||
field := L.CheckAny(2) |
|||
|
|||
hash := L.NewTable() |
|||
L.SetTable(hash, lua.LString("key"), key) |
|||
L.SetTable(hash, lua.LString("field"), field) |
|||
L.SetTable(hash, lua.LString("val"), lua.LNumber(1)) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("delete_2_"+stringutil.UUID()), hash) |
|||
return 0 |
|||
} |
|||
|
|||
func redisRPush(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
val := L.CheckAny(2) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("insert_3_"+key), val) |
|||
return 0 |
|||
} |
|||
|
|||
func redisLRem(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
val := L.CheckAny(2) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("delete_3_"+key), val) |
|||
return 0 |
|||
} |
|||
|
|||
func redisSAdd(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
val := L.CheckAny(2) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("insert_4_"+key), val) |
|||
return 0 |
|||
} |
|||
|
|||
func redisSRem(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
val := L.CheckAny(2) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("delete_4_"+key), val) |
|||
return 0 |
|||
} |
|||
|
|||
func redisZAdd(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
score := L.CheckAny(2) |
|||
val := L.CheckAny(3) |
|||
|
|||
hash := L.NewTable() |
|||
L.SetTable(hash, lua.LString("key"), lua.LString(key)) |
|||
L.SetTable(hash, lua.LString("score"), score) |
|||
L.SetTable(hash, lua.LString("val"), val) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("insert_5_"+stringutil.UUID()), hash) |
|||
return 0 |
|||
} |
|||
|
|||
func redisZRem(L *lua.LState) int { |
|||
key := L.CheckString(1) |
|||
val := L.CheckAny(2) |
|||
|
|||
ret := L.GetGlobal(_globalRET) |
|||
L.SetTable(ret, lua.LString("delete_5_"+key), val) |
|||
return 0 |
|||
} |
|||
|
|||
func DoRedisOps(input map[string]interface{}, previous map[string]interface{}, action string, rule *global.Rule) ([]*model.RedisRespond, error) { |
|||
L := _pool.Get() |
|||
defer _pool.Put(L) |
|||
|
|||
row := L.NewTable() |
|||
paddingTable(L, row, input) |
|||
ret := L.NewTable() |
|||
L.SetGlobal(_globalRET, ret) |
|||
L.SetGlobal(_globalROW, row) |
|||
L.SetGlobal(_globalACT, lua.LString(action)) |
|||
|
|||
if action == canal.UpdateAction { |
|||
oldRow := L.NewTable() |
|||
paddingTable(L, oldRow, previous) |
|||
L.SetGlobal(_globalOLDROW, oldRow) |
|||
} |
|||
|
|||
funcFromProto := L.NewFunctionFromProto(rule.LuaProto) |
|||
L.Push(funcFromProto) |
|||
err := L.PCall(0, lua.MultRet, nil) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
ls := make([]*model.RedisRespond, 0, ret.Len()) |
|||
ret.ForEach(func(k lua.LValue, v lua.LValue) { |
|||
resp := new(model.RedisRespond) |
|||
kk := lvToString(k) |
|||
resp.Action = kk[0:6] |
|||
resp.Structure = structureName(kk[7:8]) |
|||
if resp.Action == canal.DeleteAction { |
|||
resp.Key = kk[9:len(kk)] |
|||
resp.Val = lvToInterface(v, true) |
|||
} else { |
|||
if resp.Structure == global.RedisStructureHash { |
|||
key := L.GetTable(v, lua.LString("key")) |
|||
field := L.GetTable(v, lua.LString("field")) |
|||
val := L.GetTable(v, lua.LString("val")) |
|||
resp.Key = key.String() |
|||
resp.Field = lvToString(field) |
|||
resp.Val = lvToInterface(val, true) |
|||
} else if resp.Structure == global.RedisStructureSortedSet { |
|||
key := L.GetTable(v, lua.LString("key")) |
|||
score := L.GetTable(v, lua.LString("score")) |
|||
val := L.GetTable(v, lua.LString("val")) |
|||
resp.Key = key.String() |
|||
scoreTemp := lvToString(score) |
|||
resp.Score = stringutil.ToFloat64Safe(scoreTemp) |
|||
resp.Val = lvToInterface(val, true) |
|||
} else { |
|||
resp.Key = kk[9:len(kk)] |
|||
resp.Val = lvToInterface(v, true) |
|||
} |
|||
} |
|||
|
|||
ls = append(ls, resp) |
|||
}) |
|||
|
|||
return ls, nil |
|||
} |
|||
|
|||
func structureName(code string) string { |
|||
switch code { |
|||
case "1": |
|||
return global.RedisStructureString |
|||
case "2": |
|||
return global.RedisStructureHash |
|||
case "3": |
|||
return global.RedisStructureList |
|||
case "4": |
|||
return global.RedisStructureSet |
|||
case "5": |
|||
return global.RedisStructureSortedSet |
|||
} |
|||
|
|||
return "" |
|||
} |
|||
@ -0,0 +1,39 @@ |
|||
package luaengine |
|||
|
|||
import ( |
|||
lua "github.com/yuin/gopher-lua" |
|||
|
|||
"go-mysql-transfer/global" |
|||
) |
|||
|
|||
func scriptModule(L *lua.LState) int { |
|||
t := L.NewTable() |
|||
L.SetFuncs(t, _scriptModuleApi) |
|||
L.Push(t) |
|||
return 1 |
|||
} |
|||
|
|||
var _scriptModuleApi = map[string]lua.LGFunction{ |
|||
"rawRow": rawRow, |
|||
"rawAction": rawAction, |
|||
} |
|||
|
|||
func DoScript(input map[string]interface{}, action string, rule *global.Rule) error { |
|||
L := _pool.Get() |
|||
defer _pool.Put(L) |
|||
|
|||
row := L.NewTable() |
|||
paddingTable(L, row, input) |
|||
|
|||
L.SetGlobal(_globalROW, row) |
|||
L.SetGlobal(_globalACT, lua.LString(action)) |
|||
|
|||
funcFromProto := L.NewFunctionFromProto(rule.LuaProto) |
|||
L.Push(funcFromProto) |
|||
err := L.PCall(0, lua.MultRet, nil) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,69 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package service |
|||
|
|||
import ( |
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/service/election" |
|||
) |
|||
|
|||
var ( |
|||
_transferService *TransferService |
|||
_electionService election.Service |
|||
_clusterService *ClusterService |
|||
) |
|||
|
|||
func Initialize() error { |
|||
transferService := &TransferService{ |
|||
loopStopSignal: make(chan struct{}, 1), |
|||
} |
|||
err := transferService.initialize() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
_transferService = transferService |
|||
|
|||
if global.Cfg().IsCluster() { |
|||
_clusterService = &ClusterService{ |
|||
electionSignal: make(chan bool, 1), |
|||
} |
|||
_electionService = election.NewElection(_clusterService.electionSignal) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func StartUp() { |
|||
if global.Cfg().IsCluster() { |
|||
_clusterService.boot() |
|||
} else { |
|||
_transferService.StartUp() |
|||
} |
|||
} |
|||
|
|||
func Close() { |
|||
_transferService.Close() |
|||
} |
|||
|
|||
func TransferServiceIns() *TransferService { |
|||
return _transferService |
|||
} |
|||
|
|||
func ClusterServiceIns() *ClusterService { |
|||
return _clusterService |
|||
} |
|||
@ -0,0 +1,382 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package service |
|||
|
|||
import ( |
|||
"fmt" |
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"go.uber.org/atomic" |
|||
"log" |
|||
"regexp" |
|||
"strings" |
|||
"sync" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/model" |
|||
"go-mysql-transfer/service/endpoint" |
|||
"go-mysql-transfer/util/dates" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
// 存量数据
|
|||
type StockService struct { |
|||
canal *canal.Canal |
|||
endpoint endpoint.Endpoint |
|||
|
|||
queueCh chan []*model.RowRequest |
|||
counter map[string]int64 |
|||
lockOfCounter sync.Mutex |
|||
totalRows map[string]int64 |
|||
wg sync.WaitGroup |
|||
shutoff *atomic.Bool |
|||
} |
|||
|
|||
func NewStockService() *StockService { |
|||
return &StockService{ |
|||
queueCh: make(chan []*model.RowRequest, global.Cfg().Maxprocs), |
|||
counter: make(map[string]int64), |
|||
totalRows: make(map[string]int64), |
|||
shutoff: atomic.NewBool(false), |
|||
} |
|||
} |
|||
|
|||
func (s *StockService) Run() error { |
|||
canalCfg := canal.NewDefaultConfig() |
|||
canalCfg.Addr = global.Cfg().Addr |
|||
canalCfg.User = global.Cfg().User |
|||
canalCfg.Password = global.Cfg().Password |
|||
canalCfg.Charset = global.Cfg().Charset |
|||
canalCfg.Flavor = global.Cfg().Flavor |
|||
canalCfg.ServerID = global.Cfg().SlaveID |
|||
canalCfg.Dump.ExecutionPath = global.Cfg().DumpExec |
|||
canalCfg.Dump.DiscardErr = false |
|||
canalCfg.Dump.SkipMasterData = global.Cfg().SkipMasterData |
|||
|
|||
if c, err := canal.NewCanal(canalCfg); err != nil { |
|||
errors.Trace(err) |
|||
} else { |
|||
s.canal = c |
|||
} |
|||
|
|||
if err := s.completeRules(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
s.addDumpDatabaseOrTable() |
|||
|
|||
endpoint := endpoint.NewEndpoint(s.canal) |
|||
if err := endpoint.Connect(); err != nil { |
|||
log.Println(err.Error()) |
|||
return errors.Trace(err) |
|||
} |
|||
s.endpoint = endpoint |
|||
|
|||
startTime := dates.NowMillisecond() |
|||
log.Println(fmt.Sprintf("bulk size: %d", global.Cfg().BulkSize)) |
|||
for _, rule := range global.RuleInsList() { |
|||
if rule.OrderByColumn == "" { |
|||
return errors.New("empty order_by_column not allowed") |
|||
} |
|||
|
|||
exportColumns := s.exportColumns(rule) |
|||
fullName := fmt.Sprintf("%s.%s", rule.Schema, rule.Table) |
|||
log.Println(fmt.Sprintf("开始导出 %s", fullName)) |
|||
|
|||
res, err := s.canal.Execute(fmt.Sprintf("select count(1) from %s", fullName)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
totalRow, err := res.GetInt(0, 0) |
|||
s.totalRows[fullName] = totalRow |
|||
log.Println(fmt.Sprintf("%s 共 %d 条数据", fullName, totalRow)) |
|||
|
|||
s.counter[fullName] = 0 |
|||
|
|||
var batch int64 |
|||
size := global.Cfg().BulkSize |
|||
if batch%size == 0 { |
|||
batch = totalRow / size |
|||
} else { |
|||
batch = (totalRow / size) + 1 |
|||
} |
|||
|
|||
var processed atomic.Int64 |
|||
for i := 0; i < global.Cfg().Maxprocs; i++ { |
|||
s.wg.Add(1) |
|||
go func(_fullName, _columns string, _rule *global.Rule) { |
|||
for { |
|||
processed.Inc() |
|||
requests, err := s.export(_fullName, _columns, processed.Load(), _rule) |
|||
if err != nil { |
|||
logs.Error(err.Error()) |
|||
s.shutoff.Store(true) |
|||
break |
|||
} |
|||
|
|||
s.imports(_fullName, requests) |
|||
if processed.Load() > batch { |
|||
break |
|||
} |
|||
} |
|||
s.wg.Done() |
|||
}(fullName, exportColumns, rule) |
|||
} |
|||
} |
|||
|
|||
s.wg.Wait() |
|||
|
|||
fmt.Println(fmt.Sprintf("共耗时 ：%d（毫秒）", dates.NowMillisecond()-startTime)) |
|||
|
|||
for k, v := range s.totalRows { |
|||
vv, ok := s.counter[k] |
|||
if ok { |
|||
fmt.Println(fmt.Sprintf("表： %s，共：%d 条数据，成功导入：%d 条", k, v, vv)) |
|||
if v > vv { |
|||
fmt.Println("存在导入错误的数据，具体请至日志查看") |
|||
} |
|||
} |
|||
} |
|||
|
|||
s.endpoint.Close() // 关闭客户端
|
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *StockService) export(fullName, columns string, batch int64, rule *global.Rule) ([]*model.RowRequest, error) { |
|||
if s.shutoff.Load() { |
|||
return nil, errors.New("shutoff") |
|||
} |
|||
|
|||
offset := s.offset(batch) |
|||
sql := s.buildSql(fullName, columns, offset, rule) |
|||
logs.Infof("export sql : %s", sql) |
|||
resultSet, err := s.canal.Execute(sql) |
|||
if err != nil { |
|||
logs.Errorf("数据导出错误: %s - %s", sql, err.Error()) |
|||
return nil, err |
|||
} |
|||
rowNumber := resultSet.RowNumber() |
|||
requests := make([]*model.RowRequest, 0, rowNumber) |
|||
for i := 0; i < rowNumber; i++ { |
|||
rowValues := make([]interface{}, 0, len(rule.TableInfo.Columns)) |
|||
request := new(model.RowRequest) |
|||
for j := 0; j < len(rule.TableInfo.Columns); j++ { |
|||
val, err := resultSet.GetValue(i, j) |
|||
if err != nil { |
|||
logs.Errorf("数据导出错误: %s - %s", sql, err.Error()) |
|||
break |
|||
} |
|||
rowValues = append(rowValues, val) |
|||
request.Action = canal.InsertAction |
|||
request.RuleKey = global.RuleKey(rule.Schema, rule.Table) |
|||
request.Row = rowValues |
|||
} |
|||
requests = append(requests, request) |
|||
} |
|||
|
|||
return requests, nil |
|||
} |
|||
|
|||
// 构造SQL
|
|||
func (s *StockService) buildSql(fullName, columns string, offset int64, rule *global.Rule) string { |
|||
size := global.Cfg().BulkSize |
|||
if len(rule.TableInfo.PKColumns) == 0 { |
|||
return fmt.Sprintf("select %s from %s order by %s limit %d,%d", columns, fullName, rule.OrderByColumn, offset, size) |
|||
} |
|||
|
|||
i := rule.TableInfo.PKColumns[0] |
|||
n := rule.TableInfo.GetPKColumn(i).Name |
|||
t := "select b.* from (select %s from %s order by %s limit %d,%d) a left join %s b on a.%s=b.%s" |
|||
sql := fmt.Sprintf(t, n, fullName, rule.OrderByColumn, offset, size, fullName, n, n) |
|||
return sql |
|||
} |
|||
|
|||
func (s *StockService) imports(fullName string, requests []*model.RowRequest) { |
|||
if s.shutoff.Load() { |
|||
return |
|||
} |
|||
|
|||
succeeds := s.endpoint.Stock(requests) |
|||
count := s.incCounter(fullName, succeeds) |
|||
log.Println(fmt.Sprintf("%s 导入数据 %d 条", fullName, count)) |
|||
} |
|||
|
|||
func (s *StockService) exportColumns(rule *global.Rule) string { |
|||
if rule.IncludeColumnConfig != "" { |
|||
var columns string |
|||
includes := strings.Split(rule.IncludeColumnConfig, ",") |
|||
for _, c := range rule.TableInfo.Columns { |
|||
for _, e := range includes { |
|||
var column string |
|||
if strings.ToUpper(e) == strings.ToUpper(c.Name) { |
|||
column = c.Name |
|||
} else { |
|||
column = "null as " + c.Name |
|||
} |
|||
|
|||
if columns != "" { |
|||
columns = columns + "," |
|||
} |
|||
columns = columns + column |
|||
} |
|||
} |
|||
return columns |
|||
} |
|||
|
|||
if rule.ExcludeColumnConfig != "" { |
|||
var columns string |
|||
excludes := strings.Split(rule.ExcludeColumnConfig, ",") |
|||
for _, c := range rule.TableInfo.Columns { |
|||
for _, e := range excludes { |
|||
var column string |
|||
if strings.ToUpper(e) == strings.ToUpper(c.Name) { |
|||
column = "null as " + c.Name |
|||
} else { |
|||
column = c.Name |
|||
} |
|||
|
|||
if columns != "" { |
|||
columns = columns + "," |
|||
} |
|||
columns = columns + column |
|||
} |
|||
} |
|||
return columns |
|||
} |
|||
|
|||
return "*" |
|||
} |
|||
|
|||
func (s *StockService) offset(currentPage int64) int64 { |
|||
var offset int64 |
|||
|
|||
if currentPage > 0 { |
|||
offset = (currentPage - 1) * global.Cfg().BulkSize |
|||
} |
|||
|
|||
return offset |
|||
} |
|||
|
|||
func (s *StockService) Close() { |
|||
s.canal.Close() |
|||
} |
|||
|
|||
func (s *StockService) incCounter(name string, n int64) int64 { |
|||
s.lockOfCounter.Lock() |
|||
defer s.lockOfCounter.Unlock() |
|||
|
|||
c, ok := s.counter[name] |
|||
if ok { |
|||
c = c + n |
|||
s.counter[name] = c |
|||
} |
|||
|
|||
return c |
|||
} |
|||
|
|||
func (s *StockService) completeRules() error { |
|||
wildcards := make(map[string]bool) |
|||
for _, rc := range global.Cfg().RuleConfigs { |
|||
if rc.Table == "*" { |
|||
return errors.Errorf("wildcard * is not allowed for table name") |
|||
} |
|||
|
|||
if regexp.QuoteMeta(rc.Table) != rc.Table { //通配符
|
|||
if _, ok := wildcards[global.RuleKey(rc.Schema, rc.Schema)]; ok { |
|||
return errors.Errorf("duplicate wildcard table defined for %s.%s", rc.Schema, rc.Table) |
|||
} |
|||
|
|||
tableName := rc.Table |
|||
if rc.Table == "*" { |
|||
tableName = "." + rc.Table |
|||
} |
|||
sql := fmt.Sprintf(`SELECT table_name FROM information_schema.tables WHERE |
|||
table_name RLIKE "%s" AND table_schema = "%s";`, tableName, rc.Schema) |
|||
res, err := s.canal.Execute(sql) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
for i := 0; i < res.Resultset.RowNumber(); i++ { |
|||
tableName, _ := res.GetString(i, 0) |
|||
newRule, err := global.RuleDeepClone(rc) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
newRule.Table = tableName |
|||
ruleKey := global.RuleKey(rc.Schema, tableName) |
|||
global.AddRuleIns(ruleKey, newRule) |
|||
} |
|||
} else { |
|||
newRule, err := global.RuleDeepClone(rc) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
ruleKey := global.RuleKey(rc.Schema, rc.Table) |
|||
global.AddRuleIns(ruleKey, newRule) |
|||
} |
|||
} |
|||
|
|||
for _, rule := range global.RuleInsList() { |
|||
tableMata, err := s.canal.GetTable(rule.Schema, rule.Table) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
if len(tableMata.PKColumns) == 0 { |
|||
if !global.Cfg().SkipNoPkTable { |
|||
return errors.Errorf("%s.%s must have a PK for a column", rule.Schema, rule.Table) |
|||
} |
|||
} |
|||
if len(tableMata.PKColumns) > 1 { |
|||
rule.IsCompositeKey = true // 组合主键
|
|||
} |
|||
rule.TableInfo = tableMata |
|||
rule.TableColumnSize = len(tableMata.Columns) |
|||
|
|||
if err := rule.Initialize(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
if err := rule.CompileLuaScript(global.Cfg().DataDir); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *StockService) addDumpDatabaseOrTable() { |
|||
var schema string |
|||
schemas := make(map[string]int) |
|||
tables := make([]string, 0, global.RuleInsTotal()) |
|||
for _, rule := range global.RuleInsList() { |
|||
schema = rule.Table |
|||
schemas[rule.Schema] = 1 |
|||
tables = append(tables, rule.Table) |
|||
} |
|||
if len(schemas) == 1 { |
|||
s.canal.AddDumpTables(schema, tables...) |
|||
} else { |
|||
keys := make([]string, 0, len(schemas)) |
|||
for key := range schemas { |
|||
keys = append(keys, key) |
|||
} |
|||
s.canal.AddDumpDatabases(keys...) |
|||
} |
|||
} |
|||
@ -0,0 +1,354 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package service |
|||
|
|||
import ( |
|||
"fmt" |
|||
"log" |
|||
"regexp" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/canal" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
"go.uber.org/atomic" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/service/endpoint" |
|||
"go-mysql-transfer/storage" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
const _transferLoopInterval = 1 |
|||
|
|||
type TransferService struct { |
|||
canal *canal.Canal |
|||
canalCfg *canal.Config |
|||
canalHandler *handler |
|||
canalEnable atomic.Bool |
|||
lockOfCanal sync.Mutex |
|||
firstsStart atomic.Bool |
|||
|
|||
wg sync.WaitGroup |
|||
endpoint endpoint.Endpoint |
|||
endpointEnable atomic.Bool |
|||
positionDao storage.PositionStorage |
|||
loopStopSignal chan struct{} |
|||
} |
|||
|
|||
func (s *TransferService) initialize() error { |
|||
s.canalCfg = canal.NewDefaultConfig() |
|||
s.canalCfg.Addr = global.Cfg().Addr |
|||
s.canalCfg.User = global.Cfg().User |
|||
s.canalCfg.Password = global.Cfg().Password |
|||
s.canalCfg.Charset = global.Cfg().Charset |
|||
s.canalCfg.Flavor = global.Cfg().Flavor |
|||
s.canalCfg.ServerID = global.Cfg().SlaveID |
|||
s.canalCfg.Dump.ExecutionPath = global.Cfg().DumpExec |
|||
s.canalCfg.Dump.DiscardErr = false |
|||
s.canalCfg.Dump.SkipMasterData = global.Cfg().SkipMasterData |
|||
|
|||
if err := s.createCanal(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if err := s.completeRules(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
s.addDumpDatabaseOrTable() |
|||
|
|||
positionDao := storage.NewPositionStorage() |
|||
if err := positionDao.Initialize(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
s.positionDao = positionDao |
|||
|
|||
// endpoint
|
|||
endpoint := endpoint.NewEndpoint(s.canal) |
|||
if err := endpoint.Connect(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
// 异步，必须要ping下才能确定连接成功
|
|||
if global.Cfg().IsMongodb() { |
|||
err := endpoint.Ping() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
s.endpoint = endpoint |
|||
s.endpointEnable.Store(true) |
|||
metrics.SetDestState(metrics.DestStateOK) |
|||
|
|||
s.firstsStart.Store(true) |
|||
s.startLoop() |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *TransferService) run() error { |
|||
current, err := s.positionDao.Get() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
s.wg.Add(1) |
|||
go func(p mysql.Position) { |
|||
s.canalEnable.Store(true) |
|||
log.Println(fmt.Sprintf("transfer run from position(%s %d)", p.Name, p.Pos)) |
|||
if err := s.canal.RunFrom(p); err != nil { |
|||
log.Println(fmt.Sprintf("start transfer : %v", err)) |
|||
logs.Errorf("canal : %v", errors.ErrorStack(err)) |
|||
if s.canalHandler != nil { |
|||
s.canalHandler.stopListener() |
|||
} |
|||
s.canalEnable.Store(false) |
|||
} |
|||
|
|||
logs.Info("Canal is Closed") |
|||
s.canalEnable.Store(false) |
|||
s.canal = nil |
|||
s.wg.Done() |
|||
}(current) |
|||
|
|||
// canal未提供回调，停留一秒，确保RunFrom启动成功
|
|||
time.Sleep(time.Second) |
|||
return nil |
|||
} |
|||
|
|||
func (s *TransferService) StartUp() { |
|||
s.lockOfCanal.Lock() |
|||
defer s.lockOfCanal.Unlock() |
|||
|
|||
if s.firstsStart.Load() { |
|||
s.canalHandler = newHandler() |
|||
s.canal.SetEventHandler(s.canalHandler) |
|||
s.canalHandler.startListener() |
|||
s.firstsStart.Store(false) |
|||
s.run() |
|||
} else { |
|||
s.restart() |
|||
} |
|||
} |
|||
|
|||
func (s *TransferService) restart() { |
|||
if s.canal != nil { |
|||
s.canal.Close() |
|||
s.wg.Wait() |
|||
} |
|||
|
|||
s.createCanal() |
|||
s.addDumpDatabaseOrTable() |
|||
s.canalHandler = newHandler() |
|||
s.canal.SetEventHandler(s.canalHandler) |
|||
s.canalHandler.startListener() |
|||
s.run() |
|||
} |
|||
|
|||
func (s *TransferService) stopDump() { |
|||
s.lockOfCanal.Lock() |
|||
defer s.lockOfCanal.Unlock() |
|||
|
|||
if s.canal == nil { |
|||
return |
|||
} |
|||
|
|||
if !s.canalEnable.Load() { |
|||
return |
|||
} |
|||
|
|||
if s.canalHandler != nil { |
|||
s.canalHandler.stopListener() |
|||
s.canalHandler = nil |
|||
} |
|||
|
|||
s.canal.Close() |
|||
s.wg.Wait() |
|||
|
|||
log.Println("dumper stopped") |
|||
} |
|||
|
|||
func (s *TransferService) Close() { |
|||
s.stopDump() |
|||
s.loopStopSignal <- struct{}{} |
|||
} |
|||
|
|||
func (s *TransferService) Position() (mysql.Position, error) { |
|||
return s.positionDao.Get() |
|||
} |
|||
|
|||
func (s *TransferService) createCanal() error { |
|||
for _, rc := range global.Cfg().RuleConfigs { |
|||
s.canalCfg.IncludeTableRegex = append(s.canalCfg.IncludeTableRegex, rc.Schema+"\\."+rc.Table) |
|||
} |
|||
var err error |
|||
s.canal, err = canal.NewCanal(s.canalCfg) |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
func (s *TransferService) completeRules() error { |
|||
wildcards := make(map[string]bool) |
|||
for _, rc := range global.Cfg().RuleConfigs { |
|||
if rc.Table == "*" { |
|||
return errors.Errorf("wildcard * is not allowed for table name") |
|||
} |
|||
|
|||
if regexp.QuoteMeta(rc.Table) != rc.Table { //通配符
|
|||
if _, ok := wildcards[global.RuleKey(rc.Schema, rc.Schema)]; ok { |
|||
return errors.Errorf("duplicate wildcard table defined for %s.%s", rc.Schema, rc.Table) |
|||
} |
|||
|
|||
tableName := rc.Table |
|||
if rc.Table == "*" { |
|||
tableName = "." + rc.Table |
|||
} |
|||
sql := fmt.Sprintf(`SELECT table_name FROM information_schema.tables WHERE |
|||
table_name RLIKE "%s" AND table_schema = "%s";`, tableName, rc.Schema) |
|||
res, err := s.canal.Execute(sql) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
for i := 0; i < res.Resultset.RowNumber(); i++ { |
|||
tableName, _ := res.GetString(i, 0) |
|||
newRule, err := global.RuleDeepClone(rc) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
newRule.Table = tableName |
|||
ruleKey := global.RuleKey(rc.Schema, tableName) |
|||
global.AddRuleIns(ruleKey, newRule) |
|||
} |
|||
} else { |
|||
newRule, err := global.RuleDeepClone(rc) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
ruleKey := global.RuleKey(rc.Schema, rc.Table) |
|||
global.AddRuleIns(ruleKey, newRule) |
|||
} |
|||
} |
|||
|
|||
for _, rule := range global.RuleInsList() { |
|||
tableMata, err := s.canal.GetTable(rule.Schema, rule.Table) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
if len(tableMata.PKColumns) == 0 { |
|||
if !global.Cfg().SkipNoPkTable { |
|||
return errors.Errorf("%s.%s must have a PK for a column", rule.Schema, rule.Table) |
|||
} |
|||
} |
|||
if len(tableMata.PKColumns) > 1 { |
|||
rule.IsCompositeKey = true // 组合主键
|
|||
} |
|||
rule.TableInfo = tableMata |
|||
rule.TableColumnSize = len(tableMata.Columns) |
|||
|
|||
if err := rule.Initialize(); err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if rule.LuaEnable() { |
|||
if err := rule.CompileLuaScript(global.Cfg().DataDir); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *TransferService) addDumpDatabaseOrTable() { |
|||
var schema string |
|||
schemas := make(map[string]int) |
|||
tables := make([]string, 0, global.RuleInsTotal()) |
|||
for _, rule := range global.RuleInsList() { |
|||
schema = rule.Table |
|||
schemas[rule.Schema] = 1 |
|||
tables = append(tables, rule.Table) |
|||
} |
|||
if len(schemas) == 1 { |
|||
s.canal.AddDumpTables(schema, tables...) |
|||
} else { |
|||
keys := make([]string, 0, len(schemas)) |
|||
for key := range schemas { |
|||
keys = append(keys, key) |
|||
} |
|||
s.canal.AddDumpDatabases(keys...) |
|||
} |
|||
} |
|||
|
|||
func (s *TransferService) updateRule(schema, table string) error { |
|||
rule, ok := global.RuleIns(global.RuleKey(schema, table)) |
|||
if ok { |
|||
tableInfo, err := s.canal.GetTable(schema, table) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if len(tableInfo.PKColumns) == 0 { |
|||
if !global.Cfg().SkipNoPkTable { |
|||
return errors.Errorf("%s.%s must have a PK for a column", rule.Schema, rule.Table) |
|||
} |
|||
} |
|||
|
|||
if len(tableInfo.PKColumns) > 1 { |
|||
rule.IsCompositeKey = true |
|||
} |
|||
|
|||
rule.TableInfo = tableInfo |
|||
rule.TableColumnSize = len(tableInfo.Columns) |
|||
|
|||
err = rule.AfterUpdateTableInfo() |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *TransferService) startLoop() { |
|||
go func() { |
|||
ticker := time.NewTicker(_transferLoopInterval * time.Second) |
|||
defer ticker.Stop() |
|||
for { |
|||
select { |
|||
case <-ticker.C: |
|||
if !s.endpointEnable.Load() { |
|||
err := s.endpoint.Ping() |
|||
if err != nil { |
|||
log.Println("destination not available,see the log file for details") |
|||
logs.Error(err.Error()) |
|||
} else { |
|||
s.endpointEnable.Store(true) |
|||
if global.Cfg().IsRabbitmq() { |
|||
s.endpoint.Connect() |
|||
} |
|||
s.StartUp() |
|||
metrics.SetDestState(metrics.DestStateOK) |
|||
} |
|||
} |
|||
case <-s.loopStopSignal: |
|||
return |
|||
} |
|||
} |
|||
}() |
|||
} |
|||
@ -0,0 +1,71 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package storage |
|||
|
|||
import ( |
|||
"github.com/juju/errors" |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
"github.com/vmihailenco/msgpack" |
|||
"go.etcd.io/bbolt" |
|||
) |
|||
|
|||
type boltPositionStorage struct { |
|||
Name string |
|||
Pos uint32 |
|||
} |
|||
|
|||
func (s *boltPositionStorage) Initialize() error { |
|||
return _bolt.Update(func(tx *bbolt.Tx) error { |
|||
bt := tx.Bucket(_positionBucket) |
|||
data := bt.Get(_fixPositionId) |
|||
if data != nil { |
|||
return nil |
|||
} |
|||
|
|||
bytes, err := msgpack.Marshal(mysql.Position{}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
return bt.Put(_fixPositionId, bytes) |
|||
}) |
|||
} |
|||
|
|||
func (s *boltPositionStorage) Save(pos mysql.Position) error { |
|||
return _bolt.Update(func(tx *bbolt.Tx) error { |
|||
bt := tx.Bucket(_positionBucket) |
|||
data, err := msgpack.Marshal(pos) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
return bt.Put(_fixPositionId, data) |
|||
}) |
|||
} |
|||
|
|||
func (s *boltPositionStorage) Get() (mysql.Position, error) { |
|||
var entity mysql.Position |
|||
err := _bolt.View(func(tx *bbolt.Tx) error { |
|||
bt := tx.Bucket(_positionBucket) |
|||
data := bt.Get(_fixPositionId) |
|||
if data == nil { |
|||
return errors.NotFoundf("PositionStorage") |
|||
} |
|||
return msgpack.Unmarshal(data, &entity) |
|||
}) |
|||
|
|||
return entity, err |
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package storage |
|||
|
|||
import "go-mysql-transfer/global" |
|||
|
|||
type ElectionStorage interface { |
|||
Elect() error |
|||
} |
|||
|
|||
func NewElectionStorage(conf *global.Config) PositionStorage { |
|||
if conf.IsCluster() { |
|||
if conf.IsZk() { |
|||
return &zkPositionStorage{} |
|||
} |
|||
if conf.IsEtcd() { |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,66 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package storage |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
|
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/util/etcds" |
|||
) |
|||
|
|||
type etcdPositionStorage struct { |
|||
} |
|||
|
|||
func (s *etcdPositionStorage) Initialize() error { |
|||
data, err := json.Marshal(mysql.Position{}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
err = etcds.CreateIfNecessary(global.Cfg().ZkPositionDir(), string(data), _etcdOps) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (s *etcdPositionStorage) Save(pos mysql.Position) error { |
|||
data, err := json.Marshal(pos) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return etcds.Save(global.Cfg().ZkPositionDir(), string(data), _etcdOps) |
|||
} |
|||
|
|||
func (s *etcdPositionStorage) Get() (mysql.Position, error) { |
|||
var entity mysql.Position |
|||
|
|||
data, _, err := etcds.Get(global.Cfg().ZkPositionDir(), _etcdOps) |
|||
if err != nil { |
|||
return entity, err |
|||
} |
|||
|
|||
err = json.Unmarshal(data, &entity) |
|||
|
|||
return entity, err |
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package storage |
|||
|
|||
import ( |
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
) |
|||
|
|||
type PositionStorage interface { |
|||
Initialize() error |
|||
Save(pos mysql.Position) error |
|||
Get() (mysql.Position, error) |
|||
} |
|||
|
|||
func NewPositionStorage() PositionStorage { |
|||
if global.Cfg().IsCluster() { |
|||
if global.Cfg().IsZk() { |
|||
return &zkPositionStorage{} |
|||
} |
|||
if global.Cfg().IsEtcd() { |
|||
return &etcdPositionStorage{} |
|||
} |
|||
} |
|||
|
|||
return &boltPositionStorage{} |
|||
} |
|||
@ -0,0 +1,185 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package storage |
|||
|
|||
import ( |
|||
"errors" |
|||
"fmt" |
|||
"path/filepath" |
|||
"strings" |
|||
"time" |
|||
|
|||
"github.com/samuel/go-zookeeper/zk" |
|||
"go.etcd.io/bbolt" |
|||
"go.etcd.io/etcd/clientv3" |
|||
etcdlog "go.etcd.io/etcd/pkg/logutil" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/util/byteutil" |
|||
"go-mysql-transfer/util/files" |
|||
"go-mysql-transfer/util/logagent" |
|||
"go-mysql-transfer/util/zookeepers" |
|||
) |
|||
|
|||
const ( |
|||
_boltFilePath = "db" |
|||
_boltFileName = "data.db" |
|||
_boltFileMode = 0600 |
|||
) |
|||
|
|||
var ( |
|||
_positionBucket = []byte("Position") |
|||
_fixPositionId = byteutil.Uint64ToBytes(uint64(1)) |
|||
|
|||
_bolt *bbolt.DB |
|||
_zkConn *zk.Conn |
|||
_zkStatusSignal <-chan zk.Event |
|||
_zkAddresses []string |
|||
|
|||
_etcdConn *clientv3.Client |
|||
_etcdOps clientv3.KV |
|||
) |
|||
|
|||
func Initialize() error { |
|||
if err := initBolt(); err != nil { |
|||
return err |
|||
} |
|||
|
|||
if global.Cfg().IsZk() { |
|||
if err := initZk(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
if global.Cfg().IsEtcd() { |
|||
if err := initEtcd(); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func initBolt() error { |
|||
blotStorePath := filepath.Join(global.Cfg().DataDir, _boltFilePath) |
|||
if err := files.MkdirIfNecessary(blotStorePath); err != nil { |
|||
return errors.New(fmt.Sprintf("create boltdb store : %s", err.Error())) |
|||
} |
|||
|
|||
boltFilePath := filepath.Join(blotStorePath, _boltFileName) |
|||
bolt, err := bbolt.Open(boltFilePath, _boltFileMode, bbolt.DefaultOptions) |
|||
if err != nil { |
|||
return errors.New(fmt.Sprintf("open boltdb: %s", err.Error())) |
|||
} |
|||
|
|||
err = bolt.Update(func(tx *bbolt.Tx) error { |
|||
tx.CreateBucketIfNotExists(_positionBucket) |
|||
return nil |
|||
}) |
|||
|
|||
_bolt = bolt |
|||
|
|||
return err |
|||
} |
|||
|
|||
func initZk() error { |
|||
option := zk.WithLogger(logagent.NewZkLoggerAgent()) |
|||
list := strings.Split(global.Cfg().Cluster.ZkAddrs, ",") |
|||
conn, sig, err := zk.Connect(list, time.Second, option) //*10)
|
|||
|
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
if global.Cfg().Cluster.ZkAuthentication != "" { |
|||
err = conn.AddAuth("digest", []byte(global.Cfg().Cluster.ZkAuthentication)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
err = zookeepers.CreateDirIfNecessary(global.Cfg().ZkRootDir(), conn) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
err = zookeepers.CreateDirIfNecessary(global.Cfg().ZkClusterDir(), conn) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
_zkAddresses = list |
|||
_zkConn = conn |
|||
_zkStatusSignal = sig |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func initEtcd() error { |
|||
etcdlog.DefaultZapLoggerConfig = logagent.EtcdZapLoggerConfig() |
|||
clientv3.SetLogger(logagent.NewEtcdLoggerAgent()) |
|||
|
|||
list := strings.Split(global.Cfg().Cluster.EtcdAddrs, ",") |
|||
config := clientv3.Config{ |
|||
Endpoints: list, |
|||
Username: global.Cfg().Cluster.EtcdUser, |
|||
Password: global.Cfg().Cluster.EtcdPassword, |
|||
DialTimeout: 1 * time.Second, |
|||
} |
|||
|
|||
client, err := clientv3.New(config) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
_etcdConn = client |
|||
_etcdOps = clientv3.NewKV(_etcdConn) |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func ZKConn() *zk.Conn { |
|||
return _zkConn |
|||
} |
|||
|
|||
func ZKStatusSignal() <-chan zk.Event { |
|||
return _zkStatusSignal |
|||
} |
|||
|
|||
func ZKAddresses() []string { |
|||
return _zkAddresses |
|||
} |
|||
|
|||
func EtcdConn() *clientv3.Client { |
|||
return _etcdConn |
|||
} |
|||
|
|||
func EtcdOps() clientv3.KV { |
|||
return _etcdOps |
|||
} |
|||
|
|||
func Close() { |
|||
if _bolt != nil { |
|||
_bolt.Close() |
|||
} |
|||
if _zkConn != nil { |
|||
_zkConn.Close() |
|||
} |
|||
if _etcdConn != nil { |
|||
_etcdConn.Close() |
|||
} |
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package storage |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
|
|||
"github.com/siddontang/go-mysql/mysql" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/util/zookeepers" |
|||
) |
|||
|
|||
type zkPositionStorage struct { |
|||
} |
|||
|
|||
func (s *zkPositionStorage) Initialize() error { |
|||
pos, err := json.Marshal(mysql.Position{}) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
err = zookeepers.CreateDirWithDataIfNecessary(global.Cfg().ZkPositionDir(), pos ,_zkConn) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
err = zookeepers.CreateDirIfNecessary(global.Cfg().ZkNodesDir(), _zkConn) |
|||
return err |
|||
} |
|||
|
|||
func (s *zkPositionStorage) Save(pos mysql.Position) error { |
|||
_, stat, err := _zkConn.Get(global.Cfg().ZkPositionDir()) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
data, err := json.Marshal(pos) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
_, err = _zkConn.Set(global.Cfg().ZkPositionDir(), data, stat.Version) |
|||
|
|||
return err |
|||
} |
|||
|
|||
func (s *zkPositionStorage) Get() (mysql.Position, error) { |
|||
var entity mysql.Position |
|||
|
|||
data, _, err := _zkConn.Get(global.Cfg().ZkPositionDir()) |
|||
if err != nil { |
|||
return entity, err |
|||
} |
|||
|
|||
err = json.Unmarshal(data, &entity) |
|||
|
|||
return entity, err |
|||
} |
|||
@ -0,0 +1,98 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package byteutil |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/binary" |
|||
"encoding/json" |
|||
) |
|||
|
|||
func StrToBytes(u string) []byte { |
|||
return []byte(u) |
|||
} |
|||
|
|||
func BytesToStr(u []byte) string { |
|||
return string(u) |
|||
} |
|||
|
|||
func Uint64ToBytes(u uint64) []byte { |
|||
buf := make([]byte, 8) |
|||
binary.BigEndian.PutUint64(buf, u) |
|||
return buf |
|||
} |
|||
|
|||
func Int64ToBytes(u int64) []byte { |
|||
buf := make([]byte, 8) |
|||
binary.BigEndian.PutUint64(buf, uint64(u)) |
|||
return buf |
|||
} |
|||
|
|||
func BytesToUint64(b []byte) uint64 { |
|||
if b == nil { |
|||
return 0 |
|||
} |
|||
return binary.BigEndian.Uint64(b) |
|||
} |
|||
|
|||
func BytesToInt64(b []byte) int64 { |
|||
if b == nil { |
|||
return 0 |
|||
} |
|||
return int64(binary.BigEndian.Uint64(b)) |
|||
} |
|||
|
|||
func Uint8ToBytes(u uint8) ([]byte, error) { |
|||
bytesBuffer := bytes.NewBuffer([]byte{}) |
|||
err := binary.Write(bytesBuffer, binary.BigEndian, &u) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return bytesBuffer.Bytes(), nil |
|||
} |
|||
|
|||
func BytesToUint8(b []byte) (uint8, error) { |
|||
bytesBuffer := bytes.NewBuffer(b) |
|||
var tmp uint8 |
|||
err := binary.Read(bytesBuffer, binary.BigEndian, &tmp) |
|||
if err != nil { |
|||
return 0, err |
|||
} |
|||
return tmp, nil |
|||
} |
|||
|
|||
func BytesToUint32(b []byte) uint32 { |
|||
if b == nil { |
|||
return 0 |
|||
} |
|||
return binary.BigEndian.Uint32(b) |
|||
} |
|||
|
|||
func Uint32ToBytes(u uint32) []byte { |
|||
buf := make([]byte, 4) |
|||
binary.BigEndian.PutUint32(buf, u) |
|||
return buf |
|||
} |
|||
|
|||
func JsonBytes(v interface{}) []byte { |
|||
bytes, err := json.Marshal(v) |
|||
if nil != err { |
|||
return nil |
|||
} |
|||
return bytes |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
package collections |
|||
|
|||
import "go-mysql-transfer/util/stringutil" |
|||
|
|||
func Contain(array []string, v interface{}) bool { |
|||
vvv := stringutil.ToString(v) |
|||
for _, vv := range array { |
|||
if vv == vvv { |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
} |
|||
@ -0,0 +1,82 @@ |
|||
package collections |
|||
|
|||
import ( |
|||
"sync" |
|||
) |
|||
|
|||
type BlockingQueue struct { |
|||
q *Queue |
|||
cond *sync.Cond |
|||
} |
|||
|
|||
func NewBlockingQueue() *BlockingQueue { |
|||
bq := &BlockingQueue{} |
|||
bq.q = NewQueue() |
|||
bq.cond = sync.NewCond(&bq.q.lock) |
|||
return bq |
|||
} |
|||
|
|||
// 将元素插入到队列末尾
|
|||
func (bq *BlockingQueue) Offer(value interface{}) { |
|||
bq.cond.L.Lock() |
|||
defer bq.cond.L.Unlock() |
|||
|
|||
element := &node{value: value} |
|||
if bq.q.size == 0 { |
|||
bq.q.first = element |
|||
bq.q.last = element |
|||
} else { |
|||
bq.q.last.next = element |
|||
bq.q.last = element |
|||
} |
|||
bq.q.size++ |
|||
|
|||
bq.cond.Signal() |
|||
} |
|||
|
|||
//获取队首元素，若成功，则返回队首元素；否则返回null
|
|||
func (bq *BlockingQueue) Peek() (interface{}, bool) { |
|||
return bq.q.Peek() |
|||
} |
|||
|
|||
//移除并获取队首元素，若成功，则返回队首元素；否则返回null
|
|||
func (bq *BlockingQueue) Poll() (interface{}, bool) { |
|||
return bq.q.Poll() |
|||
} |
|||
|
|||
func (bq *BlockingQueue) Size() int { |
|||
return bq.q.Size() |
|||
} |
|||
|
|||
func (bq *BlockingQueue) Clear() { |
|||
bq.q.Clear() |
|||
} |
|||
|
|||
// 移除并返回队列头部的元素, 如果队列为空，则阻塞
|
|||
func (bq *BlockingQueue) Take() interface{} { |
|||
bq.cond.L.Lock() |
|||
defer bq.cond.L.Unlock() |
|||
|
|||
for { |
|||
if bq.q.size == 0 { |
|||
bq.cond.Wait() // 此处唤醒
|
|||
} else { |
|||
break |
|||
} |
|||
} |
|||
|
|||
if bq.q.size == 1 { |
|||
val := bq.q.first.value |
|||
bq.q.first = nil |
|||
bq.q.last = nil |
|||
bq.q.size = 0 |
|||
return val |
|||
} |
|||
|
|||
element := bq.q.first |
|||
bq.q.first = bq.q.first.next |
|||
val := element.value |
|||
element = nil |
|||
bq.q.size-- |
|||
return val |
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package collections |
|||
|
|||
import "sync" |
|||
|
|||
type List struct { |
|||
lock sync.RWMutex //互斥锁
|
|||
|
|||
elements []interface{} |
|||
size int |
|||
} |
|||
|
|||
func NewList(values ...interface{}) *List { |
|||
l := &List{} |
|||
if len(values) > 0 { |
|||
l.Add(values...) |
|||
} |
|||
return l |
|||
} |
|||
|
|||
func (s *List) within(index int) bool { |
|||
return index >= 0 && index < s.size |
|||
} |
|||
|
|||
func (s *List) Add(values ...interface{}) { |
|||
s.lock.Lock() |
|||
defer s.lock.Unlock() |
|||
|
|||
if len(values) > 0 { |
|||
s.elements = append(s.elements, values) |
|||
s.size = s.size + len(values) |
|||
} |
|||
} |
|||
|
|||
func (s *List) Get(index int) interface{} { |
|||
s.lock.RLock() |
|||
defer s.lock.RUnlock() |
|||
|
|||
if !s.within(index) { |
|||
return nil |
|||
} |
|||
|
|||
return s.elements[index] |
|||
} |
|||
|
|||
func (s *List) Remove(index int) { |
|||
s.lock.Lock() |
|||
defer s.lock.Unlock() |
|||
|
|||
if !s.within(index) { |
|||
return |
|||
} |
|||
|
|||
s.elements = append(s.elements[:index], s.elements[index+1:]...) |
|||
|
|||
s.size-- |
|||
} |
|||
|
|||
func (s *List) Size() int { |
|||
s.lock.RLock() |
|||
defer s.lock.RUnlock() |
|||
|
|||
return s.size |
|||
} |
|||
@ -0,0 +1,111 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package collections |
|||
|
|||
import "sync" |
|||
|
|||
type Queue struct { |
|||
lock sync.RWMutex |
|||
|
|||
first *node |
|||
last *node |
|||
size int |
|||
} |
|||
|
|||
type node struct { |
|||
value interface{} |
|||
next *node |
|||
} |
|||
|
|||
func NewQueue(values ...interface{}) *Queue { |
|||
q := &Queue{} |
|||
if len(values) > 0 { |
|||
q.Offer(values...) |
|||
} |
|||
return q |
|||
} |
|||
|
|||
// 将元素插入到队列末尾
|
|||
func (q *Queue) Offer(values ...interface{}) { |
|||
q.lock.Lock() |
|||
defer q.lock.Unlock() |
|||
|
|||
for _, value := range values { |
|||
element := &node{value: value} |
|||
if q.size == 0 { |
|||
q.first = element |
|||
q.last = element |
|||
} else { |
|||
q.last.next = element |
|||
q.last = element |
|||
} |
|||
q.size++ |
|||
} |
|||
} |
|||
|
|||
//获取队首元素，若成功，则返回队首元素；否则返回null
|
|||
func (q *Queue) Peek() (interface{}, bool) { |
|||
q.lock.RLock() |
|||
defer q.lock.RUnlock() |
|||
|
|||
if q.size == 0 { |
|||
return nil, false |
|||
} |
|||
return q.first.value, true |
|||
} |
|||
|
|||
//移除并获取队首元素，若成功，则返回队首元素；否则返回null
|
|||
func (q *Queue) Poll() (interface{}, bool) { |
|||
q.lock.Lock() |
|||
defer q.lock.Unlock() |
|||
|
|||
if q.size == 0 { |
|||
return nil, false |
|||
} |
|||
|
|||
if q.size == 1 { |
|||
val := q.first.value |
|||
q.first = nil |
|||
q.last = nil |
|||
q.size = 0 |
|||
return val, true |
|||
} |
|||
|
|||
element := q.first |
|||
q.first = q.first.next |
|||
val := element.value |
|||
element = nil |
|||
q.size-- |
|||
return val, true |
|||
} |
|||
|
|||
func (q *Queue) Size() int { |
|||
q.lock.RLock() |
|||
defer q.lock.RUnlock() |
|||
|
|||
return q.size |
|||
} |
|||
|
|||
func (q *Queue) Clear() { |
|||
q.lock.Lock() |
|||
defer q.lock.Unlock() |
|||
|
|||
q.size = 0 |
|||
q.first = nil |
|||
q.last = nil |
|||
} |
|||
@ -0,0 +1,87 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package dates |
|||
|
|||
import ( |
|||
"time" |
|||
) |
|||
|
|||
const ( |
|||
// 日期格式
|
|||
DayFormatter = "2006-01-02" |
|||
// 日期时间格式--分
|
|||
DayTimeMinuteFormatter = "2006-01-02 15:04" |
|||
// 日期时间格式--秒
|
|||
DayTimeSecondFormatter = "2006-01-02 15:04:05" |
|||
// 日期时间格式--毫秒
|
|||
DayTimeMillisecondFormatter = "2006-01-02 15:04:05.sss" |
|||
// 时间格式--秒
|
|||
TimeSecondFormatter = "15:04:05" |
|||
) |
|||
|
|||
// 默认格式2006-01-02 15:04:05
|
|||
func NowFormatted() string { |
|||
return time.Now().Format(DayTimeSecondFormatter) |
|||
} |
|||
|
|||
func NowLayout(layout string) string { |
|||
return time.Now().Format(layout) |
|||
} |
|||
|
|||
func Layout(date time.Time, layout string) string { |
|||
return date.Format(layout) |
|||
} |
|||
|
|||
func DefaultLayout(time time.Time) string { |
|||
return time.Format(DayTimeSecondFormatter) |
|||
} |
|||
|
|||
func FromDefaultLayout(str string) time.Time { |
|||
loc, _ := time.LoadLocation("Local") |
|||
theTime, _ := time.ParseInLocation(DayTimeSecondFormatter, str, loc) |
|||
return theTime |
|||
} |
|||
|
|||
// 当前的毫秒时间戳
|
|||
func NowMillisecond() int64 { |
|||
return time.Now().UnixNano() / 1e6 |
|||
} |
|||
|
|||
func PastDayDate(pastDay int) time.Time { |
|||
return time.Now().AddDate(0, 0, -pastDay) |
|||
} |
|||
|
|||
func FutureDayDate(futureDay int) time.Time { |
|||
return time.Now().AddDate(0, 0, futureDay) |
|||
} |
|||
|
|||
func WeekStartDayDate() time.Time { |
|||
now := time.Now() |
|||
offset := int(time.Monday - now.Weekday()) |
|||
if offset > 0 { |
|||
offset = -6 |
|||
} |
|||
weekStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) |
|||
return weekStart |
|||
} |
|||
|
|||
func MonthStartDayDate() time.Time { |
|||
year, month, _ := time.Now().Date() |
|||
monthStart := time.Date(year, month, 1, 0, 0, 0, 0, time.Local) |
|||
return monthStart |
|||
} |
|||
@ -0,0 +1,124 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package dates |
|||
|
|||
import ( |
|||
"strings" |
|||
) |
|||
|
|||
const ( |
|||
yyyy = "2006" |
|||
yy = "06" |
|||
mmmm = "January" |
|||
mmm = "Jan" |
|||
mm = "04" |
|||
m = "1" |
|||
|
|||
dddd = "Monday" |
|||
ddd = "Mon" |
|||
dd = "02" |
|||
|
|||
HHT = "03" |
|||
HH = "15" |
|||
MM = "01" |
|||
SS = "05" |
|||
ss = "05" |
|||
tt = "PM" |
|||
Z = "MST" |
|||
ZZZ = "MST" |
|||
|
|||
o = "Z07:00" |
|||
) |
|||
|
|||
func ConvertGoFormat(format string) string { |
|||
var goFormate = format |
|||
if strings.Contains(goFormate, "YYYY") { |
|||
goFormate = strings.Replace(goFormate, "YYYY", yyyy, -1) |
|||
} else if strings.Contains(goFormate, "yyyy") { |
|||
goFormate = strings.Replace(goFormate, "yyyy", yyyy, -1) |
|||
} else if strings.Contains(goFormate, "YY") { |
|||
goFormate = strings.Replace(goFormate, "YY", yy, -1) |
|||
} else if strings.Contains(goFormate, "yy") { |
|||
goFormate = strings.Replace(goFormate, "yy", yy, -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "MMMM") { |
|||
goFormate = strings.Replace(goFormate, "MMMM", mmmm, -1) |
|||
} else if strings.Contains(goFormate, "mmmm") { |
|||
goFormate = strings.Replace(goFormate, "mmmm", mmmm, -1) |
|||
} else if strings.Contains(goFormate, "MMM") { |
|||
goFormate = strings.Replace(goFormate, "MMM", mmm, -1) |
|||
} else if strings.Contains(goFormate, "mmm") { |
|||
goFormate = strings.Replace(goFormate, "mmm", mmm, -1) |
|||
} else if strings.Contains(goFormate, "mm") { |
|||
goFormate = strings.Replace(goFormate, "mm", mm, -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "dddd") { |
|||
goFormate = strings.Replace(goFormate, "dddd", dddd, -1) |
|||
} else if strings.Contains(goFormate, "ddd") { |
|||
goFormate = strings.Replace(goFormate, "ddd", ddd, -1) |
|||
} else if strings.Contains(goFormate, "dd") { |
|||
goFormate = strings.Replace(goFormate, "dd", dd, -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "tt") { |
|||
if strings.Contains(goFormate, "HH") { |
|||
goFormate = strings.Replace(goFormate, "HH", HHT, -1) |
|||
} else if strings.Contains(goFormate, "hh") { |
|||
goFormate = strings.Replace(goFormate, "hh", HHT, -1) |
|||
} |
|||
goFormate = strings.Replace(goFormate, "tt", tt, -1) |
|||
} else { |
|||
if strings.Contains(goFormate, "HH") { |
|||
goFormate = strings.Replace(goFormate, "HH", HH, -1) |
|||
} else if strings.Contains(goFormate, "hh") { |
|||
goFormate = strings.Replace(goFormate, "hh", HH, -1) |
|||
} |
|||
goFormate = strings.Replace(goFormate, "tt", "", -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "MM") { |
|||
goFormate = strings.Replace(goFormate, "MM", MM, -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "SS") { |
|||
goFormate = strings.Replace(goFormate, "SS", SS, -1) |
|||
} else if strings.Contains(goFormate, "ss") { |
|||
goFormate = strings.Replace(goFormate, "ss", SS, -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "ZZZ") { |
|||
goFormate = strings.Replace(goFormate, "ZZZ", ZZZ, -1) |
|||
} else if strings.Contains(goFormate, "zzz") { |
|||
goFormate = strings.Replace(goFormate, "zzz", ZZZ, -1) |
|||
} else if strings.Contains(goFormate, "Z") { |
|||
goFormate = strings.Replace(goFormate, "Z", Z, -1) |
|||
} else if strings.Contains(goFormate, "z") { |
|||
goFormate = strings.Replace(goFormate, "z", Z, -1) |
|||
} |
|||
|
|||
if strings.Contains(goFormate, "tt") { |
|||
goFormate = strings.Replace(goFormate, "tt", tt, -1) |
|||
} |
|||
if strings.Contains(goFormate, "o") { |
|||
goFormate = strings.Replace(goFormate, "o", o, -1) |
|||
} |
|||
|
|||
return goFormate |
|||
} |
|||
@ -0,0 +1,166 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package etcds |
|||
|
|||
import ( |
|||
"context" |
|||
"time" |
|||
|
|||
"github.com/juju/errors" |
|||
"go.etcd.io/etcd/clientv3" |
|||
) |
|||
|
|||
const _etcdOpsTimeout = 1 * time.Second |
|||
|
|||
type Node struct { |
|||
Key string |
|||
Value []byte |
|||
Revision int64 |
|||
} |
|||
|
|||
func CreateIfNecessary(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
_, err := ops.Txn(ctx).If( |
|||
clientv3.Compare(clientv3.ModRevision(key), "=", 0), |
|||
).Then( |
|||
clientv3.OpPut(key, val, opts...), |
|||
).Commit() |
|||
|
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
//if !resp.Succeeded {
|
|||
// return errors.AlreadyExistsf("key %s in etcd", key)
|
|||
//}
|
|||
|
|||
return nil |
|||
} |
|||
|
|||
func Get(key string, ops clientv3.KV) ([]byte, int64, error) { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
resp, err := ops.Get(ctx, key) |
|||
if err != nil { |
|||
return nil, -1, errors.Trace(err) |
|||
} |
|||
|
|||
if len(resp.Kvs) == 0 { |
|||
return nil, -1, errors.NotFoundf("key %s in etcd", key) |
|||
} |
|||
|
|||
return resp.Kvs[0].Value, resp.Header.Revision, nil |
|||
} |
|||
|
|||
func HasChildren(key string, ops clientv3.KV) (bool, error) { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
resp, err := ops.Get(ctx, key, clientv3.WithPrefix()) |
|||
if err != nil { |
|||
return false, errors.Trace(err) |
|||
} |
|||
|
|||
length := len(key) |
|||
for _, kv := range resp.Kvs { |
|||
key := string(kv.Key) |
|||
if len(key) > length { |
|||
return true, nil |
|||
} |
|||
} |
|||
|
|||
return false, nil |
|||
} |
|||
|
|||
func List(key string, ops clientv3.KV) (map[string]*Node, error) { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
ret := make(map[string]*Node) |
|||
|
|||
resp, err := ops.Get(ctx, key, clientv3.WithPrefix()) |
|||
if err != nil { |
|||
return ret, errors.Trace(err) |
|||
} |
|||
|
|||
length := len(key) |
|||
for _, kv := range resp.Kvs { |
|||
key := string(kv.Key) |
|||
if len(key) <= length { |
|||
continue |
|||
} |
|||
node := &Node{ |
|||
Key: key, |
|||
Value: kv.Value, |
|||
Revision: kv.Version, |
|||
} |
|||
ret[key] = node |
|||
} |
|||
|
|||
return ret, nil |
|||
} |
|||
|
|||
func Save(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
resp, err := ops.Txn(ctx).If( |
|||
clientv3.Compare(clientv3.ModRevision(key), ">", 0), |
|||
).Then( |
|||
clientv3.OpPut(key, val, opts...), |
|||
).Commit() |
|||
|
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
if !resp.Succeeded { |
|||
return errors.NotFoundf("key %s in etcd", key) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
// UpdateOrCreate updates a key/value, if the key does not exist then create, or update
|
|||
func UpdateOrCreate(key, val string, ops clientv3.KV, opts ...clientv3.OpOption) error { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
_, err := ops.Do(ctx, clientv3.OpPut(key, val, opts...)) |
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func Delete(key string, ops clientv3.KV, opts ...clientv3.OpOption) error { |
|||
ctx, cancel := context.WithTimeout(context.Background(), _etcdOpsTimeout) |
|||
defer cancel() |
|||
|
|||
_, err := ops.Delete(ctx, key, opts...) |
|||
|
|||
if err != nil { |
|||
return errors.Trace(err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,49 @@ |
|||
package files |
|||
|
|||
import ( |
|||
"os" |
|||
) |
|||
|
|||
// 判断给定的文件路径是否存在
|
|||
func IsExist(path string) bool { |
|||
_, err := os.Stat(path) |
|||
if err == nil { |
|||
return true |
|||
} |
|||
if os.IsNotExist(err) { |
|||
return false |
|||
} |
|||
return false |
|||
} |
|||
|
|||
// 判断给定的路径是否是文件夹
|
|||
func IsDir(path string) bool { |
|||
if stat, err := os.Stat(path); err == nil { |
|||
return stat.IsDir() |
|||
} |
|||
return false |
|||
} |
|||
|
|||
// 给定的文件不存在则创建
|
|||
func CreateFileIfNecessary(path string) bool { |
|||
_, err := os.Stat(path) |
|||
if err != nil && os.IsNotExist(err) { |
|||
if file, err := os.Create(path); err == nil { |
|||
file.Close() |
|||
} |
|||
} |
|||
exist := IsExist(path) |
|||
return exist |
|||
} |
|||
|
|||
// 给定的目录不存在则创建
|
|||
func MkdirIfNecessary(path string) error { |
|||
_, err := os.Stat(path) |
|||
if err != nil && os.IsNotExist(err) { |
|||
if err := os.MkdirAll(path, os.ModePerm); err != nil { |
|||
// os.Chmod(path, 0777)
|
|||
return err |
|||
} |
|||
} |
|||
return nil |
|||
} |
|||
@ -0,0 +1,68 @@ |
|||
package httpclient |
|||
|
|||
import "net/http" |
|||
|
|||
// 参数
|
|||
type H map[string]interface{} |
|||
|
|||
// 重试条件
|
|||
type RetryConditionFunc func(*http.Response) bool |
|||
|
|||
// 请求条件
|
|||
type requestCriteria struct { |
|||
timeout int |
|||
retryCount int |
|||
retryInterval int |
|||
retryConditions []RetryConditionFunc |
|||
headers H |
|||
} |
|||
|
|||
func newRequestCriteria() *requestCriteria { |
|||
return &requestCriteria{ |
|||
headers: make(H), |
|||
} |
|||
} |
|||
|
|||
// 设置请求头
|
|||
func (c *requestCriteria) AddHeader(name string, value interface{}) { |
|||
c.headers[name] = value |
|||
} |
|||
|
|||
// 设置请求头
|
|||
func (c *requestCriteria) AddHeaders(values H) { |
|||
for k, v := range values { |
|||
c.headers[k] = v |
|||
} |
|||
} |
|||
|
|||
// 设置超时时间，单位为秒
|
|||
func (c *requestCriteria) SetTimeout(_timeout int) { |
|||
c.timeout = _timeout |
|||
} |
|||
|
|||
// 设置重试次数
|
|||
func (c *requestCriteria) SetRetryCount(_retryCount int) { |
|||
c.retryCount = _retryCount |
|||
} |
|||
|
|||
// 设置重试间隔时间，单位为秒
|
|||
func (c *requestCriteria) SetRetryInterval(_retryInterval int) { |
|||
c.retryInterval = _retryInterval |
|||
} |
|||
|
|||
// 添加重试条件
|
|||
func (c *requestCriteria) AddRetryConditionFunc(_retryCondition RetryConditionFunc) { |
|||
if _retryCondition != nil { |
|||
c.retryConditions = append(c.retryConditions, _retryCondition) |
|||
} |
|||
} |
|||
|
|||
// 判断是否需要重试
|
|||
func (c *requestCriteria) needRetry(res *http.Response) bool { |
|||
for _, condition := range c.retryConditions { |
|||
if condition(res) { |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
} |
|||
@ -0,0 +1,389 @@ |
|||
package httpclient |
|||
|
|||
import ( |
|||
"bytes" |
|||
"encoding/json" |
|||
"io" |
|||
"io/ioutil" |
|||
"mime/multipart" |
|||
"net/http" |
|||
"net/url" |
|||
"os" |
|||
"strings" |
|||
"time" |
|||
|
|||
"github.com/pkg/errors" |
|||
|
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
const ( |
|||
_contentTypeForm = 1 |
|||
_contentTypeJson = 2 |
|||
) |
|||
|
|||
type FormFile string |
|||
|
|||
// 执行器
|
|||
type executor struct { |
|||
client *HttpClient |
|||
criteria *requestCriteria |
|||
|
|||
addr string |
|||
method string |
|||
expectStatus int |
|||
} |
|||
|
|||
type GetOrDeleteExecutor struct { |
|||
executor |
|||
parameters H |
|||
} |
|||
|
|||
type PostOrPutExecutor struct { |
|||
executor |
|||
body interface{} |
|||
contentType int |
|||
} |
|||
|
|||
// 全局Criteria覆盖本地Criteria
|
|||
func (s *executor) overrideCriteria() { |
|||
global := s.client.criteria |
|||
local := s.criteria |
|||
|
|||
if local.retryCount == 0 { |
|||
local.retryCount = global.retryCount |
|||
} |
|||
|
|||
if local.retryInterval == 0 { |
|||
local.retryInterval = global.retryInterval |
|||
} |
|||
|
|||
for _, retryCondition := range global.retryConditions { |
|||
local.retryConditions = append(local.retryConditions, retryCondition) |
|||
} |
|||
|
|||
for k, v := range global.headers { |
|||
if _, exist := local.headers[k]; !exist { |
|||
local.headers[k] = v |
|||
} |
|||
} |
|||
} |
|||
|
|||
// 执行Request
|
|||
func (s *executor) execute(request *http.Request) (*http.Response, error) { |
|||
s.overrideCriteria() |
|||
|
|||
for k, v := range s.criteria.headers { |
|||
request.Header.Add(k, stringutil.ToString(v)) |
|||
} |
|||
|
|||
startTime := time.Now().UnixNano() |
|||
res, err := s.client.inner.Do(request) |
|||
latency := (time.Now().UnixNano() - startTime) / int64(time.Millisecond) |
|||
|
|||
if nil == err { |
|||
s.client.logger.Sugar().Infof("请求成功, %s | %s | %d | %d(毫秒)", request.Method, request.URL.String(), res.StatusCode, latency) |
|||
} |
|||
|
|||
if s.criteria.retryCount > 0 && s.criteria.needRetry(res) { |
|||
for i := 0; i < s.criteria.retryCount; i++ { |
|||
s.client.logger.Sugar().Infof("第%d次重试： %s | %s )", i+1, request.Method, request.URL.String()) |
|||
|
|||
res, err = s.client.inner.Do(request) |
|||
if err != nil { |
|||
s.client.logger.Error(err.Error()) |
|||
} |
|||
if !s.criteria.needRetry(res) || (i+1) == s.criteria.retryCount { |
|||
return res, err |
|||
} |
|||
<-time.After(time.Duration(s.criteria.retryInterval) * time.Second) |
|||
} |
|||
} |
|||
|
|||
if s.expectStatus != 0 && s.expectStatus != res.StatusCode { |
|||
defer res.Body.Close() |
|||
return nil, errors.Errorf("Response status code : %d (%s)", res.StatusCode, http.StatusText(res.StatusCode)) |
|||
} |
|||
|
|||
return res, err |
|||
} |
|||
|
|||
// 转换Response为string
|
|||
func (s *executor) responseAsString(response *http.Response) (string, error) { |
|||
defer response.Body.Close() |
|||
|
|||
if data, err := ioutil.ReadAll(response.Body); err == nil { |
|||
return string(data), nil |
|||
} |
|||
|
|||
return "", nil |
|||
} |
|||
|
|||
// 转换Response为string
|
|||
func (s *executor) responseAsEntity(response *http.Response) (*RespondEntity, error) { |
|||
defer response.Body.Close() |
|||
|
|||
data, err := ioutil.ReadAll(response.Body) |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
|
|||
return &RespondEntity{ |
|||
statusCode: response.StatusCode, |
|||
data: data, |
|||
}, nil |
|||
} |
|||
|
|||
// 设置请求头
|
|||
func (r *GetOrDeleteExecutor) AddHeader(name string, value interface{}) *GetOrDeleteExecutor { |
|||
r.criteria.AddHeader(name, value) |
|||
return r |
|||
} |
|||
|
|||
// 设置请求头
|
|||
func (r *GetOrDeleteExecutor) SetHeaders(values H) *GetOrDeleteExecutor { |
|||
r.criteria.AddHeaders(values) |
|||
return r |
|||
} |
|||
|
|||
// 设置重试次数
|
|||
func (r *GetOrDeleteExecutor) SetRetryCount(_retryCount int) *GetOrDeleteExecutor { |
|||
r.criteria.SetRetryCount(_retryCount) |
|||
return r |
|||
} |
|||
|
|||
// 设置重试间隔时间，单位为秒
|
|||
func (r *GetOrDeleteExecutor) SetRetryInterval(_retryInterval int) *GetOrDeleteExecutor { |
|||
r.criteria.SetRetryInterval(_retryInterval) |
|||
return r |
|||
} |
|||
|
|||
// 添加重试条件
|
|||
func (r *GetOrDeleteExecutor) AddRetryConditionFunc(_retryCondition RetryConditionFunc) *GetOrDeleteExecutor { |
|||
r.criteria.AddRetryConditionFunc(_retryCondition) |
|||
return r |
|||
} |
|||
|
|||
// 设置查询参数
|
|||
func (r *GetOrDeleteExecutor) AddParameter(name string, value interface{}) *GetOrDeleteExecutor { |
|||
r.parameters[name] = value |
|||
return r |
|||
} |
|||
|
|||
// 设置查询参数
|
|||
func (r *GetOrDeleteExecutor) AddParameters(values H) *GetOrDeleteExecutor { |
|||
for k, v := range values { |
|||
r.parameters[k] = v |
|||
} |
|||
return r |
|||
} |
|||
|
|||
// 设置预期请求状态
|
|||
func (r *GetOrDeleteExecutor) SetExpectStatus(expect int) *GetOrDeleteExecutor { |
|||
r.expectStatus = expect |
|||
return r |
|||
} |
|||
|
|||
// 执行请求
|
|||
func (r *GetOrDeleteExecutor) Do() (*http.Response, error) { |
|||
values := make(url.Values) |
|||
for k, v := range r.parameters { |
|||
values.Set(k, stringutil.ToString(v)) |
|||
} |
|||
|
|||
url := stringutil.UrlValuesToQueryString(r.addr, values) |
|||
req, err := http.NewRequest(r.method, url, nil) |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
|
|||
return r.execute(req) |
|||
} |
|||
|
|||
func (r *GetOrDeleteExecutor) DoForString() (string, error) { |
|||
res, err := r.Do() |
|||
if nil != err { |
|||
return "", err |
|||
} |
|||
return r.responseAsString(res) |
|||
} |
|||
|
|||
func (r *GetOrDeleteExecutor) DoForEntity() (*RespondEntity, error) { |
|||
res, err := r.Do() |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
return r.responseAsEntity(res) |
|||
} |
|||
|
|||
// 设置请求头
|
|||
func (r *PostOrPutExecutor) AddHeader(name string, value interface{}) *PostOrPutExecutor { |
|||
r.criteria.AddHeader(name, value) |
|||
return r |
|||
} |
|||
|
|||
// 设置请求头
|
|||
func (r *PostOrPutExecutor) SetHeaders(values H) *PostOrPutExecutor { |
|||
r.criteria.AddHeaders(values) |
|||
return r |
|||
} |
|||
|
|||
// 设置重试次数
|
|||
func (r *PostOrPutExecutor) SetRetryCount(_retryCount int) *PostOrPutExecutor { |
|||
r.criteria.SetRetryCount(_retryCount) |
|||
return r |
|||
} |
|||
|
|||
// 设置重试间隔时间，单位为秒
|
|||
func (r *PostOrPutExecutor) SetRetryInterval(_retryInterval int) *PostOrPutExecutor { |
|||
r.criteria.SetRetryInterval(_retryInterval) |
|||
return r |
|||
} |
|||
|
|||
// 添加重试条件
|
|||
func (r *PostOrPutExecutor) AddRetryConditionFunc(_retryCondition RetryConditionFunc) *PostOrPutExecutor { |
|||
r.criteria.AddRetryConditionFunc(_retryCondition) |
|||
return r |
|||
} |
|||
|
|||
// 设置预期请求状态
|
|||
func (r *PostOrPutExecutor) SetExpectStatus(expect int) *PostOrPutExecutor { |
|||
r.expectStatus = expect |
|||
return r |
|||
} |
|||
|
|||
func (r *PostOrPutExecutor) SetBodyAsForm(body H) *PostOrPutExecutor { |
|||
r.contentType = _contentTypeForm |
|||
r.body = body |
|||
return r |
|||
} |
|||
|
|||
// 设置请求的contentType 为：
|
|||
// "application/json"
|
|||
func (r *PostOrPutExecutor) SetBodyAsJson(body interface{}) *PostOrPutExecutor { |
|||
r.contentType = _contentTypeJson |
|||
r.body = body |
|||
return r |
|||
} |
|||
|
|||
// 执行请求
|
|||
func (r *PostOrPutExecutor) Do() (*http.Response, error) { |
|||
if _contentTypeForm == r.contentType { |
|||
return r.doFormRequest() |
|||
} |
|||
|
|||
var data []byte |
|||
switch r.body.(type) { |
|||
case string: |
|||
ft := r.body.(string) |
|||
data = []byte(ft) |
|||
case []byte: |
|||
data = r.body.([]byte) |
|||
default: |
|||
temp, err := json.Marshal(r.body) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
data = temp |
|||
} |
|||
|
|||
req, err := http.NewRequest(r.method, r.addr, bytes.NewReader(data)) |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
req.Header.Add("Content-Type", "application/json") |
|||
return r.execute(req) |
|||
} |
|||
|
|||
func (r *PostOrPutExecutor) DoForString() (string, error) { |
|||
res, err := r.Do() |
|||
if nil != err { |
|||
return "", err |
|||
} |
|||
return r.responseAsString(res) |
|||
} |
|||
|
|||
func (r *PostOrPutExecutor) DoForEntity() (*RespondEntity, error) { |
|||
res, err := r.Do() |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
return r.responseAsEntity(res) |
|||
} |
|||
|
|||
func (r *PostOrPutExecutor) doFormRequest() (*http.Response, error) { |
|||
data := r.body.(H) |
|||
isMultipart := false |
|||
values := make(url.Values) |
|||
for k, v := range data { |
|||
if _, ok := v.(FormFile); ok { |
|||
isMultipart = true |
|||
values = nil |
|||
break |
|||
} |
|||
values.Set(k, stringutil.ToString(v)) |
|||
} |
|||
|
|||
if !isMultipart { |
|||
req, err := http.NewRequest(r.method, r.addr, strings.NewReader(values.Encode())) |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
req.Header.Add("Content-Type", "application/x-www-form-urlencoded") |
|||
return r.execute(req) |
|||
} |
|||
|
|||
var err error |
|||
bodyBuffer := &bytes.Buffer{} |
|||
bodyWriter := multipart.NewWriter(bodyBuffer) |
|||
|
|||
var closings []*os.File |
|||
defer func() { |
|||
for _, closing := range closings { |
|||
closing.Close() |
|||
} |
|||
}() |
|||
|
|||
for k, v := range data { |
|||
switch v.(type) { |
|||
case FormFile: |
|||
vv := v.(FormFile) |
|||
var fw io.Writer |
|||
fw, err = bodyWriter.CreateFormFile(k, string(vv)) |
|||
if err != nil { |
|||
break |
|||
} |
|||
|
|||
var file *os.File |
|||
file, err = os.Open(string(vv)) |
|||
if err != nil { |
|||
break |
|||
} |
|||
closings = append(closings, file) |
|||
|
|||
_, err = io.Copy(fw, file) |
|||
if err != nil { |
|||
break |
|||
} |
|||
default: |
|||
if err := bodyWriter.WriteField(k, stringutil.ToString(v)); err != nil { |
|||
return nil, err |
|||
} |
|||
} |
|||
} |
|||
|
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
bodyWriter.Close() |
|||
|
|||
var req *http.Request |
|||
req, err = http.NewRequest(r.method, r.addr, bodyBuffer) |
|||
if nil != err { |
|||
return nil, err |
|||
} |
|||
|
|||
req.Header.Add("Content-Type", bodyWriter.FormDataContentType()) |
|||
return r.execute(req) |
|||
} |
|||
@ -0,0 +1,156 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
|
|||
package httpclient |
|||
|
|||
import ( |
|||
"net/http" |
|||
"time" |
|||
|
|||
"go.uber.org/zap" |
|||
|
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
var DefaultClient = NewClient() |
|||
|
|||
type HttpClient struct { |
|||
logger *zap.Logger |
|||
inner *http.Client |
|||
criteria *requestCriteria |
|||
} |
|||
|
|||
// 创建Client
|
|||
func NewClient() *HttpClient { |
|||
return &HttpClient{ |
|||
logger: logs.Logger(), |
|||
inner: &http.Client{}, |
|||
criteria: newRequestCriteria(), |
|||
} |
|||
} |
|||
|
|||
// 设置超时时间，单位为秒
|
|||
func (c *HttpClient) SetTimeout(timeout int) *HttpClient { |
|||
if timeout > 0 { |
|||
c.criteria.SetTimeout(timeout) |
|||
c.inner.Timeout = time.Duration(timeout) * time.Second |
|||
} |
|||
return c |
|||
} |
|||
|
|||
func (c *HttpClient) SetLogger(logger *zap.Logger) { |
|||
c.logger = logger |
|||
} |
|||
|
|||
// 获取超时时间
|
|||
func (c *HttpClient) GetTimeout() int { |
|||
return c.criteria.timeout |
|||
} |
|||
|
|||
// 设置重试次数
|
|||
func (c *HttpClient) SetRetryCount(retryCount int) *HttpClient { |
|||
c.criteria.SetRetryCount(retryCount) |
|||
return c |
|||
} |
|||
|
|||
// 获取重试次数
|
|||
func (c *HttpClient) GetRetryCount() int { |
|||
return c.criteria.retryCount |
|||
} |
|||
|
|||
// 设置重试间隔时间，单位为秒
|
|||
func (c *HttpClient) SetRetryInterval(retryInterval int) *HttpClient { |
|||
c.criteria.SetRetryInterval(retryInterval) |
|||
return c |
|||
} |
|||
|
|||
// 获取重试间隔时间
|
|||
func (c *HttpClient) GetRetryInterval() int { |
|||
return c.criteria.retryInterval |
|||
} |
|||
|
|||
// 添加重试条件
|
|||
func (c *HttpClient) AddRetryConditionFunc(retryCondition RetryConditionFunc) *HttpClient { |
|||
c.criteria.AddRetryConditionFunc(retryCondition) |
|||
return c |
|||
} |
|||
|
|||
// 设置Transport （用于确定HTTP请求的创建机制）
|
|||
// 如果为空，将会使用DefaultTransport
|
|||
func (c *HttpClient) SetTransport(transport http.RoundTripper) *HttpClient { |
|||
if transport != nil { |
|||
c.inner.Transport = transport |
|||
} |
|||
return c |
|||
} |
|||
|
|||
// 添加请求头
|
|||
func (c *HttpClient) AddHeader(key string, val string) *HttpClient { |
|||
c.criteria.AddHeader(key, val) |
|||
return c |
|||
} |
|||
|
|||
// 添加请求头
|
|||
func (c *HttpClient) AddHeaders(values H) *HttpClient { |
|||
c.criteria.AddHeaders(values) |
|||
return c |
|||
} |
|||
|
|||
// Get请求
|
|||
func (c *HttpClient) GET(url string) *GetOrDeleteExecutor { |
|||
t := &GetOrDeleteExecutor{ |
|||
parameters: make(H), |
|||
} |
|||
t.client = c |
|||
t.method = http.MethodGet |
|||
t.addr = url |
|||
t.criteria = newRequestCriteria() |
|||
return t |
|||
} |
|||
|
|||
// Delete请求
|
|||
func (c *HttpClient) DELETE(url string) *GetOrDeleteExecutor { |
|||
t := &GetOrDeleteExecutor{ |
|||
parameters: make(H), |
|||
} |
|||
t.client = c |
|||
t.method = http.MethodDelete |
|||
t.addr = url |
|||
t.criteria = newRequestCriteria() |
|||
return t |
|||
} |
|||
|
|||
// Post请求
|
|||
func (c *HttpClient) POST(url string) *PostOrPutExecutor { |
|||
t := &PostOrPutExecutor{} |
|||
t.client = c |
|||
t.method = http.MethodPost |
|||
t.addr = url |
|||
t.criteria = newRequestCriteria() |
|||
return t |
|||
} |
|||
|
|||
// Put请求
|
|||
func (c *HttpClient) PUT(url string) *PostOrPutExecutor { |
|||
t := &PostOrPutExecutor{} |
|||
t.client = c |
|||
t.method = http.MethodPut |
|||
t.addr = url |
|||
t.criteria = newRequestCriteria() |
|||
return t |
|||
} |
|||
@ -0,0 +1,269 @@ |
|||
package httpclient |
|||
|
|||
import ( |
|||
"fmt" |
|||
"io/ioutil" |
|||
"net/http" |
|||
"testing" |
|||
) |
|||
|
|||
func TestHttpClientGet(t *testing.T) { |
|||
res, err := DefaultClient.GET("http://httpbin.org/get").Do() |
|||
if err != nil { |
|||
t.Error("get failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
} |
|||
|
|||
func TestHttpClientTimeout(t *testing.T) { |
|||
res, err := DefaultClient.SetTimeout(10).GET("http://127.0.0.1:8801/zombie").Do() |
|||
if err != nil { |
|||
t.Error("get failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
} |
|||
|
|||
func TestHttpClientGetForString(t *testing.T) { |
|||
str, err := DefaultClient.GET("http://httpbin.org/get").DoForString() |
|||
if err != nil { |
|||
t.Error("get failed", err) |
|||
} |
|||
|
|||
println(str) |
|||
} |
|||
|
|||
func TestHttpClientGetWithParameters(t *testing.T) { |
|||
// 实际请求地址：http://httpbin.org/get?age=18&name=wf
|
|||
str, err := DefaultClient.GET("http://httpbin.org/get").AddParameters(H{ |
|||
"token": "12596358412", |
|||
}).AddParameters(H{ |
|||
"name": "wf", |
|||
"age": 18, |
|||
}).DoForString() |
|||
|
|||
if err != nil { |
|||
t.Error("get failed", err) |
|||
} |
|||
|
|||
println(str) |
|||
} |
|||
|
|||
func retryCondition(res *http.Response) bool { |
|||
if res == nil { // Response为空时候重试
|
|||
return true |
|||
} |
|||
if res.StatusCode != http.StatusOK { // 状态不是200的时候重试
|
|||
return true |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
func TestHttpClientRetry(t *testing.T) { |
|||
NewClient(). |
|||
SetRetryCount(3). // 重试次数
|
|||
SetRetryInterval(5). // 重试间隔 （秒）
|
|||
AddRetryConditionFunc(retryCondition). // 重试条件
|
|||
GET("http://test.org/get"). |
|||
Do() |
|||
|
|||
// log like:
|
|||
//{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"}
|
|||
//{"level":"info","msg":"第1次重试： GET | http://test.org/get )"}
|
|||
//{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"}
|
|||
//{"level":"info","msg":"第2次重试： GET | http://test.org/get )"}
|
|||
//{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"}
|
|||
//{"level":"info","msg":"第3次重试： GET | http://test.org/get )"}
|
|||
//{"level":"error","msg":"Get \"http://test.org/get\": dial tcp: lookup test.org: no such host"}
|
|||
} |
|||
|
|||
func TestHttpClientPostForm(t *testing.T) { |
|||
// Content-Type is : application/x-www-form-urlencoded
|
|||
res, err := DefaultClient.POST("http://httpbin.org/post"). |
|||
SetBodyAsForm(H{ |
|||
"name": "wf", |
|||
"age": 18, |
|||
}).Do() |
|||
|
|||
if err != nil { |
|||
t.Error("post failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
|
|||
data, err := ioutil.ReadAll(res.Body) |
|||
if nil != err { |
|||
t.Error("read failed", err) |
|||
} |
|||
|
|||
fmt.Println(string(data)) |
|||
} |
|||
|
|||
type Person struct { |
|||
Name string |
|||
Age int |
|||
} |
|||
|
|||
func TestHttpClientPostJson(t *testing.T) { |
|||
// Content-Type is : application/json
|
|||
res, err := DefaultClient.POST("http://httpbin.org/post"). |
|||
SetBodyAsJson(H{ |
|||
"name": "wf", |
|||
"age": 18, |
|||
}).Do() |
|||
|
|||
// 支持对象类型
|
|||
//person := &Person{
|
|||
// Name: "wf",
|
|||
// Age: 18,
|
|||
//}
|
|||
//res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(person).Do()
|
|||
|
|||
// 支持字符串类型
|
|||
//jsonText := "{\"Name\":\"wf\",\"Age\":18}"
|
|||
//res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(jsonText).Do()
|
|||
|
|||
if err != nil { |
|||
t.Error("post failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
|
|||
data, err := ioutil.ReadAll(res.Body) |
|||
if nil != err { |
|||
t.Error("read failed", err) |
|||
} |
|||
|
|||
fmt.Println(string(data)) |
|||
} |
|||
|
|||
func TestHttpClientPostMultipart(t *testing.T) { |
|||
// Content-Type is : multipart/form-data
|
|||
res, err := DefaultClient.POST("http://httpbin.org/post"). |
|||
SetBodyAsForm(H{ |
|||
"name": "wf", |
|||
"age": 18, |
|||
"photo": FormFile("D:\\4B.jpg"), |
|||
"resume": FormFile("D:\\resume.rtf"), |
|||
}).Do() |
|||
|
|||
if err != nil { |
|||
t.Error("post failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
|
|||
data, err := ioutil.ReadAll(res.Body) |
|||
if nil != err { |
|||
t.Error("read failed", err) |
|||
} |
|||
|
|||
fmt.Println(string(data)) |
|||
} |
|||
|
|||
func TestHttpClientDelete(t *testing.T) { |
|||
str, err := DefaultClient.DELETE("http://httpbin.org/delete").DoForString() |
|||
if err != nil { |
|||
t.Error("delete failed", err) |
|||
} |
|||
|
|||
println(str) |
|||
} |
|||
|
|||
func TestHttpClientPutForm(t *testing.T) { |
|||
// Content-Type is : application/x-www-form-urlencoded
|
|||
res, err := DefaultClient.PUT("http://httpbin.org/put"). |
|||
SetBodyAsForm(H{ |
|||
"name": "wf", |
|||
"age": 18, |
|||
}).Do() |
|||
|
|||
if err != nil { |
|||
t.Error("put failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
|
|||
data, err := ioutil.ReadAll(res.Body) |
|||
if nil != err { |
|||
t.Error("read failed", err) |
|||
} |
|||
|
|||
fmt.Println(string(data)) |
|||
} |
|||
|
|||
func TestHttpClientPutJson(t *testing.T) { |
|||
// Content-Type is : application/json
|
|||
res, err := DefaultClient.PUT("http://httpbin.org/put"). |
|||
SetBodyAsJson(H{ |
|||
"name": "wf", |
|||
"age": 18, |
|||
}).Do() |
|||
|
|||
// 支持对象类型
|
|||
//person := &Person{
|
|||
// Name: "wf",
|
|||
// Age: 18,
|
|||
//}
|
|||
//res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(person).Do()
|
|||
|
|||
// 支持字符串类型
|
|||
//jsonText := "{\"Name\":\"wf\",\"Age\":18}"
|
|||
//res, err := DefaultClient.POST("http://httpbin.org/post").SetJson(jsonText).Do()
|
|||
|
|||
if err != nil { |
|||
t.Error("post failed", err) |
|||
} |
|||
|
|||
defer res.Body.Close() |
|||
|
|||
if res.StatusCode != 200 { |
|||
t.Error("Status Code not 200") |
|||
} |
|||
|
|||
data, err := ioutil.ReadAll(res.Body) |
|||
if nil != err { |
|||
t.Error("read failed", err) |
|||
} |
|||
|
|||
fmt.Println(string(data)) |
|||
} |
|||
|
|||
func TestResources(t *testing.T) { |
|||
entity, err := DefaultClient. |
|||
AddHeader("Authorization","adc8620e5164462e854f6f2e4e33ee53"). |
|||
GET("http://localhost:8090/portal/users/admin/resources"). |
|||
DoForEntity() |
|||
|
|||
if err != nil { |
|||
fmt.Println(err) |
|||
} |
|||
|
|||
fmt.Println("RespondText:",entity.DataAsString()) |
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
package httpclient |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"net/http" |
|||
) |
|||
|
|||
// 应答实体
|
|||
type RespondEntity struct { |
|||
statusCode int |
|||
data []byte |
|||
} |
|||
|
|||
func (t *RespondEntity) StatusCode() int { |
|||
return t.statusCode |
|||
} |
|||
|
|||
func (t *RespondEntity) StatusText() string { |
|||
return http.StatusText(t.statusCode) |
|||
} |
|||
|
|||
func (t *RespondEntity) Data() []byte { |
|||
return t.data |
|||
} |
|||
|
|||
func (t *RespondEntity) DataAsString() string { |
|||
return string(t.data) |
|||
} |
|||
|
|||
func (t *RespondEntity) Unmarshal(entity interface{}) error { |
|||
err := json.Unmarshal(t.data, entity) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
return nil |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package logagent |
|||
|
|||
import "go-mysql-transfer/util/logs" |
|||
|
|||
type ElsLoggerAgent struct { |
|||
} |
|||
|
|||
func NewElsLoggerAgent() *ElsLoggerAgent { |
|||
return &ElsLoggerAgent{} |
|||
} |
|||
|
|||
func (s *ElsLoggerAgent) Printf(format string, v ...interface{}) { |
|||
logs.Infof(format, v) |
|||
} |
|||
@ -0,0 +1,146 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package logagent |
|||
|
|||
import ( |
|||
"go-mysql-transfer/util/logs" |
|||
"go.uber.org/zap" |
|||
"go.uber.org/zap/zapcore" |
|||
) |
|||
|
|||
func EtcdZapLoggerConfig() zap.Config { |
|||
return zap.Config{ |
|||
Level: zap.NewAtomicLevelAt(zap.InfoLevel), |
|||
Development: false, |
|||
Sampling: &zap.SamplingConfig{ |
|||
Initial: 100, |
|||
Thereafter: 100, |
|||
}, |
|||
Encoding: "console", |
|||
// copied from "zap.NewProductionEncoderConfig" with some updates
|
|||
EncoderConfig: zapcore.EncoderConfig{ |
|||
TimeKey: "ts", |
|||
LevelKey: "level", |
|||
NameKey: "logger", |
|||
CallerKey: "caller", |
|||
MessageKey: "msg", |
|||
StacktraceKey: "stacktrace", |
|||
LineEnding: zapcore.DefaultLineEnding, |
|||
EncodeLevel: zapcore.LowercaseLevelEncoder, |
|||
EncodeTime: zapcore.ISO8601TimeEncoder, |
|||
EncodeDuration: zapcore.StringDurationEncoder, |
|||
EncodeCaller: zapcore.ShortCallerEncoder, |
|||
}, |
|||
|
|||
// Use "/dev/null" to discard all
|
|||
OutputPaths: []string{"stderr"}, |
|||
ErrorOutputPaths: []string{"stderr"}, |
|||
} |
|||
} |
|||
|
|||
type EtcdLoggerAgent struct { |
|||
} |
|||
|
|||
func NewEtcdLoggerAgent() *EtcdLoggerAgent { |
|||
return &EtcdLoggerAgent{} |
|||
} |
|||
|
|||
// Info logs to INFO log. Arguments are handled in the manner of fmt.Print.
|
|||
func (s *EtcdLoggerAgent) Info(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Infof("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println.
|
|||
func (s *EtcdLoggerAgent) Infoln(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Infof("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf.
|
|||
func (s *EtcdLoggerAgent) Infof(format string, args ...interface{}) { |
|||
logs.Infof(format, args...) |
|||
} |
|||
|
|||
// Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print.
|
|||
func (s *EtcdLoggerAgent) Warning(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Warnf("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println.
|
|||
func (s *EtcdLoggerAgent) Warningln(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Warnf("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf.
|
|||
func (s *EtcdLoggerAgent) Warningf(format string, args ...interface{}) { |
|||
logs.Warnf(format, args...) |
|||
} |
|||
|
|||
// Error logs to ERROR log. Arguments are handled in the manner of fmt.Print.
|
|||
func (s *EtcdLoggerAgent) Error(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Errorf("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println.
|
|||
func (s *EtcdLoggerAgent) Errorln(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Errorf("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf.
|
|||
func (s *EtcdLoggerAgent) Errorf(format string, args ...interface{}) { |
|||
logs.Errorf(format, args) |
|||
} |
|||
|
|||
// Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print.
|
|||
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
|
|||
// Implementations may also call os.Exit() with a non-zero exit code.
|
|||
func (s *EtcdLoggerAgent) Fatal(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Errorf("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println.
|
|||
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
|
|||
// Implementations may also call os.Exit() with a non-zero exit code.
|
|||
func (s *EtcdLoggerAgent) Fatalln(args ...interface{}) { |
|||
for _, arg := range args { |
|||
logs.Errorf("%v", arg) |
|||
} |
|||
} |
|||
|
|||
// Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf.
|
|||
// gRPC ensures that all Fatal logs will exit with os.Exit(1).
|
|||
// Implementations may also call os.Exit() with a non-zero exit code.
|
|||
func (s *EtcdLoggerAgent) Fatalf(format string, args ...interface{}) { |
|||
logs.Errorf(format, args) |
|||
} |
|||
|
|||
// V reports whether verbosity level l is at least the requested verbose level.
|
|||
func (s *EtcdLoggerAgent) V(l int) bool { return false } |
|||
@ -0,0 +1,32 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package logagent |
|||
|
|||
import "go-mysql-transfer/util/logs" |
|||
|
|||
type MetricsLoggerAgent struct { |
|||
} |
|||
|
|||
func NewMetricsLoggerAgent() *MetricsLoggerAgent { |
|||
return &MetricsLoggerAgent{} |
|||
} |
|||
|
|||
// Info logs to INFO log. Arguments are handled in the manner of fmt.Print.
|
|||
func (s *MetricsLoggerAgent) Printf(format string, v ...interface{}) { |
|||
logs.Infof(format, v) |
|||
} |
|||
@ -0,0 +1,84 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package logagent |
|||
|
|||
import ( |
|||
"fmt" |
|||
"go.uber.org/zap/zapcore" |
|||
|
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
type RocketmqLoggerAgent struct { |
|||
} |
|||
|
|||
func NewRocketmqLoggerAgent() *RocketmqLoggerAgent { |
|||
return &RocketmqLoggerAgent{} |
|||
} |
|||
|
|||
func (s *RocketmqLoggerAgent) Debug(msg string, fields map[string]interface{}) { |
|||
zapFields := make([]zapcore.Field, 0, len(fields)) |
|||
for k, v := range fields { |
|||
zapFields = append(zapFields, zapcore.Field{ |
|||
Key: k, |
|||
Type: zapcore.StringType, |
|||
String: fmt.Sprintf("%v", v), |
|||
}) |
|||
} |
|||
logs.Debug(msg, zapFields...) |
|||
} |
|||
|
|||
func (s *RocketmqLoggerAgent) Info(msg string, fields map[string]interface{}) { |
|||
zapFields := make([]zapcore.Field, 0, len(fields)) |
|||
for k, v := range fields { |
|||
zapFields = append(zapFields, zapcore.Field{ |
|||
Key: k, |
|||
Type: zapcore.StringType, |
|||
String: fmt.Sprintf("%v", v), |
|||
}) |
|||
} |
|||
logs.Info(msg, zapFields...) |
|||
} |
|||
|
|||
func (s *RocketmqLoggerAgent) Warning(msg string, fields map[string]interface{}) { |
|||
zapFields := make([]zapcore.Field, 0, len(fields)) |
|||
for k, v := range fields { |
|||
zapFields = append(zapFields, zapcore.Field{ |
|||
Key: k, |
|||
Type: zapcore.StringType, |
|||
String: fmt.Sprintf("%v", v), |
|||
}) |
|||
} |
|||
logs.Warn(msg, zapFields...) |
|||
} |
|||
|
|||
func (s *RocketmqLoggerAgent) Error(msg string, fields map[string]interface{}) { |
|||
zapFields := make([]zapcore.Field, 0, len(fields)) |
|||
for k, v := range fields { |
|||
zapFields = append(zapFields, zapcore.Field{ |
|||
Key: k, |
|||
Type: zapcore.StringType, |
|||
String: fmt.Sprintf("%v", v), |
|||
}) |
|||
} |
|||
logs.Error(msg, zapFields...) |
|||
} |
|||
|
|||
func (s *RocketmqLoggerAgent) Fatal(msg string, fields map[string]interface{}) { |
|||
s.Error(msg, fields) |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package logagent |
|||
|
|||
import "go-mysql-transfer/util/logs" |
|||
|
|||
type ZkLoggerAgent struct { |
|||
} |
|||
|
|||
func NewZkLoggerAgent() *ZkLoggerAgent { |
|||
return &ZkLoggerAgent{} |
|||
} |
|||
|
|||
func (s *ZkLoggerAgent) Printf(template string, args ...interface{}) { |
|||
logs.Infof(template, args...) |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
package nets |
|||
|
|||
import ( |
|||
"fmt" |
|||
"testing" |
|||
) |
|||
|
|||
func TestIsUsableAddr(t *testing.T) { |
|||
fmt.Println(IsUsableAddr(":8080")) |
|||
fmt.Println(IsUsableAddr(":8070")) |
|||
fmt.Println(IsUsableAddr(":8060")) |
|||
fmt.Println(IsUsableAddr(":8050")) |
|||
} |
|||
@ -0,0 +1,120 @@ |
|||
package nets |
|||
|
|||
import ( |
|||
"net" |
|||
"regexp" |
|||
"strconv" |
|||
"strings" |
|||
) |
|||
|
|||
// 检测是否为 IP 格式
|
|||
func CheckIp(addr string) bool { |
|||
if "" == addr { |
|||
return false |
|||
} |
|||
|
|||
a := net.ParseIP(addr) |
|||
if a == nil { |
|||
return false |
|||
} |
|||
|
|||
return true |
|||
} |
|||
|
|||
// 检测 地址是否为 IP:端口 格式
|
|||
func CheckHostAddr(addr string) bool { |
|||
|
|||
if "" == addr { |
|||
return false |
|||
} |
|||
|
|||
items := strings.Split(addr, ":") |
|||
if items == nil || len(items) != 2 { |
|||
return false |
|||
} |
|||
|
|||
a := net.ParseIP(items[0]) |
|||
if a == nil { |
|||
return false |
|||
} |
|||
|
|||
match, err := regexp.MatchString("^[0-9]*$", items[1]) |
|||
if err != nil { |
|||
return false |
|||
} |
|||
|
|||
i, err := strconv.Atoi(items[1]) |
|||
if err != nil { |
|||
return false |
|||
} |
|||
if i < 0 || i > 65535 { |
|||
return false |
|||
} |
|||
|
|||
if match == false { |
|||
return false |
|||
} |
|||
return true |
|||
} |
|||
|
|||
// 获取一个空闲的TCP端口
|
|||
func GetFreePort(bind string) int { |
|||
ip := ":" |
|||
if "" != bind { |
|||
ip = bind + ":" |
|||
} |
|||
|
|||
var port int |
|||
for i := 17070; i < 65536; i++ { |
|||
addr, _ := net.ResolveTCPAddr("tcp", ip+strconv.Itoa(i)) |
|||
listener, err := net.ListenTCP("tcp", addr) |
|||
if err == nil { |
|||
listener.Close() |
|||
port = i |
|||
break |
|||
} |
|||
} |
|||
return port |
|||
} |
|||
|
|||
func IsUsableTcpAddr(addr string) (bool, error) { |
|||
listener, err := net.Listen("tcp", addr) |
|||
if err != nil { |
|||
return false, err |
|||
} |
|||
listener.Close() |
|||
return true, nil |
|||
} |
|||
|
|||
func IsActiveTCPAddr(addr string) (bool, error) { |
|||
tcpAddr, _ := net.ResolveTCPAddr("tcp", addr) |
|||
conn, err := net.DialTCP("tcp", nil, tcpAddr) |
|||
if err != nil { |
|||
return false, err |
|||
} |
|||
defer conn.Close() |
|||
|
|||
return true, nil |
|||
} |
|||
|
|||
func GetIpList() ([]string, error) { |
|||
var ips []string |
|||
netInterfaces, err := net.Interfaces() |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
for i := 0; i < len(netInterfaces); i++ { |
|||
if (netInterfaces[i].Flags & net.FlagUp) != 0 { |
|||
ls, _ := netInterfaces[i].Addrs() |
|||
for _, address := range ls { |
|||
if ipNet, ok := address.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { |
|||
if ipNet.IP.To4() != nil { |
|||
ips = append(ips, ipNet.IP.String()) |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
return ips, nil |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
package snowflake |
|||
|
|||
import ( |
|||
"github.com/sony/sonyflake" |
|||
|
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
// 雪花ID工具
|
|||
|
|||
var _sf *sonyflake.Sonyflake |
|||
|
|||
func InitSnowflake(machineId uint16) { |
|||
if _sf != nil { |
|||
return |
|||
} |
|||
|
|||
var st sonyflake.Settings |
|||
st.MachineID = func() (u uint16, e error) { |
|||
return machineId, nil |
|||
} |
|||
_sf = sonyflake.NewSonyflake(st) |
|||
} |
|||
|
|||
func NextId() (uint64, error) { |
|||
id, err := _sf.NextID() |
|||
if err != nil { |
|||
logs.Errorf("snowflake NextId ：%s", err.Error()) |
|||
} |
|||
return id, nil |
|||
} |
|||
@ -0,0 +1,315 @@ |
|||
/* |
|||
* Copyright 2020-2021 the original author(https://github.com/wj596)
|
|||
* |
|||
* <p> |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
* </p> |
|||
*/ |
|||
package stringutil |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/hmac" |
|||
"crypto/md5" |
|||
"crypto/sha256" |
|||
"encoding/hex" |
|||
"encoding/json" |
|||
"fmt" |
|||
"net/url" |
|||
"regexp" |
|||
"strconv" |
|||
"strings" |
|||
"unicode" |
|||
|
|||
"github.com/satori/go.uuid" |
|||
) |
|||
|
|||
// 产生UUID
|
|||
func UUID() string { |
|||
return strings.ReplaceAll(uuid.NewV4().String(), "-", "") |
|||
} |
|||
|
|||
// 转换为Int
|
|||
func ToIntSafe(str string) int { |
|||
v, e := strconv.Atoi(str) |
|||
if nil != e { |
|||
return 0 |
|||
} |
|||
return v |
|||
} |
|||
|
|||
// 转换为Uint64
|
|||
func ToInt64Safe(str string) int64 { |
|||
v, e := strconv.ParseInt(str, 10, 64) |
|||
if nil != e { |
|||
return 0 |
|||
} |
|||
return v |
|||
} |
|||
|
|||
// 转换为Uint64
|
|||
func ToUint64Safe(str string) uint64 { |
|||
v, e := strconv.ParseUint(str, 10, 64) |
|||
if nil != e { |
|||
return 0 |
|||
} |
|||
return v |
|||
} |
|||
|
|||
// Uint64转换为String
|
|||
func Uint64ToStr(u uint64) string { |
|||
return strconv.FormatUint(u, 10) |
|||
} |
|||
|
|||
// 逗号分隔键值对转MAP,类似"name=wangjie,age=20"或者"name=wangjie|age=20"
|
|||
func CommasToMap(base string, sep string) map[string]interface{} { |
|||
ret := make(map[string]interface{}) |
|||
if "" != base && "" != sep { |
|||
kvs := strings.Split(base, sep) |
|||
for _, kv := range kvs { |
|||
temp := strings.Split(kv, "=") |
|||
if len(temp) < 2 { |
|||
continue |
|||
} |
|||
if temp[0] == "" { |
|||
continue |
|||
} |
|||
ret[temp[0]] = temp[1] |
|||
} |
|||
} |
|||
return ret |
|||
} |
|||
|
|||
func ToJsonBytes(v interface{}) []byte { |
|||
bytes, err := json.Marshal(v) |
|||
if nil != err { |
|||
return nil |
|||
} |
|||
return bytes |
|||
} |
|||
|
|||
func ToJsonString(v interface{}) string { |
|||
bytes, err := json.Marshal(v) |
|||
if nil != err { |
|||
return "" |
|||
} |
|||
return string(bytes) |
|||
} |
|||
|
|||
func ToJsonIndent(v interface{}) string { |
|||
bytes, err := json.MarshalIndent(v, "", "\t") |
|||
if nil != err { |
|||
return "" |
|||
} |
|||
return string(bytes) |
|||
} |
|||
|
|||
// url.Values转查询字符串
|
|||
func UrlValuesToQueryString(base string, parameters url.Values) string { |
|||
if len(parameters) == 0 { |
|||
return base |
|||
} |
|||
|
|||
if !strings.Contains(base, "?") { |
|||
base += "?" |
|||
} |
|||
|
|||
if strings.HasSuffix(base, "?") || strings.HasSuffix(base, "&") { |
|||
base += parameters.Encode() |
|||
} else { |
|||
base += "&" + parameters.Encode() |
|||
} |
|||
|
|||
return base |
|||
} |
|||
|
|||
// map转查询字符串
|
|||
func MapToQueryString(base string, parameters map[string]interface{}) string { |
|||
if len(parameters) == 0 { |
|||
return base |
|||
} |
|||
|
|||
exist := false |
|||
if strings.Contains(base, "?") { |
|||
exist = true |
|||
} |
|||
var buffer bytes.Buffer |
|||
buffer.WriteString(base) |
|||
for k, v := range parameters { |
|||
var temp string |
|||
if !exist { |
|||
temp = "?" + k + "=" + ToString(v) |
|||
exist = true |
|||
} else { |
|||
temp = "&" + k + "=" + ToString(v) |
|||
} |
|||
buffer.WriteString(temp) |
|||
} |
|||
return buffer.String() |
|||
} |
|||
|
|||
// 转换为字符串
|
|||
func ToString(value interface{}) string { |
|||
var key string |
|||
if value == nil { |
|||
return key |
|||
} |
|||
|
|||
switch value.(type) { |
|||
case float64: |
|||
ft := value.(float64) |
|||
key = strconv.FormatFloat(ft, 'f', -1, 64) |
|||
case float32: |
|||
ft := value.(float32) |
|||
key = strconv.FormatFloat(float64(ft), 'f', -1, 64) |
|||
case int: |
|||
it := value.(int) |
|||
key = strconv.Itoa(it) |
|||
case uint: |
|||
it := value.(uint) |
|||
key = strconv.Itoa(int(it)) |
|||
case int8: |
|||
it := value.(int8) |
|||
key = strconv.Itoa(int(it)) |
|||
case uint8: |
|||
it := value.(uint8) |
|||
key = strconv.Itoa(int(it)) |
|||
case int16: |
|||
it := value.(int16) |
|||
key = strconv.Itoa(int(it)) |
|||
case uint16: |
|||
it := value.(uint16) |
|||
key = strconv.Itoa(int(it)) |
|||
case int32: |
|||
it := value.(int32) |
|||
key = strconv.Itoa(int(it)) |
|||
case uint32: |
|||
it := value.(uint32) |
|||
key = strconv.Itoa(int(it)) |
|||
case int64: |
|||
it := value.(int64) |
|||
key = strconv.FormatInt(it, 10) |
|||
case uint64: |
|||
it := value.(uint64) |
|||
key = strconv.FormatUint(it, 10) |
|||
case string: |
|||
key = value.(string) |
|||
case []byte: |
|||
key = string(value.([]byte)) |
|||
default: |
|||
newValue, _ := json.Marshal(value) |
|||
key = string(newValue) |
|||
} |
|||
|
|||
return key |
|||
} |
|||
|
|||
// 是否为邮件格式
|
|||
func IsEmailFormat(email string) bool { |
|||
pattern := `\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*` |
|||
reg := regexp.MustCompile(pattern) |
|||
return reg.MatchString(email) |
|||
} |
|||
|
|||
// 是否为中文
|
|||
func IsChineseChar(str string) bool { |
|||
for _, r := range str { |
|||
if unicode.Is(unicode.Scripts["Han"], r) || (regexp.MustCompile("[\u3002\uff1b\uff0c\uff1a\u201c\u201d\uff08\uff09\u3001\uff1f\u300a\u300b]").MatchString(string(r))) { |
|||
return true |
|||
} |
|||
} |
|||
return false |
|||
} |
|||
|
|||
// MD5编码
|
|||
func MD5(str string) string { |
|||
h := md5.New() |
|||
h.Write([]byte(str)) |
|||
return hex.EncodeToString(h.Sum(nil)) |
|||
} |
|||
|
|||
func HmacSHA256(plaintext string, key string) string { |
|||
hash := hmac.New(sha256.New, []byte(key)) // 创建哈希算法
|
|||
hash.Write([]byte(plaintext)) // 写入数据
|
|||
return fmt.Sprintf("%X", hash.Sum(nil)) |
|||
} |
|||
|
|||
func HmacMD5(plaintext string, key string) string { |
|||
hash := hmac.New(md5.New, []byte(key)) // 创建哈希算法
|
|||
hash.Write([]byte(plaintext)) // 写入数据
|
|||
return fmt.Sprintf("%X", hash.Sum(nil)) |
|||
} |
|||
|
|||
// 驼峰式写法转为下划线写法
|
|||
func Camel2Case(name string) string { |
|||
buffer := new(bytes.Buffer) |
|||
for i, r := range name { |
|||
if unicode.IsUpper(r) { |
|||
if i != 0 { |
|||
buffer.WriteByte('_') |
|||
} |
|||
buffer.WriteRune(unicode.ToLower(r)) |
|||
} else { |
|||
buffer.WriteString(strconv.FormatInt(int64(r), 10)) |
|||
} |
|||
} |
|||
return buffer.String() |
|||
} |
|||
|
|||
// 下划线写法转为驼峰写法
|
|||
func Case2Camel(name string) string { |
|||
name = strings.Replace(name, "_", " ", -1) |
|||
name = strings.Title(name) |
|||
name = strings.Replace(name, " ", "", -1) |
|||
return Lcfirst(name) |
|||
} |
|||
|
|||
// 首字母大写
|
|||
func Ucfirst(str string) string { |
|||
for i, v := range str { |
|||
return string(unicode.ToUpper(v)) + str[i+1:] |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
// 首字母小写
|
|||
func Lcfirst(str string) string { |
|||
for i, v := range str { |
|||
return string(unicode.ToLower(v)) + str[i+1:] |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func ToFloat64Safe(str string) float64 { |
|||
v, e := strconv.ParseFloat(str, 64) |
|||
if nil != e { |
|||
return 0 |
|||
} |
|||
return v |
|||
} |
|||
|
|||
func ToUint32(str string) (uint32, error) { |
|||
v, e := strconv.ParseUint(str, 10, 32) |
|||
if nil != e { |
|||
return 0, e |
|||
} |
|||
return uint32(v), nil |
|||
} |
|||
|
|||
func ToUint32Safe(str string) uint32 { |
|||
v, e := strconv.ParseUint(str, 10, 32) |
|||
if nil != e { |
|||
return 0 |
|||
} |
|||
return uint32(v) |
|||
} |
|||
@ -0,0 +1,12 @@ |
|||
package stringutil |
|||
|
|||
import ( |
|||
"testing" |
|||
) |
|||
|
|||
func TestIsChineseChar(t *testing.T) { |
|||
println(IsChineseChar("a")) |
|||
println(IsChineseChar(",")) |
|||
println(IsChineseChar("a我b")) |
|||
println(IsChineseChar("，")) |
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
package sys |
|||
|
|||
import ( |
|||
"fmt" |
|||
"os" |
|||
"os/signal" |
|||
"path/filepath" |
|||
"strings" |
|||
"syscall" |
|||
) |
|||
|
|||
type Closer interface { |
|||
Close() |
|||
} |
|||
|
|||
// 获取程序运行路径
|
|||
func CurrentDirectory() string { |
|||
dir, err := filepath.Abs(filepath.Dir(os.Args[0])) |
|||
if err != nil { |
|||
fmt.Errorf(err.Error()) |
|||
} |
|||
return strings.Replace(dir, "\\", "/", -1) |
|||
} |
|||
|
|||
func WaitCloseSignals() { |
|||
signals := make(chan os.Signal, 1) |
|||
signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT) |
|||
<-signals |
|||
} |
|||
|
|||
func WaitCloseSignalsAndRelease(closer Closer) { |
|||
signals := make(chan os.Signal, 1) |
|||
signal.Notify(signals, os.Interrupt, os.Kill, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT) |
|||
<-signals |
|||
closer.Close() |
|||
} |
|||
@ -0,0 +1,63 @@ |
|||
package zookeepers |
|||
|
|||
import ( |
|||
"strings" |
|||
|
|||
"github.com/samuel/go-zookeeper/zk" |
|||
|
|||
"go-mysql-transfer/util/stringutil" |
|||
) |
|||
|
|||
func CreateDirIfNecessary(dir string, conn *zk.Conn) error { |
|||
exist, _, err := conn.Exists(dir) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if !exist { |
|||
_, err := conn.Create(dir, nil, 0, zk.WorldACL(zk.PermAll)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func CreateDirWithDataIfNecessary(dir string,data []byte, conn *zk.Conn) error { |
|||
exist, _, err := conn.Exists(dir) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
if !exist { |
|||
_, err := conn.Create(dir, data, 0, zk.WorldACL(zk.PermAll)) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func DeleteDir(dir string, conn *zk.Conn) error { |
|||
_, stat, err := conn.Get(dir) |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
return conn.Delete(dir, stat.Version) |
|||
} |
|||
|
|||
func JoinDir(args ...interface{}) string { |
|||
path := "" |
|||
for _, arg := range args { |
|||
if arg != "" { |
|||
p := stringutil.ToString(arg) |
|||
if strings.HasPrefix(p, "/") { |
|||
path = path + p |
|||
} else { |
|||
path = path + "/" + p |
|||
} |
|||
} |
|||
} |
|||
return path |
|||
} |
|||
@ -0,0 +1,137 @@ |
|||
package web |
|||
|
|||
import ( |
|||
"fmt" |
|||
"go-mysql-transfer/service" |
|||
"go-mysql-transfer/util/dates" |
|||
"go-mysql-transfer/util/nets" |
|||
"log" |
|||
"net/http" |
|||
"path" |
|||
"strconv" |
|||
"time" |
|||
|
|||
"github.com/gin-gonic/gin" |
|||
|
|||
"go-mysql-transfer/global" |
|||
"go-mysql-transfer/metrics" |
|||
"go-mysql-transfer/util/logs" |
|||
) |
|||
|
|||
var _server *http.Server |
|||
|
|||
func Start() error { |
|||
if !global.Cfg().EnableWebAdmin { //哨兵
|
|||
return nil |
|||
} |
|||
|
|||
gin.SetMode(gin.ReleaseMode) |
|||
g := gin.New() |
|||
//statics := "D:\\statics"
|
|||
//index := "D:\\statics\\index.html"
|
|||
|
|||
statics := "statics" |
|||
index := path.Join(statics, "index.html") |
|||
g.Static("/statics", statics) |
|||
g.LoadHTMLFiles(index) |
|||
g.GET("/", webAdminFunc) |
|||
|
|||
port := global.Cfg().WebAdminPort |
|||
listen := fmt.Sprintf(":%s", strconv.Itoa(port)) |
|||
_server = &http.Server{ |
|||
Addr: listen, |
|||
Handler: g, |
|||
ReadTimeout: 10 * time.Second, |
|||
WriteTimeout: 10 * time.Second, |
|||
MaxHeaderBytes: 1 << 20, |
|||
} |
|||
|
|||
ok, err := nets.IsUsableTcpAddr(listen) |
|||
if !ok { |
|||
return err |
|||
} |
|||
|
|||
log.Println(fmt.Sprintf("Web Admin Listen At %s", listen)) |
|||
go func() { |
|||
if err := _server.ListenAndServe(); err != nil { |
|||
logs.Error(err.Error()) |
|||
} |
|||
}() |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func webAdminFunc(c *gin.Context) { |
|||
pos, _ := service.TransferServiceIns().Position() |
|||
|
|||
var tables []string |
|||
for _, v := range global.RuleKeyList() { |
|||
tables = append(tables, v) |
|||
} |
|||
|
|||
var insertAmounts []uint64 |
|||
for _, v := range tables { |
|||
insertAmounts = append(insertAmounts, metrics.LabInsertAmount(v)) |
|||
} |
|||
|
|||
var updateAmounts []uint64 |
|||
for _, v := range tables { |
|||
updateAmounts = append(updateAmounts, metrics.LabUpdateRecord(v)) |
|||
} |
|||
|
|||
var deleteAmounts []uint64 |
|||
for _, v := range tables { |
|||
deleteAmounts = append(deleteAmounts, metrics.LabDeleteRecord(v)) |
|||
} |
|||
|
|||
h := gin.H{ |
|||
"mysql": global.Cfg().Addr, |
|||
"binName": pos.Name, |
|||
"binPos": pos.Pos, |
|||
"destName": global.Cfg().DestStdName(), |
|||
"destAddr": global.Cfg().DestAddr(), |
|||
"destState": metrics.DestState(), |
|||
"bootTime": dates.Layout(global.BootTime(), dates.DayTimeMinuteFormatter), |
|||
"insertAmount": metrics.InsertAmount(), |
|||
"updateAmount": metrics.UpdateAmount(), |
|||
"deleteAmount": metrics.DeleteAmount(), |
|||
"tables": tables, |
|||
"insertAmounts": insertAmounts, |
|||
"updateAmounts": updateAmounts, |
|||
"deleteAmounts": deleteAmounts, |
|||
"isCluster": global.Cfg().IsCluster(), |
|||
"isRedirect": false, |
|||
} |
|||
|
|||
if global.Cfg().IsCluster() { |
|||
h["isZk"] = global.Cfg().IsZk() |
|||
h["zkAddrs"] = global.Cfg().Cluster.ZkAddrs |
|||
h["etcdAddrs"] = global.Cfg().Cluster.EtcdAddrs |
|||
h["isLeader"] = global.IsLeader() |
|||
h["leader"] = global.LeaderNode() |
|||
if !global.IsLeader() { |
|||
h["isRedirect"] = true |
|||
} |
|||
|
|||
var followers []string |
|||
for _, v := range service.ClusterServiceIns().Nodes() { |
|||
if v != global.LeaderNode() { |
|||
followers = append(followers, v) |
|||
} |
|||
} |
|||
h["followers"] = followers |
|||
} |
|||
|
|||
c.HTML(200, "index.html", h) |
|||
} |
|||
|
|||
func Close() { |
|||
if _server == nil { |
|||
return |
|||
} |
|||
|
|||
err := _server.Shutdown(nil) |
|||
if err != nil { |
|||
log.Println(err.Error()) |
|||
} |
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
MIT License |
|||
|
|||
Copyright (c) 2019 layuimini |
|||
|
|||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|||
of this software and associated documentation files (the "Software"), to deal |
|||
in the Software without restriction, including without limitation the rights |
|||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|||
copies of the Software, and to permit persons to whom the Software is |
|||
furnished to do so, subject to the following conditions: |
|||
|
|||
The above copyright notice and this permission notice shall be included in all |
|||
copies or substantial portions of the Software. |
|||
|
|||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|||
SOFTWARE. |
|||
@ -0,0 +1,76 @@ |
|||
layuimini后台模板 |
|||
=============== |
|||
# 项目介绍 |
|||
最简洁、清爽、易用的layui后台框架模板。 |
|||
|
|||
项目会不定时进行更新，建议star和watch一份。 |
|||
|
|||
技术交流QQ群：[1165301500](https://jq.qq.com/?_wv=1027&k=eUm5xKG1)、[667813249🈵](https://jq.qq.com/?_wv=1027&k=5lyiE2Q)、[561838086🈵](https://jq.qq.com/?_wv=1027&k=5JRGVfe) |
|||
|
|||
# 主要特性 |
|||
* 界面足够简洁清爽，响应式且适配手机端。 |
|||
* 一个接口`几行代码而已`直接初始化整个框架，无需复杂操作。 |
|||
* 页面支持多配色方案，可自行选择喜欢的配色。 |
|||
* 支持多tab，可以打开多窗口。 |
|||
* 支持无限级菜单和对font-awesome图标库的完美支持。 |
|||
* 失效以及报错菜单无法直接打开，并给出弹出层提示`完美的线上用户体验`。 |
|||
* url地址hash定位，可以清楚看到当前tab的地址信息。 |
|||
* 刷新页面会保留当前的窗口，并且会定位当前窗口对应左侧菜单栏。 |
|||
* 支持font-awesome图标选择插件 |
|||
|
|||
|
|||
# 代码仓库(iframe 多tab版) |
|||
|
|||
### v2版 |
|||
* 在线预览地址：[http://layuimini.99php.cn/iframe/v2/index.html](http://layuimini.99php.cn/iframe/v2/index.html) |
|||
* GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/v2](https://github.com/zhongshaofa/layuimini/tree/v2) |
|||
* Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/v2](https://gitee.com/zhongshaofa/layuimini/tree/v2) |
|||
### v1版 |
|||
* 在线预览地址：[http://layuimini.99php.cn/iframe/v1/index.html](http://layuimini.99php.cn/iframe/v1/index.html) |
|||
* GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/master](https://github.com/zhongshaofa/layuimini/tree/master) |
|||
* Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/master](https://gitee.com/zhongshaofa/layuimini/tree/master) |
|||
|
|||
# 代码仓库(onepage 单页版) |
|||
|
|||
### v2版 |
|||
* 在线预览地址：[http://layuimini.99php.cn/onepage/v2/index.html](http://layuimini.99php.cn/onepage/v2/index.html) |
|||
* GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/v2-onepage](https://github.com/zhongshaofa/layuimini/tree/v2-onepage) |
|||
* Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/v2-onepage](https://gitee.com/zhongshaofa/layuimini/tree/v2-onepage) |
|||
|
|||
### v1版 |
|||
* 在线预览地址：[http://layuimini.99php.cn/onepage/v1/index.html](http://layuimini.99php.cn/onepage/v1/index.html) |
|||
* GitHub仓库地址：[https://github.com/zhongshaofa/layuimini/tree/onepage](https://github.com/zhongshaofa/layuimini/tree/onepage) |
|||
* Gitee仓库地址：[https://gitee.com/zhongshaofa/layuimini/tree/onepage](https://gitee.com/zhongshaofa/layuimini/tree/onepage) |
|||
|
|||
# 下载方式 |
|||
|
|||
### iframe v2版 |
|||
* GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b v2` |
|||
* Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b v2` |
|||
### iframe v1版 |
|||
* GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b master` |
|||
* Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b master` |
|||
### 单页版 v2版 |
|||
* GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b v2-onepage` |
|||
* Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b v2-onepage` |
|||
### 单页版 v1版 |
|||
* GitHub下载命令：`git clone https://github.com/zhongshaofa/layuimini -b onepage` |
|||
* Gitee下载命令：`git clone https://gitee.com/zhongshaofa/layuimini -b onepage` |
|||
### 发行版地址 |
|||
* GitHub发版地址：[https://github.com/zhongshaofa/layuimini/releases](https://github.com/zhongshaofa/layuimini/releases) |
|||
* Gitee发版地址：[https://gitee.com/zhongshaofa/layuimini/releases](https://gitee.com/zhongshaofa/layuimini/releases) |
|||
|
|||
# 效果预览 |
|||
> 总体预览 |
|||
|
|||
 |
|||
|
|||
# 使用说明 |
|||
|
|||
文档地址：[查看文档](http://layuimini.99php.cn/docs/) |
|||
|
|||
# 捐赠支持 |
|||
|
|||
开源项目不易，若此项目能得到你的青睐，可以捐赠支持作者持续开发与维护，感谢所有支持开源的朋友。 |
|||
|
|||
 |
|||
@ -0,0 +1,4 @@ |
|||
{ |
|||
"code": 1, |
|||
"msg": "服务端清理缓存成功" |
|||
} |
|||
@ -0,0 +1,245 @@ |
|||
{ |
|||
"homeInfo": { |
|||
"title": "首页", |
|||
"href": "page/welcome-1.html?t=1" |
|||
}, |
|||
"logoInfo": { |
|||
"title": "LAYUI MINI", |
|||
"image": "images/logo.png", |
|||
"href": "" |
|||
}, |
|||
"menuInfo": [ |
|||
{ |
|||
"title": "常规管理", |
|||
"icon": "fa fa-address-book", |
|||
"href": "", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "主页模板", |
|||
"href": "", |
|||
"icon": "fa fa-home", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "主页一", |
|||
"href": "page/welcome-1.html", |
|||
"icon": "fa fa-tachometer", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "主页二", |
|||
"href": "page/welcome-2.html", |
|||
"icon": "fa fa-tachometer", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "主页三", |
|||
"href": "page/welcome-3.html", |
|||
"icon": "fa fa-tachometer", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "菜单管理", |
|||
"href": "page/menu.html", |
|||
"icon": "fa fa-window-maximize", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "系统设置", |
|||
"href": "page/setting.html", |
|||
"icon": "fa fa-gears", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "表格示例", |
|||
"href": "page/table.html", |
|||
"icon": "fa fa-file-text", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "表单示例", |
|||
"href": "", |
|||
"icon": "fa fa-calendar", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "普通表单", |
|||
"href": "page/form.html", |
|||
"icon": "fa fa-list-alt", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "分步表单", |
|||
"href": "page/form-step.html", |
|||
"icon": "fa fa-navicon", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "登录模板", |
|||
"href": "", |
|||
"icon": "fa fa-flag-o", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "登录-1", |
|||
"href": "page/login-1.html", |
|||
"icon": "fa fa-stumbleupon-circle", |
|||
"target": "_blank" |
|||
}, |
|||
{ |
|||
"title": "登录-2", |
|||
"href": "page/login-2.html", |
|||
"icon": "fa fa-viacoin", |
|||
"target": "_blank" |
|||
}, |
|||
{ |
|||
"title": "登录-3", |
|||
"href": "page/login-3.html", |
|||
"icon": "fa fa-tags", |
|||
"target": "_blank" |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "异常页面", |
|||
"href": "", |
|||
"icon": "fa fa-home", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "404页面", |
|||
"href": "page/404.html", |
|||
"icon": "fa fa-hourglass-end", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "其它界面", |
|||
"href": "", |
|||
"icon": "fa fa-snowflake-o", |
|||
"target": "", |
|||
"child": [ |
|||
{ |
|||
"title": "按钮示例", |
|||
"href": "page/button.html", |
|||
"icon": "fa fa-snowflake-o", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "弹出层", |
|||
"href": "page/layer.html", |
|||
"icon": "fa fa-shield", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "组件管理", |
|||
"icon": "fa fa-lemon-o", |
|||
"href": "", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "图标列表", |
|||
"href": "page/icon.html", |
|||
"icon": "fa fa-dot-circle-o", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "图标选择", |
|||
"href": "page/icon-picker.html", |
|||
"icon": "fa fa-adn", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "颜色选择", |
|||
"href": "page/color-select.html", |
|||
"icon": "fa fa-dashboard", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "下拉选择", |
|||
"href": "page/table-select.html", |
|||
"icon": "fa fa-angle-double-down", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "文件上传", |
|||
"href": "page/upload.html", |
|||
"icon": "fa fa-arrow-up", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "富文本编辑器", |
|||
"href": "page/editor.html", |
|||
"icon": "fa fa-edit", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "省市县区选择器", |
|||
"href": "page/area.html", |
|||
"icon": "fa fa-rocket", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "其它管理", |
|||
"icon": "fa fa-slideshare", |
|||
"href": "", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "多级菜单", |
|||
"href": "", |
|||
"icon": "fa fa-meetup", |
|||
"target": "", |
|||
"child": [ |
|||
{ |
|||
"title": "按钮1", |
|||
"href": "page/button.html?v=1", |
|||
"icon": "fa fa-calendar", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "按钮2", |
|||
"href": "page/button.html?v=2", |
|||
"icon": "fa fa-snowflake-o", |
|||
"target": "_self", |
|||
"child": [ |
|||
{ |
|||
"title": "按钮3", |
|||
"href": "page/button.html?v=3", |
|||
"icon": "fa fa-snowflake-o", |
|||
"target": "_self" |
|||
}, |
|||
{ |
|||
"title": "表单4", |
|||
"href": "page/form.html?v=1", |
|||
"icon": "fa fa-calendar", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
} |
|||
] |
|||
} |
|||
] |
|||
}, |
|||
{ |
|||
"title": "失效菜单", |
|||
"href": "page/error.html", |
|||
"icon": "fa fa-superpowers", |
|||
"target": "_self" |
|||
} |
|||
] |
|||
} |
|||
] |
|||
} |
|||
@ -0,0 +1,254 @@ |
|||
{ |
|||
"code": 0, |
|||
"msg": "", |
|||
"count": 19, |
|||
"data": [ |
|||
{ |
|||
"authorityId": 1, |
|||
"authorityName": "系统管理", |
|||
"orderNumber": 1, |
|||
"menuUrl": null, |
|||
"menuIcon": "layui-icon-set", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": null, |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 0, |
|||
"parentId": -1 |
|||
}, |
|||
{ |
|||
"authorityId": 2, |
|||
"authorityName": "用户管理", |
|||
"orderNumber": 2, |
|||
"menuUrl": "system/user", |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": null, |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 0, |
|||
"parentId": 1 |
|||
}, |
|||
{ |
|||
"authorityId": 3, |
|||
"authorityName": "查询用户", |
|||
"orderNumber": 3, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/07/21 13:54:16", |
|||
"authority": "user:view", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/21 13:54:16", |
|||
"isMenu": 1, |
|||
"parentId": 2 |
|||
}, |
|||
{ |
|||
"authorityId": 4, |
|||
"authorityName": "添加用户", |
|||
"orderNumber": 4, |
|||
"menuUrl": null, |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "user:add", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 2 |
|||
}, |
|||
{ |
|||
"authorityId": 5, |
|||
"authorityName": "修改用户", |
|||
"orderNumber": 5, |
|||
"menuUrl": null, |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "user:edit", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 2 |
|||
}, |
|||
{ |
|||
"authorityId": 6, |
|||
"authorityName": "删除用户", |
|||
"orderNumber": 6, |
|||
"menuUrl": null, |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "user:delete", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 2 |
|||
}, |
|||
{ |
|||
"authorityId": 7, |
|||
"authorityName": "角色管理", |
|||
"orderNumber": 7, |
|||
"menuUrl": "system/role", |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": null, |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 0, |
|||
"parentId": 1 |
|||
}, |
|||
{ |
|||
"authorityId": 8, |
|||
"authorityName": "查询角色", |
|||
"orderNumber": 8, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/07/21 13:54:59", |
|||
"authority": "role:view", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/21 13:54:58", |
|||
"isMenu": 1, |
|||
"parentId": 7 |
|||
}, |
|||
{ |
|||
"authorityId": 9, |
|||
"authorityName": "添加角色", |
|||
"orderNumber": 9, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "role:add", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 7 |
|||
}, |
|||
{ |
|||
"authorityId": 10, |
|||
"authorityName": "修改角色", |
|||
"orderNumber": 10, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "role:edit", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 7 |
|||
}, |
|||
{ |
|||
"authorityId": 11, |
|||
"authorityName": "删除角色", |
|||
"orderNumber": 11, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "role:delete", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 7 |
|||
}, |
|||
{ |
|||
"authorityId": 12, |
|||
"authorityName": "角色权限管理", |
|||
"orderNumber": 12, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "role:auth", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 15:27:18", |
|||
"isMenu": 1, |
|||
"parentId": 7 |
|||
}, |
|||
{ |
|||
"authorityId": 13, |
|||
"authorityName": "权限管理", |
|||
"orderNumber": 13, |
|||
"menuUrl": "system/authorities", |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": null, |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 15:45:13", |
|||
"isMenu": 0, |
|||
"parentId": 1 |
|||
}, |
|||
{ |
|||
"authorityId": 14, |
|||
"authorityName": "查询权限", |
|||
"orderNumber": 14, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/07/21 13:55:57", |
|||
"authority": "authorities:view", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/21 13:55:56", |
|||
"isMenu": 1, |
|||
"parentId": 13 |
|||
}, |
|||
{ |
|||
"authorityId": 15, |
|||
"authorityName": "添加权限", |
|||
"orderNumber": 15, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "authorities:add", |
|||
"checked": 0, |
|||
"updateTime": "2018/06/29 11:05:41", |
|||
"isMenu": 1, |
|||
"parentId": 13 |
|||
}, |
|||
{ |
|||
"authorityId": 16, |
|||
"authorityName": "修改权限", |
|||
"orderNumber": 16, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/07/13 09:13:42", |
|||
"authority": "authorities:edit", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/13 09:13:42", |
|||
"isMenu": 1, |
|||
"parentId": 13 |
|||
}, |
|||
{ |
|||
"authorityId": 17, |
|||
"authorityName": "删除权限", |
|||
"orderNumber": 17, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": "authorities:delete", |
|||
"checked": 0, |
|||
"updateTime": "2018/06/29 11:05:41", |
|||
"isMenu": 1, |
|||
"parentId": 13 |
|||
}, |
|||
{ |
|||
"authorityId": 18, |
|||
"authorityName": "登录日志", |
|||
"orderNumber": 18, |
|||
"menuUrl": "system/loginRecord", |
|||
"menuIcon": null, |
|||
"createTime": "2018/06/29 11:05:41", |
|||
"authority": null, |
|||
"checked": 0, |
|||
"updateTime": "2018/06/29 11:05:41", |
|||
"isMenu": 0, |
|||
"parentId": 1 |
|||
}, |
|||
{ |
|||
"authorityId": 19, |
|||
"authorityName": "查询登录日志", |
|||
"orderNumber": 19, |
|||
"menuUrl": "", |
|||
"menuIcon": "", |
|||
"createTime": "2018/07/21 13:56:43", |
|||
"authority": "loginRecord:view", |
|||
"checked": 0, |
|||
"updateTime": "2018/07/21 13:56:43", |
|||
"isMenu": 1, |
|||
"parentId": 18 |
|||
} |
|||
] |
|||
} |
|||
@ -0,0 +1,127 @@ |
|||
{ |
|||
"code": 0, |
|||
"msg": "", |
|||
"count": 1000, |
|||
"data": [ |
|||
{ |
|||
"id": 10000, |
|||
"username": "user-0", |
|||
"sex": "女", |
|||
"city": "城市-0", |
|||
"sign": "签名-0", |
|||
"experience": 255, |
|||
"logins": 24, |
|||
"wealth": 82830700, |
|||
"classify": "作家", |
|||
"score": 57 |
|||
}, |
|||
{ |
|||
"id": 10001, |
|||
"username": "user-1", |
|||
"sex": "男", |
|||
"city": "城市-1", |
|||
"sign": "签名-1", |
|||
"experience": 884, |
|||
"logins": 58, |
|||
"wealth": 64928690, |
|||
"classify": "词人", |
|||
"score": 27 |
|||
}, |
|||
{ |
|||
"id": 10002, |
|||
"username": "user-2", |
|||
"sex": "女", |
|||
"city": "城市-2", |
|||
"sign": "签名-2", |
|||
"experience": 650, |
|||
"logins": 77, |
|||
"wealth": 6298078, |
|||
"classify": "酱油", |
|||
"score": 31 |
|||
}, |
|||
{ |
|||
"id": 10003, |
|||
"username": "user-3", |
|||
"sex": "女", |
|||
"city": "城市-3", |
|||
"sign": "签名-3", |
|||
"experience": 362, |
|||
"logins": 157, |
|||
"wealth": 37117017, |
|||
"classify": "诗人", |
|||
"score": 68 |
|||
}, |
|||
{ |
|||
"id": 10004, |
|||
"username": "user-4", |
|||
"sex": "男", |
|||
"city": "城市-4", |
|||
"sign": "签名-4", |
|||
"experience": 807, |
|||
"logins": 51, |
|||
"wealth": 76263262, |
|||
"classify": "作家", |
|||
"score": 6 |
|||
}, |
|||
{ |
|||
"id": 10005, |
|||
"username": "user-5", |
|||
"sex": "女", |
|||
"city": "城市-5", |
|||
"sign": "签名-5", |
|||
"experience": 173, |
|||
"logins": 68, |
|||
"wealth": 60344147, |
|||
"classify": "作家", |
|||
"score": 87 |
|||
}, |
|||
{ |
|||
"id": 10006, |
|||
"username": "user-6", |
|||
"sex": "女", |
|||
"city": "城市-6", |
|||
"sign": "签名-6", |
|||
"experience": 982, |
|||
"logins": 37, |
|||
"wealth": 57768166, |
|||
"classify": "作家", |
|||
"score": 34 |
|||
}, |
|||
{ |
|||
"id": 10007, |
|||
"username": "user-7", |
|||
"sex": "男", |
|||
"city": "城市-7", |
|||
"sign": "签名-7", |
|||
"experience": 727, |
|||
"logins": 150, |
|||
"wealth": 82030578, |
|||
"classify": "作家", |
|||
"score": 28 |
|||
}, |
|||
{ |
|||
"id": 10008, |
|||
"username": "user-8", |
|||
"sex": "男", |
|||
"city": "城市-8", |
|||
"sign": "签名-8", |
|||
"experience": 951, |
|||
"logins": 133, |
|||
"wealth": 16503371, |
|||
"classify": "词人", |
|||
"score": 14 |
|||
}, |
|||
{ |
|||
"id": 10009, |
|||
"username": "user-9", |
|||
"sex": "女", |
|||
"city": "城市-9", |
|||
"sign": "签名-9", |
|||
"experience": 484, |
|||
"logins": 25, |
|||
"wealth": 86801934, |
|||
"classify": "词人", |
|||
"score": 75 |
|||
} |
|||
] |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
{ |
|||
"code": 0, |
|||
"msg": "", |
|||
"count": 16, |
|||
"data": [ |
|||
{ "id":"001", "username":"张玉林", "sex":"女" }, |
|||
{ "id":"002", "username":"刘晓军", "sex":"男" }, |
|||
{ "id":"003", "username":"张恒", "sex":"男" }, |
|||
{ "id":"004", "username":"朱一", "sex":"男" }, |
|||
{ "id":"005", "username":"刘佳能", "sex":"女" }, |
|||
{ "id":"006", "username":"晓梅", "sex":"女" }, |
|||
{ "id":"007", "username":"马冬梅", "sex":"女" }, |
|||
{ "id":"008", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"009", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"010", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"011", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"012", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"013", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"014", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"015", "username":"刘晓庆", "sex":"女" }, |
|||
{ "id":"016", "username":"刘晓庆", "sex":"女" } |
|||
] |
|||
} |
|||
@ -0,0 +1,10 @@ |
|||
{ |
|||
"code": 1, |
|||
"msg": "上传成功", |
|||
"data": { |
|||
"url": [ |
|||
"../images/logo.png", |
|||
"../images/captcha.jpg" |
|||
] |
|||
} |
|||
} |
|||
@ -0,0 +1,803 @@ |
|||
/** |
|||
配色方案（如有需要，请自行配置） |
|||
*/ |
|||
/**头部-配色*/ |
|||
.layui-layout-admin .layui-header { |
|||
background-color: #1aa094 !important; |
|||
} |
|||
|
|||
.layui-header > ul > .layui-nav-item.layui-this, .layuimini-tool i:hover { |
|||
background-color: #197971 !important; |
|||
} |
|||
|
|||
.layui-header .layuimini-header-content > ul > .layui-nav-item.layui-this, .layuimini-tool i:hover { |
|||
background-color: #197971 !important; |
|||
} |
|||
|
|||
/**logo-配色*/ |
|||
.layui-layout-admin .layuimini-logo { |
|||
background-color: #243346 !important; |
|||
} |
|||
|
|||
/**左侧-配色*/ |
|||
.layui-side.layui-bg-black, .layui-side.layui-bg-black > .layuimini-menu-left > ul { |
|||
background-color: #2f4056 !important; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-child a:hover:not(.layui-this) { |
|||
background-color: #3b3f4b; |
|||
} |
|||
|
|||
/**左侧菜单选中-配色*/ |
|||
.layui-layout-admin .layui-nav-tree .layui-this, .layui-layout-admin .layui-nav-tree .layui-this > a, .layui-layout-admin .layui-nav-tree .layui-nav-child dd.layui-this, .layui-layout-admin .layui-nav-tree .layui-nav-child dd.layui-this a { |
|||
background-color: #1aa094 !important; |
|||
} |
|||
|
|||
|
|||
/**头部样式 */ |
|||
.layui-layout-admin .header { |
|||
position: fixed; |
|||
left: 0; |
|||
right: 0; |
|||
top: 0; |
|||
bottom: 0; |
|||
} |
|||
|
|||
.layuimini-header-menu, .layui-header { |
|||
height: 60px !important; |
|||
} |
|||
|
|||
.layuimini-header-menu > .layui-nav-item { |
|||
color: #1b1d21; |
|||
height: 60px !important; |
|||
line-height: 60px !important; |
|||
} |
|||
|
|||
.layui-header > .layui-layout-right > .layui-nav-item { |
|||
height: 60px !important; |
|||
line-height: 60px !important; |
|||
} |
|||
|
|||
.layui-layout-left { |
|||
left: 295px !important; |
|||
} |
|||
|
|||
.layui-nav.layui-layout-left.layuimini-header-menu.layuimini-pc-show { |
|||
font-weight: bold; |
|||
transition: all .2s; |
|||
} |
|||
|
|||
|
|||
/**logo演示（通用） */ |
|||
.layui-layout-admin .layuimini-logo { |
|||
font-weight: bold; |
|||
color: #ffffff !important; |
|||
height: 60px !important; |
|||
line-height: 60px !important; |
|||
overflow: hidden; |
|||
line-height: 64px; |
|||
transition: all .2s !important; |
|||
} |
|||
|
|||
.layui-layout-admin .layuimini-logo img { |
|||
display: inline-block; |
|||
height: 40px; |
|||
vertical-align: middle; |
|||
} |
|||
|
|||
.layui-layout-admin .layuimini-logo h1 { |
|||
display: inline-block; |
|||
margin: 0 0 0 12px; |
|||
color: #ffffff; |
|||
font-weight: 600; |
|||
font-size: 20px; |
|||
font-family: Avenir, Helvetica Neue, Arial, Helvetica, sans-serif; |
|||
vertical-align: middle; |
|||
} |
|||
|
|||
/**缩放工具（通用） */ |
|||
.layuimini-tool { |
|||
position: absolute !important; |
|||
top: 0; |
|||
left: 235px; |
|||
width: 60px; |
|||
height: 100%; |
|||
line-height: 60px; |
|||
text-align: center; |
|||
color: #ffffff !important; |
|||
transition: all .2s; |
|||
} |
|||
|
|||
/**缩放工具（缩放） */ |
|||
.layuimini-tool i { |
|||
display: block; |
|||
color: #bbe3df; |
|||
width: 32px; |
|||
height: 32px; |
|||
line-height: 32px; |
|||
border-radius: 3px; |
|||
text-align: center; |
|||
margin-top: 15px; |
|||
cursor: pointer; |
|||
} |
|||
|
|||
|
|||
.layuimini-page-header { |
|||
overflow: hidden; |
|||
display: block; |
|||
height: 35px; |
|||
line-height: 35px; |
|||
margin-bottom: 0; |
|||
border-radius: 0; |
|||
border-bottom: 1px solid #e1dddd; |
|||
} |
|||
|
|||
.layuimini-page-header .layui-breadcrumb { |
|||
border-top: 1px solid #f6f6f6; |
|||
padding: 0 15px; |
|||
visibility: visible; |
|||
} |
|||
|
|||
|
|||
/**左侧菜单栏 (通用) */ |
|||
.layui-side.layui-bg-black { |
|||
transition: all .2s; |
|||
} |
|||
|
|||
.layui-side.layui-bg-black > .layuimini-menu-left > ul { |
|||
transition: all .2s; |
|||
} |
|||
|
|||
.layui-side.layui-bg-black > .layuimini-menu-left > ul > .layui-nav-item:first-child { |
|||
border-top: 1px solid #4b5461; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-item a { |
|||
height: 40px; |
|||
line-height: 40px; |
|||
padding-right: 30px; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-item > a { |
|||
padding-top: 5px; |
|||
padding-bottom: 5px; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child { |
|||
background: 0 0 !important |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-more { |
|||
right: 15px; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-item a:hover { |
|||
background-color: transparent !important; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav { |
|||
background-color: transparent !important; |
|||
} |
|||
|
|||
|
|||
/**左侧菜单栏 (正常) */ |
|||
.layui-layout-body .layui-nav-itemed .layui-nav-child a, .layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child a { |
|||
padding-left: 35px; |
|||
} |
|||
|
|||
.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child a { |
|||
padding-left: 45px; |
|||
} |
|||
|
|||
.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child .layui-nav-child a { |
|||
padding-left: 55px; |
|||
} |
|||
|
|||
.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-child .layui-nav-child .layui-nav-child .layui-nav-child a { |
|||
padding-left: 65px; |
|||
} |
|||
|
|||
.layui-layout-body .layuimini-menu-left .layui-nav .layui-nav-itemed > .layui-nav-child { |
|||
padding: 5px 0; |
|||
} |
|||
|
|||
/**内容主体（通用） */ |
|||
.layui-layout-admin .layui-body { |
|||
/*position: fixed;*/ |
|||
overflow: hidden; |
|||
bottom: 0px !important; |
|||
top: 60px !important; |
|||
transition: all .2s; |
|||
} |
|||
|
|||
/**选择配色方案 */ |
|||
.layuimini-color .color-title { |
|||
padding: 10px 0 10px 20px; |
|||
border-bottom: 1px solid #d9dada; |
|||
margin-bottom: 8px; |
|||
} |
|||
|
|||
.layuimini-color .color-content { |
|||
padding: 10px 5px 0 5px; |
|||
} |
|||
|
|||
.layuimini-color .color-content ul { |
|||
list-style: none; |
|||
text-align: center; |
|||
} |
|||
|
|||
.layuimini-color .color-content ul li { |
|||
position: relative; |
|||
display: inline-block; |
|||
vertical-align: top; |
|||
width: 80px; |
|||
height: 50px; |
|||
margin: 0 15px 15px 0; |
|||
padding: 2px 2px 4px 2px; |
|||
background-color: #f2f2f2; |
|||
cursor: pointer; |
|||
font-size: 12px; |
|||
color: #666; |
|||
} |
|||
|
|||
.layuimini-color .color-content li.layui-this:after, .layuimini-color .color-content li:hover:after { |
|||
width: 100%; |
|||
height: 100%; |
|||
padding: 4px; |
|||
top: -5px; |
|||
left: -5px; |
|||
border-color: #d8d8d8; |
|||
opacity: 1; |
|||
} |
|||
|
|||
.layuimini-color .color-content li:after { |
|||
content: ''; |
|||
position: absolute; |
|||
z-index: 20; |
|||
top: 50%; |
|||
left: 50%; |
|||
width: 1px; |
|||
height: 0; |
|||
border: 1px solid #f2f2f2; |
|||
transition: all .3s; |
|||
-webkit-transition: all .3s; |
|||
opacity: 0; |
|||
} |
|||
|
|||
|
|||
/**其它 */ |
|||
.layui-tab-item { |
|||
width: 100% !important; |
|||
height: 100% !important; |
|||
} |
|||
|
|||
.layui-nav-item.layui-this { |
|||
background-color: #1b1d21; |
|||
} |
|||
|
|||
.layui-width-height { |
|||
width: 100%; |
|||
height: 95%; |
|||
} |
|||
|
|||
.layui-tab { |
|||
margin: 0 0 0 0; |
|||
z-index: 99999; |
|||
} |
|||
|
|||
.text-center { |
|||
height: 30px !important; |
|||
line-height: 30px !important; |
|||
text-align: center !important; |
|||
} |
|||
|
|||
.layui-nav { |
|||
padding: 0 !important; |
|||
} |
|||
|
|||
.layui-nav .layui-this:after, .layui-nav-bar, .layui-nav-tree .layui-nav-itemed:after { |
|||
width: 0 !important; |
|||
height: 0 !important; |
|||
} |
|||
|
|||
.layui-layout-admin .layui-side { |
|||
top: 60px !important; |
|||
} |
|||
|
|||
.layui-tab-card { |
|||
box-shadow: 0px 0px 0px #888888; |
|||
border-bottom: 0; |
|||
} |
|||
|
|||
|
|||
/*打开页面动画*/ |
|||
.layui-tab-item.layui-show { |
|||
animation: moveTop 1s; |
|||
-webkit-animation: moveTop 1s; |
|||
animation-fill-mode: both; |
|||
-webkit-animation-fill-mode: both; |
|||
position: relative; |
|||
height: 100%; |
|||
-webkit-overflow-scrolling: touch; |
|||
overflow: auto; |
|||
} |
|||
|
|||
@keyframes moveTop { |
|||
0% { |
|||
opacity: 0; |
|||
-webkit-transform: translateY(30px); |
|||
-ms-transform: translateY(30px); |
|||
transform: translateY(30px); |
|||
} |
|||
100% { |
|||
opacity: 1; |
|||
-webkit-transform: translateY(0); |
|||
-ms-transform: translateY(0); |
|||
transform: translateY(0); |
|||
} |
|||
} |
|||
|
|||
@-o-keyframes moveTop { |
|||
0% { |
|||
opacity: 0; |
|||
-webkit-transform: translateY(30px); |
|||
-ms-transform: translateY(30px); |
|||
transform: translateY(30px); |
|||
} |
|||
100% { |
|||
opacity: 1; |
|||
-webkit-transform: translateY(0); |
|||
-ms-transform: translateY(0); |
|||
transform: translateY(0); |
|||
} |
|||
} |
|||
|
|||
@-moz-keyframes moveTop { |
|||
0% { |
|||
opacity: 0; |
|||
-webkit-transform: translateY(30px); |
|||
-ms-transform: translateY(30px); |
|||
transform: translateY(30px); |
|||
} |
|||
100% { |
|||
opacity: 1; |
|||
-webkit-transform: translateY(0); |
|||
-ms-transform: translateY(0); |
|||
transform: translateY(0); |
|||
} |
|||
} |
|||
|
|||
@-webkit-keyframes moveTop { |
|||
0% { |
|||
opacity: 0; |
|||
-webkit-transform: translateY(30px); |
|||
-ms-transform: translateY(30px); |
|||
transform: translateY(30px); |
|||
} |
|||
100% { |
|||
opacity: 1; |
|||
-webkit-transform: translateY(0); |
|||
-ms-transform: translateY(0); |
|||
transform: translateY(0); |
|||
} |
|||
} |
|||
|
|||
/**自定义滚动条样式 */ |
|||
::-webkit-scrollbar { |
|||
width: 6px; |
|||
height: 6px |
|||
} |
|||
|
|||
::-webkit-scrollbar-track { |
|||
background-color: transparent; |
|||
-webkit-border-radius: 2em; |
|||
-moz-border-radius: 2em; |
|||
border-radius: 2em; |
|||
} |
|||
|
|||
::-webkit-scrollbar-thumb { |
|||
background-color: #9c9da0; |
|||
-webkit-border-radius: 2em; |
|||
-moz-border-radius: 2em; |
|||
border-radius: 2em |
|||
} |
|||
|
|||
|
|||
.layuimini-content-page { |
|||
overflow: auto; |
|||
width: 100%; |
|||
height: 100%; |
|||
} |
|||
|
|||
|
|||
/*移动端遮罩层*/ |
|||
.layuimini-make { |
|||
position: fixed; |
|||
left: 0; |
|||
right: 0; |
|||
bottom: 0; |
|||
top: 0; |
|||
z-index: 1000; |
|||
background: rgba(0, 0, 0, .5); |
|||
display: none; |
|||
} |
|||
|
|||
.layuimini-mini .layui-header { |
|||
z-index: 1001; |
|||
} |
|||
|
|||
/**初始化加载层*/ |
|||
.layuimini-loader { |
|||
position: fixed; |
|||
top: 0; |
|||
left: 0; |
|||
width: 100%; |
|||
height: 100%; |
|||
background-color: #ffffff; |
|||
z-index: 999999; |
|||
} |
|||
|
|||
.layuimini-loader .layuimini-loader-inner { |
|||
display: block; |
|||
position: relative; |
|||
left: 50%; |
|||
top: 50%; |
|||
width: 150px; |
|||
height: 150px; |
|||
margin: -75px 0 0 -75px; |
|||
border-radius: 50%; |
|||
border: 3px solid transparent; |
|||
border-top-color: #1E9FFF; |
|||
animation: spin 2s linear infinite; |
|||
} |
|||
|
|||
.layuimini-loader .layuimini-loader-inner:before { |
|||
content: ""; |
|||
position: absolute; |
|||
top: 5px; |
|||
left: 5px; |
|||
right: 5px; |
|||
bottom: 5px; |
|||
border-radius: 50%; |
|||
border: 3px solid transparent; |
|||
border-top-color: #1E9FFF; |
|||
animation: spin 3s linear infinite; |
|||
} |
|||
|
|||
.layuimini-loader .layuimini-loader-inner:after { |
|||
content: ""; |
|||
position: absolute; |
|||
top: 15px; |
|||
left: 15px; |
|||
right: 15px; |
|||
bottom: 15px; |
|||
border-radius: 50%; |
|||
border: 3px solid transparent; |
|||
border-top-color: #1E9FFF; |
|||
animation: spin 1.5s linear infinite; |
|||
} |
|||
|
|||
@keyframes spin { |
|||
0% { |
|||
transform: rotate(0deg); |
|||
} |
|||
to { |
|||
transform: rotate(1turn); |
|||
} |
|||
} |
|||
|
|||
/*系统设置*/ |
|||
|
|||
.layuimini-color .layui-word-aux { |
|||
position: absolute; |
|||
left: 60px; |
|||
top: 12px; |
|||
font-size: 12px; |
|||
} |
|||
|
|||
.layuimini-color .layui-input-block { |
|||
margin-left: 15px; |
|||
min-height: 36px; |
|||
} |
|||
|
|||
.layuimini-color .more-menu-list { |
|||
width: 100%; |
|||
margin-top: 30px; |
|||
} |
|||
|
|||
|
|||
.layuimini-color .more-menu-item:first-child { |
|||
border-top: 1px solid #e8e8e8; |
|||
} |
|||
|
|||
.layuimini-color .more-menu-item .layui-icon { |
|||
font-size: 18px; |
|||
padding-right: 10px; |
|||
} |
|||
|
|||
.layuimini-color .more-menu-item { |
|||
color: #595959; |
|||
height: 50px; |
|||
line-height: 50px; |
|||
font-size: 16px; |
|||
padding: 0 25px; |
|||
border-bottom: 1px solid #e8e8e8; |
|||
font-style: normal; |
|||
display: block; |
|||
} |
|||
|
|||
.layuimini-color .more-menu-item:hover { |
|||
background-color: whitesmoke; |
|||
} |
|||
|
|||
.layuimini-color .more-menu-item:after { |
|||
color: #8c8c8c; |
|||
right: 16px; |
|||
content: "\e602"; |
|||
position: absolute; |
|||
font-family: layui-icon !important; |
|||
} |
|||
|
|||
/** |
|||
菜单缩放 |
|||
*/ |
|||
.popup-tips .layui-layer-TipsG{ |
|||
display: none; |
|||
} |
|||
.popup-tips.layui-layer-tips .layui-layer-content{ |
|||
padding: 0; |
|||
} |
|||
.popup-tips .layui-nav-tree{ |
|||
width: 150px; |
|||
border-radius: 10px; |
|||
} |
|||
|
|||
/**左侧菜单字体间距*/ |
|||
.layuimini-menu-left .layui-nav-item a span { |
|||
letter-spacing: 1px; |
|||
} |
|||
|
|||
/**头部菜单字体间距*/ |
|||
.layui-layout-admin .layui-header .layuimini-header-menu.layuimini-pc-show,.layui-layout-admin .layui-header .layuimini-header-menu.layuimini-mobile-show { |
|||
letter-spacing: 1px; |
|||
} |
|||
|
|||
|
|||
/**左侧菜单更多下拉样式*/ |
|||
.layuimini-menu-left .layui-nav-more,.layuimini-menu-left-zoom .layui-nav-more { |
|||
font-family: layui-icon !important; |
|||
font-size: 12px; |
|||
font-style: normal; |
|||
-webkit-font-smoothing: antialiased; |
|||
-moz-osx-font-smoothing: grayscale; |
|||
overflow: hidden; |
|||
width: auto; |
|||
height: auto; |
|||
line-height: normal; |
|||
border: none; |
|||
display: inline-block; |
|||
margin-top: -6px !important; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav-child .layui-nav-more { |
|||
margin-top: -6px !important; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav .layui-nav-mored,.layuimini-menu-left .layui-nav-itemed>a .layui-nav-more{ |
|||
margin-top: -9px!important; |
|||
} |
|||
|
|||
.layuimini-menu-left-zoom.layui-nav .layui-nav-mored,.layuimini-menu-left-zoom.layui-nav-itemed>a .layui-nav-more{ |
|||
margin-top: -9px!important; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav-more:before,.layuimini-menu-left-zoom .layui-nav-more:before { |
|||
content: "\e61a"; |
|||
} |
|||
.layuimini-menu-left .layui-nav-itemed > a > .layui-nav-more,.layuimini-menu-left-zoom .layui-nav-itemed > a > .layui-nav-more { |
|||
transform: rotate(180deg); |
|||
-ms-transform: rotate(180deg); |
|||
-moz-transform: rotate(180deg); |
|||
-webkit-transform: rotate(180deg); |
|||
-o-transform: rotate(180deg); |
|||
width: 12px; |
|||
text-align: center; |
|||
border-style:none; |
|||
} |
|||
|
|||
.layuimini-menu-left .layui-nav-itemed > a > .layui-nav-more:before,.layuimini-menu-left-zoom .layui-nav-itemed > a > .layui-nav-more:before { |
|||
content: '\e61a'; |
|||
background-color: transparent; |
|||
display: inline-block; |
|||
vertical-align: middle; |
|||
} |
|||
|
|||
/**修复左侧菜单字体不对齐的问题*/ |
|||
.layuimini-menu-left .layui-nav-item a .fa,.layuimini-menu-left .layui-nav-item a .layui-icon{ |
|||
width: 20px; |
|||
} |
|||
|
|||
|
|||
|
|||
/** |
|||
PC版样式 |
|||
*/ |
|||
@media screen and (min-width: 1025px) { |
|||
/**头部样式（缩放） */ |
|||
.layuimini-mini .layui-layout-left.layuimini-header-menu.layuimini-pc-show { |
|||
left: 155px !important; |
|||
} |
|||
|
|||
/**logo演示（缩放） */ |
|||
.layuimini-mini .layui-layout-admin .layuimini-logo { |
|||
width: 60px !important; |
|||
} |
|||
|
|||
.layuimini-mini .layui-layout-admin .layuimini-logo h1 { |
|||
display: none; |
|||
} |
|||
|
|||
/**左侧菜单栏（缩放） */ |
|||
.layuimini-mini .layuimini-menu-left { |
|||
width: 80px !important; |
|||
} |
|||
|
|||
.layuimini-mini .layui-side.layui-bg-black, .layuimini-mini .layuimini-menu-left > ul, .layuimini-mini .layuimini-menu-left > ul li i { |
|||
width: 60px !important; |
|||
} |
|||
|
|||
.layuimini-mini .layuimini-menu-left > ul li span:first-child { |
|||
display: none; |
|||
} |
|||
|
|||
.layuimini-mini .layuimini-menu-left > ul li span:last-child { |
|||
float: right; |
|||
right: 7px; |
|||
} |
|||
|
|||
.layuimini-mini .layuimini-menu-left .layui-nav .layui-nav-item a { |
|||
height: 40px; |
|||
line-height: 40px; |
|||
padding-right: 0px !important; |
|||
} |
|||
|
|||
/**内容主体（缩放） */ |
|||
.layuimini-mini .layui-layout-admin { |
|||
left: 60px !important; |
|||
} |
|||
|
|||
.layuimini-mini .layuimini-tool { |
|||
left: 95px !important; |
|||
} |
|||
|
|||
.layuimini-pc-show{ |
|||
display: block; |
|||
} |
|||
.layuimini-mobile-show{ |
|||
display: none; |
|||
} |
|||
|
|||
/**菜单缩放*/ |
|||
.layuimini-mini .layuimini-menu-left .layui-nav-more,.layuimini-mini .layuimini-menu-left .layui-nav-child{ |
|||
display: none!important; |
|||
} |
|||
|
|||
} |
|||
|
|||
/** |
|||
手机自适应样式 |
|||
*/ |
|||
@media screen and (max-width: 1024px) { |
|||
|
|||
.layuimini-pc-show{ |
|||
display: none; |
|||
} |
|||
.layuimini-mobile-show{ |
|||
display: block; |
|||
} |
|||
|
|||
.layuimini-header-content { |
|||
left: 0; |
|||
} |
|||
|
|||
.layui-layout-admin .layui-body .layui-tab-item.layui-show { |
|||
border-top: 1px solid #e2e2e2; |
|||
} |
|||
|
|||
.layuimini-all .layui-layout-left.layuimini-header-menu { |
|||
left: 15px !important |
|||
} |
|||
|
|||
.layuimini-mini .layui-layout-left.layuimini-header-menu { |
|||
left: 205px !important |
|||
} |
|||
|
|||
.layui-layout-admin .layui-nav.layui-layout-right > li:not(.layuimini-setting) { |
|||
width: 40px !important; |
|||
} |
|||
|
|||
.layui-layout-admin .layui-nav.layui-layout-right > li:not(.layuimini-setting) a { |
|||
padding: 0 15px; |
|||
} |
|||
|
|||
.layuimini-all .layui-layout-admin .layui-body { |
|||
left: 0px !important; |
|||
} |
|||
|
|||
.layuimini-mini .layui-layout-admin .layuimini-menu-left, .layuimini-mini .layui-header .layuimini-logo { |
|||
left: 0; |
|||
transition: left .2s; |
|||
z-index: 1001 !important; |
|||
} |
|||
|
|||
.layuimini-all .layui-layout-admin .layuimini-menu-left, .layuimini-all .layui-header .layuimini-logo { |
|||
left: -200px; |
|||
transition: left .2s; |
|||
top: 0; |
|||
z-index: 1002; |
|||
} |
|||
|
|||
.layuimini-mini .layui-layout-admin .layui-body { |
|||
left: 0!important; |
|||
transition: left .2s; |
|||
top: 0; |
|||
z-index: 998; |
|||
} |
|||
|
|||
.layuimini-mini .layuimini-make { |
|||
display: block; |
|||
} |
|||
|
|||
.layuimini-multi-module .layuimini-header-content .layuimini-tool { |
|||
display: none; |
|||
} |
|||
|
|||
.layuimini-single-module .layuimini-header-content .layuimini-tool { |
|||
left: 15px; |
|||
} |
|||
|
|||
.layuimini-mini .layuimini-site-mobile { |
|||
display: none !important; |
|||
} |
|||
|
|||
.layuimini-site-mobile { |
|||
display: block !important; |
|||
position: fixed; |
|||
z-index: 100000; |
|||
bottom: 15px; |
|||
left: 15px; |
|||
width: 40px; |
|||
height: 40px; |
|||
line-height: 40px; |
|||
border-radius: 2px; |
|||
text-align: center; |
|||
background-color: rgba(0, 0, 0, .7); |
|||
color: #fff; |
|||
} |
|||
|
|||
.layuimini-header-content { |
|||
z-index: 997; |
|||
} |
|||
|
|||
.layuimini-content-page { |
|||
-webkit-overflow-scrolling: touch; |
|||
} |
|||
|
|||
/*修复UC之类的浏览器点击无效*/ |
|||
.layuimini-make { |
|||
cursor: pointer; |
|||
} |
|||
|
|||
.layuimini-site-mobile { |
|||
cursor: pointer; |
|||
} |
|||
|
|||
} |
|||
|
|||
@media screen and (max-width: 550px){ |
|||
|
|||
/**头部右侧数据*/ |
|||
.layuimini-multi-module.layuimini-mini .layuimini-header-content .layui-layout-right { |
|||
display: none; |
|||
} |
|||
} |
|||
@ -0,0 +1,13 @@ |
|||
.layuimini-content-page{background-color:#f2f2f2!important;} |
|||
.layuimini-container {border:1px solid #f2f2f2;border-radius:5px;background-color:#f2f2f2} |
|||
.layuimini-main {margin:10px 10px 10px 10px;border:5px solid #ffffff;border-radius:5px;background-color:#ffffff} |
|||
.layui-breadcrumb>* {font-size: 13px;!important;} |
|||
|
|||
/**必填红点 */ |
|||
.layuimini-form>.layui-form-item>.required:after {content:'*';color:red;position:absolute;margin-left:4px;font-weight:bold;line-height:1.8em;top:6px;right:5px;} |
|||
.layuimini-form>.layui-form-item>.layui-form-label {width:120px !important;} |
|||
.layuimini-form>.layui-form-item>.layui-input-block {margin-left:150px !important;} |
|||
.layuimini-form>.layui-form-item>.layui-input-block >tip {display:inline-block;margin-top:10px;line-height:10px;font-size:10px;color:#a29c9c;} |
|||
|
|||
/**搜索框*/ |
|||
.layuimini-container .table-search-fieldset {margin: 0;border: 1px solid #e6e6e6;padding: 10px 20px 5px 20px;color: #6b6b6b;} |
|||
@ -0,0 +1,95 @@ |
|||
/*头部右侧背景色 headerRightBg */ |
|||
.layui-layout-admin .layui-header { |
|||
background-color: #ffffff !important; |
|||
} |
|||
|
|||
/*头部右侧选中背景色 headerRightBgThis */ |
|||
.layui-layout-admin .layui-header .layuimini-header-content > ul > .layui-nav-item.layui-this, .layuimini-tool i:hover { |
|||
background-color: #e4e4e4 !important; |
|||
} |
|||
|
|||
/*头部右侧字体颜色 headerRightColor */ |
|||
.layui-layout-admin .layui-header .layui-nav .layui-nav-item a { |
|||
color: rgba(107, 107, 107, 0.7); |
|||
} |
|||
|
|||
/**头部右侧下拉字体颜色 headerRightChildColor */ |
|||
.layui-layout-admin .layui-header .layui-nav .layui-nav-item .layui-nav-child a { |
|||
color: rgba(107, 107, 107, 0.7) !important; |
|||
} |
|||
|
|||
/*头部右侧鼠标选中 headerRightColorThis */ |
|||
.layui-header .layuimini-menu-header-pc.layui-nav .layui-nav-item a:hover, .layui-header .layuimini-header-menu.layuimini-pc-show.layui-nav .layui-this a { |
|||
color: #565656 !important; |
|||
} |
|||
|
|||
/*头部右侧更多下拉颜色 headerRightNavMore */ |
|||
.layui-header .layui-nav .layui-nav-more { |
|||
border-top-color: rgba(160, 160, 160, 0.7) !important; |
|||
} |
|||
|
|||
/*头部右侧更多下拉颜色 headerRightNavMore */ |
|||
.layui-header .layui-nav .layui-nav-mored, .layui-header .layui-nav-itemed > a .layui-nav-more { |
|||
border-color: transparent transparent rgba(160, 160, 160, 0.7) !important; |
|||
} |
|||
|
|||
/**头部右侧更多下拉配置色 headerRightNavMoreBg headerRightNavMoreColor */ |
|||
.layui-header .layui-nav .layui-nav-child dd.layui-this a, .layui-header .layui-nav-child dd.layui-this, .layui-layout-admin .layui-header .layui-nav .layui-nav-item .layui-nav-child .layui-this a { |
|||
background-color: #1E9FFF !important; |
|||
color: #ffffff !important; |
|||
} |
|||
|
|||
/*头部缩放按钮样式 headerRightToolColor */ |
|||
.layui-layout-admin .layui-header .layuimini-tool i { |
|||
color: #565656; |
|||
} |
|||
|
|||
/*logo背景颜色 headerLogoBg */ |
|||
.layui-layout-admin .layuimini-logo { |
|||
background-color: #192027 !important; |
|||
} |
|||
|
|||
/*logo字体颜色 headerLogoColor */ |
|||
.layui-layout-admin .layuimini-logo h1 { |
|||
color: rgb(191, 187, 187); |
|||
} |
|||
|
|||
/*左侧菜单更多下拉样式 leftMenuNavMore */ |
|||
.layuimini-menu-left .layui-nav .layui-nav-more, .layuimini-menu-left-zoom.layui-nav .layui-nav-more { |
|||
border-top-color: rgb(191, 187, 187); |
|||
} |
|||
|
|||
/*左侧菜单更多下拉样式 leftMenuNavMore */ |
|||
.layuimini-menu-left .layui-nav .layui-nav-mored, .layuimini-menu-left .layui-nav-itemed > a .layui-nav-more, .layuimini-menu-left-zoom.layui-nav .layui-nav-mored, .layuimini-menu-left-zoom.layui-nav-itemed > a .layui-nav-more { |
|||
border-color: transparent transparent rgb(191, 187, 187) !important; |
|||
} |
|||
|
|||
/*左侧菜单背景 leftMenuBg */ |
|||
.layui-side.layui-bg-black, .layui-side.layui-bg-black > .layuimini-menu-left > ul, .layuimini-menu-left-zoom > ul { |
|||
background-color: #28333E !important; |
|||
} |
|||
|
|||
/*左侧菜单选中背景 leftMenuBgThis */ |
|||
.layuimini-menu-left .layui-nav-tree .layui-this, .layuimini-menu-left .layui-nav-tree .layui-this > a, .layuimini-menu-left .layui-nav-tree .layui-nav-child dd.layui-this, .layuimini-menu-left .layui-nav-tree .layui-nav-child dd.layui-this a, .layuimini-menu-left-zoom.layui-nav-tree .layui-this, .layuimini-menu-left-zoom.layui-nav-tree .layui-this > a, .layuimini-menu-left-zoom.layui-nav-tree .layui-nav-child dd.layui-this, .layuimini-menu-left-zoom.layui-nav-tree .layui-nav-child dd.layui-this a { |
|||
background-color: #1E9FFF !important |
|||
} |
|||
|
|||
/*左侧菜单子菜单背景 leftMenuChildBg */ |
|||
.layuimini-menu-left .layui-nav-itemed > .layui-nav-child { |
|||
background-color: #0c0f13 !important; |
|||
} |
|||
|
|||
/*左侧菜单字体颜色 leftMenuColor */ |
|||
.layuimini-menu-left .layui-nav .layui-nav-item a, .layuimini-menu-left-zoom.layui-nav .layui-nav-item a { |
|||
color: rgb(191, 187, 187) !important; |
|||
} |
|||
|
|||
/*左侧菜单选中字体颜色 leftMenuColorThis */ |
|||
.layuimini-menu-left .layui-nav .layui-nav-item a:hover, .layuimini-menu-left .layui-nav .layui-this a, .layuimini-menu-left-zoom.layui-nav .layui-nav-item a:hover, .layuimini-menu-left-zoom.layui-nav .layui-this a { |
|||
color: #ffffff !important; |
|||
} |
|||
|
|||
/**tab选项卡选中颜色 tabActiveColor */ |
|||
.layuimini-tab .layui-tab-title .layui-this .layuimini-tab-active { |
|||
background-color: #1e9fff; |
|||
} |
|||
|
After Width: | Height: | Size: 26 KiB |
|
After Width: | Height: | Size: 2.4 KiB |
|
After Width: | Height: | Size: 50 KiB |
|
After Width: | Height: | Size: 4.2 KiB |
|
After Width: | Height: | Size: 157 KiB |
|
After Width: | Height: | Size: 2.7 KiB |
|
After Width: | Height: | Size: 671 KiB |
|
After Width: | Height: | Size: 74 KiB |
@ -0,0 +1,68 @@ |
|||
<!DOCTYPE html> |
|||
<html> |
|||
<head> |
|||
<meta charset="utf-8"> |
|||
<title>go-mysql-transfer</title> |
|||
<meta name="renderer" content="webkit"> |
|||
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> |
|||
<meta http-equiv="Access-Control-Allow-Origin" content="*"> |
|||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"> |
|||
<meta name="apple-mobile-web-app-status-bar-style" content="black"> |
|||
<meta name="apple-mobile-web-app-capable" content="yes"> |
|||
<meta name="format-detection" content="telephone=no"> |
|||
<link rel="icon" href="images/favicon.ico"> |
|||
<link rel="stylesheet" href="lib/layui-v2.5.5/css/layui.css" media="all"> |
|||
<link rel="stylesheet" href="lib/font-awesome-4.7.0/css/font-awesome.min.css" media="all"> |
|||
<link rel="stylesheet" href="css/layuimini.css?v=2.0.1" media="all"> |
|||
<link rel="stylesheet" href="css/themes/default.css" media="all"> |
|||
<link rel="stylesheet" href="css/public.css" media="all"> |
|||
<!--[if lt IE 9]> |
|||
<script src="https://cdn.staticfile.org/html5shiv/r29/html5.min.js"></script> |
|||
<script src="https://cdn.staticfile.org/respond.js/1.4.2/respond.min.js"></script> |
|||
<![endif]--> |
|||
<style id="layuimini-bg-color"> |
|||
</style> |
|||
</head> |
|||
|
|||
<div class="layuimini-content-page"></div> |
|||
|
|||
<script src="lib/layui-v2.5.5/layui.js" charset="utf-8"></script> |
|||
<script src="js/lay-config.js?v=2.0.0" charset="utf-8"></script> |
|||
<script> |
|||
layui.use(['jquery', 'layer', 'miniAdmin', 'miniTongji'], function () { |
|||
var $ = layui.jquery, |
|||
layer = layui.layer, |
|||
miniAdmin = layui.miniAdmin, |
|||
miniTongji = layui.miniTongji; |
|||
|
|||
var options = { |
|||
iniUrl: "api/init.json", // 初始化接口 |
|||
clearUrl: "api/clear.json", // 缓存清理接口 |
|||
renderPageVersion: true, // 初始化页面是否加版本号 |
|||
bgColorDefault: false, // 主题默认配置 |
|||
multiModule: true, // 是否开启多模块 |
|||
menuChildOpen: false, // 是否默认展开菜单 |
|||
loadingTime: 0, // 初始化加载时间 |
|||
pageAnim: true, // 切换菜单动画 |
|||
}; |
|||
miniAdmin.render(options); |
|||
|
|||
// 百度统计代码，只统计指定域名 |
|||
miniTongji.render({ |
|||
specific: true, |
|||
domains: [ |
|||
'99php.cn', |
|||
'layuimini.99php.cn', |
|||
'layuimini-onepage.99php.cn', |
|||
], |
|||
}); |
|||
|
|||
$('.login-out').on("click", function () { |
|||
layer.msg('退出登录成功', function () { |
|||
window.location = 'page/login-3.html'; |
|||
}); |
|||
}); |
|||
}); |
|||
</script> |
|||
</body> |
|||
</html> |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* date:2019/08/16 |
|||
* author:Mr.Chung |
|||
* description:此处放layui自定义扩展 |
|||
*/ |
|||
|
|||
window.rootPath = (function (src) { |
|||
src = document.scripts[document.scripts.length - 1].src; |
|||
return src.substring(0, src.lastIndexOf("/") + 1); |
|||
})(); |
|||
|
|||
layui.config({ |
|||
base: rootPath + "lay-module/", |
|||
version: true |
|||
}).extend({ |
|||
miniAdmin: "layuimini/miniAdmin", // layuimini后台扩展
|
|||
miniMenu: "layuimini/miniMenu", // layuimini菜单扩展
|
|||
miniPage: "layuimini/miniPage", // layuimini 单页扩展
|
|||
miniTheme: "layuimini/miniTheme", // layuimini 主题扩展
|
|||
miniTongji: "layuimini/miniTongji", // layuimini 统计扩展
|
|||
step: 'step-lay/step', // 分步表单扩展
|
|||
treetable: 'treetable-lay/treetable', //table树形扩展
|
|||
tableSelect: 'tableSelect/tableSelect', // table选择扩展
|
|||
iconPickerFa: 'iconPicker/iconPickerFa', // fa图标选择扩展
|
|||
echarts: 'echarts/echarts', // echarts图表扩展
|
|||
echartsTheme: 'echarts/echartsTheme', // echarts图表主题扩展
|
|||
wangEditor: 'wangEditor/wangEditor', // wangEditor富文本扩展
|
|||
layarea: 'layarea/layarea', // 省市县区三级联动下拉选择器
|
|||
}); |
|||
File diff suppressed because one or more lines are too long
@ -0,0 +1,492 @@ |
|||
layui.define(function(exports) { |
|||
exports('echartsTheme', |
|||
{ |
|||
"color": [ |
|||
"#3fb1e3", |
|||
"#6be6c1", |
|||
"#626c91", |
|||
"#a0a7e6", |
|||
"#c4ebad", |
|||
"#96dee8" |
|||
], |
|||
"backgroundColor": "rgba(252,252,252,0)", |
|||
"textStyle": {}, |
|||
"title": { |
|||
"textStyle": { |
|||
"color": "#666666" |
|||
}, |
|||
"subtextStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"line": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": "3" |
|||
} |
|||
}, |
|||
"lineStyle": { |
|||
"normal": { |
|||
"width": "4" |
|||
} |
|||
}, |
|||
"symbolSize": "10", |
|||
"symbol": "emptyCircle", |
|||
"smooth": true |
|||
}, |
|||
"radar": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": "3" |
|||
} |
|||
}, |
|||
"lineStyle": { |
|||
"normal": { |
|||
"width": "4" |
|||
} |
|||
}, |
|||
"symbolSize": "10", |
|||
"symbol": "emptyCircle", |
|||
"smooth": true |
|||
}, |
|||
"bar": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"barBorderWidth": 0, |
|||
"barBorderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"barBorderWidth": 0, |
|||
"barBorderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"pie": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"scatter": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"boxplot": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"parallel": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"sankey": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"funnel": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"gauge": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
}, |
|||
"emphasis": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
} |
|||
}, |
|||
"candlestick": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"color": "#e6a0d2", |
|||
"color0": "transparent", |
|||
"borderColor": "#e6a0d2", |
|||
"borderColor0": "#3fb1e3", |
|||
"borderWidth": "2" |
|||
} |
|||
} |
|||
}, |
|||
"graph": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"borderWidth": 0, |
|||
"borderColor": "#ccc" |
|||
} |
|||
}, |
|||
"lineStyle": { |
|||
"normal": { |
|||
"width": "1", |
|||
"color": "#cccccc" |
|||
} |
|||
}, |
|||
"symbolSize": "10", |
|||
"symbol": "emptyCircle", |
|||
"smooth": true, |
|||
"color": [ |
|||
"#3fb1e3", |
|||
"#6be6c1", |
|||
"#626c91", |
|||
"#a0a7e6", |
|||
"#c4ebad", |
|||
"#96dee8" |
|||
], |
|||
"label": { |
|||
"normal": { |
|||
"textStyle": { |
|||
"color": "#ffffff" |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
"map": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"areaColor": "#eeeeee", |
|||
"borderColor": "#aaaaaa", |
|||
"borderWidth": 0.5 |
|||
}, |
|||
"emphasis": { |
|||
"areaColor": "rgba(63,177,227,0.25)", |
|||
"borderColor": "#3fb1e3", |
|||
"borderWidth": 1 |
|||
} |
|||
}, |
|||
"label": { |
|||
"normal": { |
|||
"textStyle": { |
|||
"color": "#ffffff" |
|||
} |
|||
}, |
|||
"emphasis": { |
|||
"textStyle": { |
|||
"color": "rgb(63,177,227)" |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
"geo": { |
|||
"itemStyle": { |
|||
"normal": { |
|||
"areaColor": "#eeeeee", |
|||
"borderColor": "#aaaaaa", |
|||
"borderWidth": 0.5 |
|||
}, |
|||
"emphasis": { |
|||
"areaColor": "rgba(63,177,227,0.25)", |
|||
"borderColor": "#3fb1e3", |
|||
"borderWidth": 1 |
|||
} |
|||
}, |
|||
"label": { |
|||
"normal": { |
|||
"textStyle": { |
|||
"color": "#ffffff" |
|||
} |
|||
}, |
|||
"emphasis": { |
|||
"textStyle": { |
|||
"color": "rgb(63,177,227)" |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
"categoryAxis": { |
|||
"axisLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": "#cccccc" |
|||
} |
|||
}, |
|||
"axisTick": { |
|||
"show": false, |
|||
"lineStyle": { |
|||
"color": "#333" |
|||
} |
|||
}, |
|||
"axisLabel": { |
|||
"show": true, |
|||
"textStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"splitLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": [ |
|||
"#eeeeee" |
|||
] |
|||
} |
|||
}, |
|||
"splitArea": { |
|||
"show": false, |
|||
"areaStyle": { |
|||
"color": [ |
|||
"rgba(250,250,250,0.05)", |
|||
"rgba(200,200,200,0.02)" |
|||
] |
|||
} |
|||
} |
|||
}, |
|||
"valueAxis": { |
|||
"axisLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": "#cccccc" |
|||
} |
|||
}, |
|||
"axisTick": { |
|||
"show": false, |
|||
"lineStyle": { |
|||
"color": "#333" |
|||
} |
|||
}, |
|||
"axisLabel": { |
|||
"show": true, |
|||
"textStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"splitLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": [ |
|||
"#eeeeee" |
|||
] |
|||
} |
|||
}, |
|||
"splitArea": { |
|||
"show": false, |
|||
"areaStyle": { |
|||
"color": [ |
|||
"rgba(250,250,250,0.05)", |
|||
"rgba(200,200,200,0.02)" |
|||
] |
|||
} |
|||
} |
|||
}, |
|||
"logAxis": { |
|||
"axisLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": "#cccccc" |
|||
} |
|||
}, |
|||
"axisTick": { |
|||
"show": false, |
|||
"lineStyle": { |
|||
"color": "#333" |
|||
} |
|||
}, |
|||
"axisLabel": { |
|||
"show": true, |
|||
"textStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"splitLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": [ |
|||
"#eeeeee" |
|||
] |
|||
} |
|||
}, |
|||
"splitArea": { |
|||
"show": false, |
|||
"areaStyle": { |
|||
"color": [ |
|||
"rgba(250,250,250,0.05)", |
|||
"rgba(200,200,200,0.02)" |
|||
] |
|||
} |
|||
} |
|||
}, |
|||
"timeAxis": { |
|||
"axisLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": "#cccccc" |
|||
} |
|||
}, |
|||
"axisTick": { |
|||
"show": false, |
|||
"lineStyle": { |
|||
"color": "#333" |
|||
} |
|||
}, |
|||
"axisLabel": { |
|||
"show": true, |
|||
"textStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"splitLine": { |
|||
"show": true, |
|||
"lineStyle": { |
|||
"color": [ |
|||
"#eeeeee" |
|||
] |
|||
} |
|||
}, |
|||
"splitArea": { |
|||
"show": false, |
|||
"areaStyle": { |
|||
"color": [ |
|||
"rgba(250,250,250,0.05)", |
|||
"rgba(200,200,200,0.02)" |
|||
] |
|||
} |
|||
} |
|||
}, |
|||
"toolbox": { |
|||
"iconStyle": { |
|||
"normal": { |
|||
"borderColor": "#999999" |
|||
}, |
|||
"emphasis": { |
|||
"borderColor": "#666666" |
|||
} |
|||
} |
|||
}, |
|||
"legend": { |
|||
"textStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"tooltip": { |
|||
"axisPointer": { |
|||
"lineStyle": { |
|||
"color": "#cccccc", |
|||
"width": 1 |
|||
}, |
|||
"crossStyle": { |
|||
"color": "#cccccc", |
|||
"width": 1 |
|||
} |
|||
} |
|||
}, |
|||
"timeline": { |
|||
"lineStyle": { |
|||
"color": "#626c91", |
|||
"width": 1 |
|||
}, |
|||
"itemStyle": { |
|||
"normal": { |
|||
"color": "#626c91", |
|||
"borderWidth": 1 |
|||
}, |
|||
"emphasis": { |
|||
"color": "#626c91" |
|||
} |
|||
}, |
|||
"controlStyle": { |
|||
"normal": { |
|||
"color": "#626c91", |
|||
"borderColor": "#626c91", |
|||
"borderWidth": 0.5 |
|||
}, |
|||
"emphasis": { |
|||
"color": "#626c91", |
|||
"borderColor": "#626c91", |
|||
"borderWidth": 0.5 |
|||
} |
|||
}, |
|||
"checkpointStyle": { |
|||
"color": "#3fb1e3", |
|||
"borderColor": "rgba(63,177,227,0.15)" |
|||
}, |
|||
"label": { |
|||
"normal": { |
|||
"textStyle": { |
|||
"color": "#626c91" |
|||
} |
|||
}, |
|||
"emphasis": { |
|||
"textStyle": { |
|||
"color": "#626c91" |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
"visualMap": { |
|||
"color": [ |
|||
"#2a99c9", |
|||
"#afe8ff" |
|||
] |
|||
}, |
|||
"dataZoom": { |
|||
"backgroundColor": "rgba(255,255,255,0)", |
|||
"dataBackgroundColor": "rgba(222,222,222,1)", |
|||
"fillerColor": "rgba(114,230,212,0.25)", |
|||
"handleColor": "#cccccc", |
|||
"handleSize": "100%", |
|||
"textStyle": { |
|||
"color": "#999999" |
|||
} |
|||
}, |
|||
"markPoint": { |
|||
"label": { |
|||
"normal": { |
|||
"textStyle": { |
|||
"color": "#ffffff" |
|||
} |
|||
}, |
|||
"emphasis": { |
|||
"textStyle": { |
|||
"color": "#ffffff" |
|||
} |
|||
} |
|||
} |
|||
} |
|||
}); |
|||
}); |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue