DataX

DataX
DataX 是阿里巴巴集团内被广泛使用的离线数据同步工具/平台，实现包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、DRDS 等各种异构数据源之间高效的数据同步功能。

下载安装datax.tar，下载链接https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md

MysqlWriter 插件实现了写入数据到 Mysql 主库的目的表的功能。在底层实现上， MysqlWriter 通过 JDBC 连接远程 Mysql 数据库，并执行相应的 insert into ... 或者 ( replace into ...) 的 sql 语句将数据写入 Mysql，内部会分批次提交入库，需要数据库本身采用 innodb 引擎。

MysqlWriter 面向ETL开发工程师，他们使用 MysqlWriter 从数仓导入数据到 Mysql。同时 MysqlWriter 亦可以作为数据迁移工具为DBA等用户提供服务。

[root@scripts]# tar -xvf datax.tar

[root@datax]# ls
bin conf job lib log log_perf plugin script tmp

[root@job]# ls
job.json

[root@job]# vi mysql2mysql.json

[root@job]# cat mysql2mysql.json
{
"job": {
"setting": {
"speed": {
"channel": 8
},
"errorLimit": {
"record": 0,
"percentage": 0.02
}
},
"content": [
{
"reader": {
"name": "mysqlreader",
"parameter": {
"username": "username",
"password": "password",
"column": [
"order_id",
"parent_order_id",
"chartered_id",
"multi_order_id",
"carpool_order_id",
"order_no",
"city_id",
"type",
"order_service_type",
"booking_user_id",
"booking_phone",
"rider_phone",
"booking_start_time",
"booking_start_addr",
"booking_end_addr",
"booking_start_short_addr",
"booking_end_short_addr",
"driver_id",
"license_plates",
"order_status",
"channels_num",
"is_display",
"dissent_flag",
"is_carpool",
"create_date",
"update_date",
"sub_order_count",
"sub_order_flag",
"push_driver_type",
"buyout_flag",
"is_order_others",
"auto_level_up",
"finished_date",
"order_create_date",
"order_update_date",
"cancel_type",
"business_id",
"order_type",
"fact_date",
"fact_start_addr",
"fact_end_addr",
"cancelorder_penalty"
],
"connection": [
{
"table": [
"yoon_table" #表名
],
"jdbcUrl": [
"jdbc:mysql://192.168.11.111:3306/yoon" #连接地址、端口、数据库
]
}
],
"where": "order_create_date >='2019-05-01 00:00:00' and order_create_date < '2019-06-01 00:00:00'" #根据where条件传输数据，全表导的话就把where条件去掉即可
}
},
"writer": {
"name": "mysqlwriter",
"parameter": {
"writeMode": "replace",
"username": "username",
"password": "password",
"column": [
"order_id",
"parent_order_id",
"chartered_id",
"multi_order_id",
"carpool_order_id",
"order_no",
"city_id",
"type",
"order_service_type",
"booking_user_id",
"booking_phone",
"rider_phone",
"booking_start_time",
"booking_start_addr",
"booking_end_addr",
"booking_start_short_addr",
"booking_end_short_addr",
"driver_id",
"license_plates",
"order_status",
"channels_num",
"is_display",
"dissent_flag",
"is_carpool",
"create_date",
"update_date",
"sub_order_count",
"sub_order_flag",
"push_driver_type",
"buyout_flag",
"is_order_others",
"auto_level_up",
"finished_date",
"order_create_date",
"order_update_date",
"cancel_type",
"business_id",
"order_type",
"fact_date",
"fact_start_addr",
"fact_end_addr",
"cancelorder_penalty"
],
"connection": [
{
"jdbcUrl": "jdbc:mysql://192.168.11.222:3306/hank?useUnicode=true&characterEncoding=utf8", #连接地址、端口、数据库
"table": [
"yoon_hank" #表名
]
}
]
}
}
}
]
}
}