复制现象是，slave线程状态正常，但是sql 线程不应用，所以delay越来越大，查看复制状态
mysql> show slave statusG
*************************** 1. row ***************************
Slave_IO_State: System lock
Master_Host:
Master_User: repl
Master_Port: xxxx
Connect_Retry: 60
Master_Log_File: mysql-bin.000056
Read_Master_Log_Pos: 384142784
Relay_Log_File: relay-log.000014
Relay_Log_Pos: 6517796
Relay_Master_Log_File: mysql-bin.000053
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 6517623
Relay_Log_Space: 4657584528
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 206184
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 102121028
Master_UUID: 7eeec54f-6464-11e7-a16a-20f17c59daa2
Master_Info_File: mysql.slave_master_info
SQL_Delay: 0
SQL_Remaining_Delay: NULL
Slave_SQL_Running_State: Waiting for Slave Worker queue
Master_Retry_Count: 86400
Master_Bind:
Last_IO_Error_Timestamp:
Last_SQL_Error_Timestamp:
Master_SSL_Crl:
Master_SSL_Crlpath:
Retrieved_Gtid_Set: 7eeec54f-6464-11e7-a16a-20f17c59daa2:53936753-54857989
Executed_Gtid_Set: 7eeec54f-6464-11e7-a16a-20f17c59daa2:1-54494096,
ff8f1aa3-7431-11e6-888c-1051722626c0:42306882-42306884
Auto_Position: 1
Replicate_Rewrite_DB:
Channel_Name:
Master_TLS_Version:
---TRANSACTION 339088058, ACTIVE 43636 sec setting auto-inc lock
mysql tables in use 1, locked 1
1 lock struct(s), heap size 1136, 0 row lock(s), undo log entries 7842577
MySQL thread id 16, OS thread handle 140560924931840, query id 38813829 System lock
--------
mysql> select * from information_schema.innodb_trxG *************************** 1. row *************************** trx_id: 339088058 trx_state: RUNNING trx_started: 2017-11-03 20:53:34 trx_requested_lock_id: NULL trx_wait_started: NULL trx_weight: 7846747 trx_mysql_thread_id: 16 trx_query: NULL trx_operation_state: setting auto-inc lock trx_tables_in_use: 1 trx_tables_locked: 1 trx_lock_structs: 1 trx_lock_memory_bytes: 1136 trx_rows_locked: 0 trx_rows_modified: 7846746 trx_concurrency_tickets: 0 trx_isolation_level: READ COMMITTED trx_unique_checks: 1 trx_foreign_key_checks: 1 trx_last_foreign_key_error: NULL trx_adaptive_hash_latched: 0 trx_adaptive_hash_timeout: 0 trx_is_read_only: 0 trx_autocommit_non_locking: 0 1 row in set (0.00 sec) mysql> show processlist; +------+-------------+--------------------+-------------------+---------+--------+---------------------------------------+------------------+ | Id | User | Host | db | Command | Time | State | Info | +------+-------------+--------------------+-------------------+---------+--------+---------------------------------------+------------------+ | 14 | system user | | NULL | Connect | 155776 | Waiting for master to send event | NULL | | 15 | system user | | NULL | Connect | 0 | Waiting for Slave Worker queue | NULL | | 16 | system user | | NULL | Connect | 209694 | System lock | NULL | | 17 | system user | | NULL | Connect | 207404 | Waiting for an event from Coordinator | NULL | | 18 | system user | | NULL | Connect | 213656 | Waiting for an event from Coordinator | NULL | | 19 | system user | | NULL | Connect | 217230 | Waiting for an event from Coordinator | NULL | | 1519 | checksum | | | Sleep | 1 | | NULL | | 1628 | root | localhost | NULL | Query | 0 | starting | show processlist | +------+-------------+--------------------+-------------------+---------+--------+---------------------------------------+------------------+ 8 rows in set (0.00 sec)
发现id号为16的work线程从2017-11-03 20:53:34卡到现在，查看正在应用的relay log
[root@mysql]# mysqlbinlog -vv --base64-output='decode-rows' --start-position=6517796 relay-log.000014 | more /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/; /*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/; DELIMITER /*!*/; # at 6517796 #171101 23:40:01 server id 102121028 end_log_pos 6517688 CRC32 0xa6c47c8a GTID last_committed=2245 sequence_number=2246 SET @@SESSION.GTID_NEXT= '7eeec54f-6464-11e7-a16a-20f17c59daa2:54494097'/*!*/; # at 6517861 #171101 21:41:33 server id 102121028 end_log_pos 6517773 CRC32 0x5abb9023 Query thread_id=203424280 exec_time=0 error_code=0 SET TIMESTAMP=1509543693/*!*/; SET @@session.pseudo_thread_id=203424280/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/; SET @@session.sql_mode=1344798720/*!*/; SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/; /*!C utf8 *//*!*/; SET @@session.character_set_client=33,@@session.collation_connection=33,@@session.collation_server=33/*!*/; SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; BEGIN /*!*/; # at 6517946 #171101 21:41:33 server id 102121028 end_log_pos 6517867 CRC32 0x474f27f3 Table_map: `nxin_qlw_business`.`wm_midwarehousebalance` mappe d to number 58798 # at 6518040 #171101 21:41:33 server id 102121028 end_log_pos 6517993 CRC32 0x6c1671e8 Write_rows: table id 58798 flags: STMT_END_F ### INSERT INTO .....
......
# at 6518166 #171101 21:41:33 server id 102121028 end_log_pos 6518087 CRC32 0x4b319525 Table_map: `nxin_qlw_business`.`wm_midwarehousebalance` mappe d to number 58798 # at 6518260 #171101 21:41:33 server id 102121028 end_log_pos 6518213 CRC32 0x2342008d Write_rows: table id 58798 flags: STMT_END_F --More--
16号work线程正在执行的事务是批量的insert，线上语句是 insert into ... select ... ，是一个大的事务。这种情况复制会很慢
解决方法是
过滤正在操作的表不再复制
stop slave; # mysql 5.7 新增 在线filter CHANGE REPLICATION FILTER REPLICATE_WILD_IGNORE_TABLE = ('db.t'); start slave
此时再看sql 线程应用正常，待主从无延迟后，恢复db.t表
关闭过滤
stop slave ; CHANGE REPLICATION FILTER REPLICATE_WILD_IGNORE_TABLE = ();
---恢复内容结束---