PXC node is crashing we enable GTID replication slave and run disable keys from a table on the master host.

git clone https://github.com/Percona-QA/pxc-qa

BASEDIR=/ssd/ramesh/Percona-XtraDB-Cluster_8.0.17.8-27dev.4.3_debug.Linux.x86_64
WORKDIR=/ssd/ramesh

$BASEDIR/bin/mysqld --no-defaults --datadir=$WORKDIR/node1 --initialize-insecure  --basedir=$BASEDIR
$BASEDIR/bin/mysqld --no-defaults  --datadir=$WORKDIR/node2 --initialize-insecure  --basedir=$BASEDIR
$BASEDIR/bin/mysqld --no-defaults --datadir=$WORKDIR/psnode1 --initialize-insecure --basedir=$BASEDIR


$BASEDIR/bin/mysqld --defaults-file=$WORKDIR/conf/ps1.cnf --datadir=$WORKDIR/psnode1 --basedir=$BASEDIR --log-error=$WORKDIR/log/psnode1.err &

$BASEDIR/bin/mysqld --defaults-file=$WORKDIR/conf/node1.cnf --datadir=$WORKDIR/node1 --basedir=$BASEDIR --wsrep-provider=$BASEDIR/lib/libgalera_smm.so --wsrep-new-cluster --log-error=$WORKDIR/log/node1.err &

$BASEDIR/bin/mysqld --defaults-file=$WORKDIR/conf/node2.cnf --datadir=$WORKDIR/node2 --basedir=$BASEDIR --wsrep-provider=$BASEDIR/lib/libgalera_smm.so  --log-error=$WORKDIR/log/node2.err &

$BASEDIR/bin/mysql -uroot -S$WORKDIR/node2/mysql.sock

CHANGE MASTER TO MASTER_HOST='127.0.0.1', MASTER_PORT=21100, MASTER_USER='root', MASTER_AUTO_POSITION=1; START SLAVE;
\q

$BASEDIR/bin/mysql -uroot -S/tmp/psnode1.sock -e'create database test;'


cd $WORKDIR/pxc-qa/randgen/

perl $WORKDIR/pxc-qa/randgen/gendata.pl --dsn=dbi:mysql:host=127.0.0.1:port=21100:user=root:database=test --spec=$WORKDIR/pxc-qa/randgen/conf/galera/galera_stress.zz


node1.cnf

[mysqld]
wsrep-debug=1
wsrep_node_incoming_address=127.0.0.1
wsrep_node_address=127.0.0.1
innodb_file_per_table
innodb_autoinc_lock_mode=2
innodb_flush_log_at_trx_commit=0
wsrep_sst_method=xtrabackup-v2
core-file
log-output=none
wsrep_slave_threads=8
wsrep_cluster_address=gcomm://127.0.0.1:15108,127.0.0.1:15208,127.0.0.1:15308,
log_error_verbosity=3
port=15100
wsrep_provider_options='gmcast.listen_addr=tcp://127.0.0.1:15108;socket.ssl_key=/ssd/ramesh/cert/server-key.pem;socket.ssl_cert=/ssd/ramesh/cert/server-cert.pem;socket.ssl_ca=/ssd/ramesh/cert/ca.pem'
socket = /ssd/ramesh/node1/mysql.sock
server_id=11
gtid_mode=ON
enforce_gtid_consistency=ON
log_slave_updates=ON
log_bin=binlog
binlog_format=ROW
master_info_repository=TABLE
relay_log_info_repository=TABLE
pxc_encrypt_cluster_traffic = OFF

node2.cnf

[mysqld]
wsrep-debug=1
wsrep_node_incoming_address=127.0.0.1
wsrep_node_address=127.0.0.1
innodb_file_per_table
innodb_autoinc_lock_mode=2
innodb_flush_log_at_trx_commit=0
wsrep_sst_method=xtrabackup-v2
core-file
log-output=none
wsrep_slave_threads=8
wsrep_cluster_address=gcomm://127.0.0.1:15108,127.0.0.1:15208,127.0.0.1:15308,
log_error_verbosity=3
port=15200
wsrep_provider_options='gmcast.listen_addr=tcp://127.0.0.1:15208;socket.ssl_key=/ssd/ramesh/cert/server-key.pem;socket.ssl_cert=/ssd/ramesh/cert/server-cert.pem;socket.ssl_ca=/ssd/ramesh/cert/ca.pem'
socket = /ssd/ramesh/node2/mysql.sock
server_id=12
gtid_mode=ON
enforce_gtid_consistency=ON
log_slave_updates=ON
log_bin=binlog
binlog_format=ROW
master_info_repository=TABLE
relay_log_info_repository=TABLE
pxc_encrypt_cluster_traffic = OFF


ps1.cnf

[mysqld]
innodb_file_per_table
innodb_autoinc_lock_mode=2
innodb_flush_log_at_trx_commit=0
core-file
log-output=none
port=21100
log_error_verbosity=3
socket=/tmp/psnode1.sock
server_id=101
gtid_mode=ON
enforce_gtid_consistency=ON
log_slave_updates=ON
log_bin=binlog
binlog_format=ROW
master_info_repository=TABLE
relay_log_info_repository=TABLE

 

#0  __pthread_kill (threadid=<optimized out>, signo=6) at ../sysdeps/unix/sysv/linux/pthread_kill.c:62
#1  0x000000000494c8f3 in my_write_core (sig=6) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/mysys/stacktrace.cc:305
#2  0x000000000379995d in handle_fatal_signal (sig=6) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/signal_handler.cc:181
#3  <signal handler called>
#4  0x00007fc189962428 in __GI_raise ([email protected]=6) at ../sysdeps/unix/sysv/linux/raise.c:54
#5  0x00007fc18996402a in __GI_abort () at abort.c:89
#6  0x0000000004dcf03f in ut_dbg_assertion_failed (expr=0x69800ae "false", file=0x6980010 "/mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/clone/clone0repl.cc", line=163) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/ut/ut0dbg.cc:102
#7  0x0000000004efbeee in Clone_persist_gtid::check_gtid_prepare (this=0x7fc14c2ea7a0, thd=0x7fc0800008e0, trx=0x7fc1688b42f0, found_gtid=true, [email protected]: false) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/clone/clone0repl.cc:163
#8  0x0000000004efbd54 in Clone_persist_gtid::trx_check_set (this=0x7fc14c2ea7a0, trx=0x7fc1688b42f0, prepare=true, rollback=false) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/clone/clone0repl.cc:120
#9  0x0000000004dc9468 in trx_undo_gtid_add_update_undo (trx=0x7fc1688b42f0, prepare=true, rollback=false) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/trx/trx0undo.cc:599
#10 0x0000000004db8644 in trx_prepare_for_mysql (trx=0x7fc1688b42f0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/trx/trx0trx.cc:3055
#11 0x0000000004a7ca6c in innobase_xa_prepare (hton=0x988e7b0, thd=0x7fc0800008e0, prepare_trx=true) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/innobase/handler/ha_innodb.cc:21696
#12 0x00000000031588e2 in ha_prepare_low (thd=0x7fc0800008e0, all=true) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/handler.cc:2422
#13 0x0000000004596418 in MYSQL_BIN_LOG::prepare (this=0x7d26d20 <mysql_bin_log>, thd=0x7fc0800008e0, all=true) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/binlog.cc:8354
#14 0x0000000003156613 in ha_commit_trans (thd=0x7fc0800008e0, all=true, ignore_global_read_lock=false) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/handler.cc:1795
#15 0x00000000037255da in trans_commit_implicit (thd=0x7fc0800008e0, ignore_global_read_lock=false) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/transaction.cc:369
#16 0x0000000003663a95 in simple_rename_or_index_change (thd=0x7fc0800008e0, new_schema=..., table_list=0x7fc080013490, keys_onoff=Alter_info::DISABLE, alter_ctx=0x7fc1501ccdd0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/sql_table.cc:15101
#17 0x0000000003667ae9 in mysql_alter_table (thd=0x7fc0800008e0, new_db=0x7fc080013a38 "test", new_name=0x0, create_info=0x7fc1501ce220, table_list=0x7fc080013490, alter_info=0x7fc1501ce330) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/sql_table.cc:16196
#18 0x00000000034984a3 in Sql_cmd_alter_table::execute (this=0x7fc080013a40, thd=0x7fc0800008e0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/sql_alter.cc:574
#19 0x0000000003595272 in mysql_execute_command (thd=0x7fc0800008e0, first_level=true) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/sql_parse.cc:5313
#20 0x00000000035984a8 in mysql_parse (thd=0x7fc0800008e0, parser_state=0x7fc1501d07a0, update_userstat=true) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/sql_parse.cc:6188
#21 0x0000000004550e91 in Query_log_event::do_apply_event (this=0x7fc08000e750, rli=0x7fc08000a2f0, query_arg=0x7fc08000e4c9 "ALTER TABLE `table0_innodb_int` DISABLE KEYS /* QNO 6 CON_ID 13 */", q_len_arg=66) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/log_event.cc:4738
#22 0x000000000454f5e7 in Query_log_event::do_apply_event (this=0x7fc08000e750, rli=0x7fc08000a2f0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/log_event.cc:4395
#23 0x000000000454b040 in Log_event::apply_event (this=0x7fc08000e750, rli=0x7fc08000a2f0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/log_event.cc:3188
#24 0x0000000003a54e05 in wsrep_apply_events (thd=0x7fc0800008e0, rli=0x7fc08000a2f0, events_buf=0x7fc16ef2dfb8, buf_len=0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/wsrep_applier.cc:201
#25 0x00000000037b082b in Wsrep_high_priority_service::apply_toi (this=0x7fc1501d2c90, ws_meta=..., data=...) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/wsrep_high_priority_service.cc:409
#26 0x0000000005499284 in apply_toi (provider=..., high_priority_service=..., ws_handle=..., ws_meta=..., data=...) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/wsrep-lib/src/server_state.cpp:461
#27 0x000000000549c438 in wsrep::server_state::on_apply (this=0x97b48a0, high_priority_service=..., ws_handle=..., ws_meta=..., data=...) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/wsrep-lib/src/server_state.cpp:1110
#28 0x00000000054b2ac3 in wsrep::high_priority_service::apply (this=0x7fc1501d2c90, ws_handle=..., ws_meta=..., data=...) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/wsrep-lib/include/wsrep/high_priority_service.hpp:46
#29 0x00000000054afd3c in (anonymous namespace)::apply_cb (ctx=0x7fc1501d2c90, wsh=0x7fc1501d17c0, flags=69, buf=0x7fc1501d17e0, meta=0x7fc1501d1b70, exit_loop=0x7fc1501d1a2d) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/wsrep-lib/src/wsrep_provider_v26.cpp:500
#30 0x00007fc1770da101 in galera::TrxHandleSlave::apply (this=0x7fc08000e1c0, recv_ctx=0x7fc1501d2c90, apply_cb=0x54afb0b <(anonymous namespace)::apply_cb(void*, wsrep_ws_handle_t const*, uint32_t, wsrep_buf_t const*, wsrep_trx_meta_t const*, wsrep_bool_t*)>, meta=..., [email protected]: false) at galera/src/trx_handle.cpp:418
#31 0x00007fc17712874b in galera::ReplicatorSMM::apply_trx (this=0x97e6080, recv_ctx=0x7fc1501d2c90, ts=...) at galera/src/replicator_smm.cpp:585
#32 0x00007fc177132d51 in galera::ReplicatorSMM::process_trx (this=0x97e6080, recv_ctx=0x7fc1501d2c90, ts_ptr=...) at galera/src/replicator_smm.cpp:2269
#33 0x00007fc1771072ad in galera::GcsActionSource::process_writeset (this=0x97f2ed0, recv_ctx=0x7fc1501d2c90, act=..., [email protected]: false) at galera/src/gcs_action_source.cpp:62
#34 0x00007fc17710795c in galera::GcsActionSource::dispatch (this=0x97f2ed0, recv_ctx=0x7fc1501d2c90, act=..., [email protected]: false) at galera/src/gcs_action_source.cpp:109
#35 0x00007fc1771080c0 in galera::GcsActionSource::process (this=0x97f2ed0, recv_ctx=0x7fc1501d2c90, [email protected]: false) at galera/src/gcs_action_source.cpp:182
#36 0x00007fc177127b20 in galera::ReplicatorSMM::async_recv (this=0x97e6080, recv_ctx=0x7fc1501d2c90) at galera/src/replicator_smm.cpp:460
#37 0x00007fc17715aab6 in galera_recv (gh=0x97b6b10, recv_ctx=0x7fc1501d2c90) at galera/src/wsrep_provider.cpp:236
#38 0x00000000054b0aee in wsrep::wsrep_provider_v26::run_applier (this=0x97b6af0, applier_ctx=0x7fc1501d2c90) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/wsrep-lib/src/wsrep_provider_v26.cpp:715
#39 0x00000000037de3ad in wsrep_replication_process (thd=0x7fc0800008e0, arg=0x0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/wsrep_thd.cc:59
#40 0x000000000338063d in start_wsrep_THD (arg=0x9a2d510) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/sql/mysqld.cc:9157
#41 0x000000000512ea71 in pfs_spawn_thread (arg=0x98911d0) at /mnt/jenkins/workspace/pxc80-build/BUILD_TYPE/debug/Host/min-xenial-x64/storage/perfschema/pfs.cc:2855
#42 0x00007fc18b6876ba in start_thread (arg=0x7fc1501d3700) at pthread_create.c:333
#43 0x00007fc189a3441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109