Uploaded image for project: 'Percona Server for MySQL'
  1. Percona Server for MySQL
  2. PS-7815

RocksDB table crashes during pstress runs myrocks::ha_rocksdb::reset_blob_buffer

Details

    • Bug
    • Status: Done
    • Medium
    • Resolution: Fixed
    • 8.0.x, 8.0.25-15 (Q2 2021), 8.0.26-16 (Q3 2021)
    • 8.0.27-18 (Q4 2021)
    • MyRocks

    Description

      During pstress runs, a crash is seen with RocksDB table. The crash can be easily reproduced with the attached SQL file.

      Below is the stacktrace:

      #0  __pthread_kill (threadid=<optimized out>, signo=11) at ../sysdeps/unix/sysv/linux/pthread_kill.c:56
      #1  0x0000564c4743f74f in my_write_core (sig=11) at /home/mohit.joshi/mysql-8.0/mysys/stacktrace.cc:409
      #2  0x0000564c46126801 in handle_fatal_signal (sig=11) at /home/mohit.joshi/mysql-8.0/sql/signal_handler.cc:194
      #3  <signal handler called>
      #4  0x0000564c4743754d in my_free (ptr=0x1) at /home/mohit.joshi/mysql-8.0/mysys/my_malloc.cc:146
      #5  0x00007f48e394e525 in myrocks::ha_rocksdb::reset_blob_buffer (this=0x7f48c0445458, total_size=15)
          at /home/mohit.joshi/mysql-8.0/storage/rocksdb/ha_rocksdb.cc:15483
      #6  0x00007f48e39a3d36 in myrocks::Rdb_key_def::unpack_record (this=0x7f48c0356310, table=0x7f48c02f31c0, buf=0x7f48c0339fa8 "\377", 
          packed_key=0x7f4994287eb0, unpack_info=0x7f4994287ec0, verify_row_debug_checksums=false)
          at /home/mohit.joshi/mysql-8.0/storage/rocksdb/rdb_datadic.cc:2143
      #7  0x00007f48e3933616 in myrocks::ha_rocksdb::find_icp_matching_index_rec (this=0x7f48c035f388, move_forward=true, 
          buf=0x7f48c0339fa8 "\377") at /home/mohit.joshi/mysql-8.0/storage/rocksdb/ha_rocksdb.cc:8828
      #8  0x00007f48e3931f35 in myrocks::ha_rocksdb::read_row_from_secondary_key (this=0x7f48c035f388, buf=0x7f48c0339fa8 "\377", kd=..., 
          move_forward=true) at /home/mohit.joshi/mysql-8.0/storage/rocksdb/ha_rocksdb.cc:8353
      #9  0x00007f48e3933306 in myrocks::ha_rocksdb::index_read_map_impl (this=0x7f48c035f388, buf=0x7f48c0339fa8 "\377", 
          key=0x7f48b8870818 ">\f", keypart_map=1, find_flag=HA_READ_KEY_OR_NEXT, end_key=0x7f48c0445598)
          at /home/mohit.joshi/mysql-8.0/storage/rocksdb/ha_rocksdb.cc:8755
      #10 0x00007f48e39328d1 in myrocks::ha_rocksdb::read_range_first (this=0x7f48c035f388, start_key=0x7f48c0446cc8, end_key=0x7f48c0445598, 
          eq_range_arg=false, sorted=false) at /home/mohit.joshi/mysql-8.0/storage/rocksdb/ha_rocksdb.cc:8548
      #11 0x0000564c45a2f731 in native_part::Partition_base::read_range_first_in_part (this=0x7f48c0445458, part_id=0, buf=0x0, 
          start_key=0x7f48c0446cc8, end_key=0x7f48c0445598, sorted=false) at /home/mohit.joshi/mysql-8.0/sql/partitioning/partition_base.cc:2721
      #12 0x0000564c45d63cff in Partition_helper::handle_unordered_scan_next_partition (this=0x7f48c0446c80, buf=0x7f48c0339fa8 "\377")
          at /home/mohit.joshi/mysql-8.0/sql/partitioning/partition_handler.cc:2389
      #13 0x0000564c45d62b01 in Partition_helper::common_index_read (this=0x7f48c0446c80, buf=0x7f48c0339fa8 "\377", have_start_key=true)
          at /home/mohit.joshi/mysql-8.0/sql/partitioning/partition_handler.cc:1913
      #14 0x0000564c45d635a3 in Partition_helper::ph_read_range_first (this=0x7f48c0446c80, start_key=0x7f48c0445540, end_key=0x7f48c0445560, 
          eq_range_arg=false, sorted=false) at /home/mohit.joshi/mysql-8.0/sql/partitioning/partition_handler.cc:2211
      #15 0x0000564c45a389c9 in native_part::Partition_base::read_range_first (this=0x7f48c0445458, start_key=0x7f48c0445540, 
          end_key=0x7f48c0445560, eq_range_value=false, sorted=false) at /home/mohit.joshi/mysql-8.0/sql/partitioning/partition_base.h:432
      #16 0x0000564c45a5ceed in handler::multi_range_read_next (this=0x7f48c0445458, range_info=0x7f49942883f8)
          at /home/mohit.joshi/mysql-8.0/sql/handler.cc:6734
      #17 0x0000564c45a5cbcf in handler::ha_multi_range_read_next (this=0x7f48c0445458, range_info=0x7f49942883f8)
          at /home/mohit.joshi/mysql-8.0/sql/handler.cc:6672
      #18 0x0000564c45d05bd5 in QUICK_RANGE_SELECT::get_next (this=0x7f48b7f9e4e0) at /home/mohit.joshi/mysql-8.0/sql/opt_range.cc:10110
      #19 0x0000564c45d7fb69 in IndexRangeScanIterator::Read (this=0x7f48babba530) at /home/mohit.joshi/mysql-8.0/sql/records.cc:311
      #20 0x0000564c460425d5 in Query_expression::ExecuteIteratorQuery (this=0x7f48babb6228, thd=0x7f48c0000be0)
          at /home/mohit.joshi/mysql-8.0/sql/sql_union.cc:1231
      --Type <RET> for more, q to quit, c to continue without paging--
      #21 0x0000564c4604295e in Query_expression::execute (this=0x7f48babb6228, thd=0x7f48c0000be0)
          at /home/mohit.joshi/mysql-8.0/sql/sql_union.cc:1284
      #22 0x0000564c45f838a4 in Sql_cmd_dml::execute_inner (this=0x7f48babb89f8, thd=0x7f48c0000be0)
          at /home/mohit.joshi/mysql-8.0/sql/sql_select.cc:791
      #23 0x0000564c45f82d62 in Sql_cmd_dml::execute (this=0x7f48babb89f8, thd=0x7f48c0000be0) at /home/mohit.joshi/mysql-8.0/sql/sql_select.cc:575
      #24 0x0000564c45efcd58 in mysql_execute_command (thd=0x7f48c0000be0, first_level=true) at /home/mohit.joshi/mysql-8.0/sql/sql_parse.cc:4677
      #25 0x0000564c45efee7a in dispatch_sql_command (thd=0x7f48c0000be0, parser_state=0x7f4994289aa0, update_userstat=false)
          at /home/mohit.joshi/mysql-8.0/sql/sql_parse.cc:5273
      #26 0x0000564c45ef4449 in dispatch_command (thd=0x7f48c0000be0, com_data=0x7f499428ab70, command=COM_QUERY)
          at /home/mohit.joshi/mysql-8.0/sql/sql_parse.cc:1938
      #27 0x0000564c45ef27b9 in do_command (thd=0x7f48c0000be0) at /home/mohit.joshi/mysql-8.0/sql/sql_parse.cc:1386
      #28 0x0000564c4610db78 in handle_connection (arg=0x564c4ef91f90)
          at /home/mohit.joshi/mysql-8.0/sql/conn_handler/connection_handler_per_thread.cc:307
      #29 0x0000564c47ccc8cc in pfs_spawn_thread (arg=0x564c4f25c580) at /home/mohit.joshi/mysql-8.0/storage/perfschema/pfs.cc:2899
      #30 0x00007f49a9d4d609 in start_thread (arg=<optimized out>) at pthread_create.c:477
      #31 0x00007f49a9c74293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

      Testcase:

      CREATE TABLE tt_5_p (ip_col INT, b0 BLOB, i1 INT, b2 BLOB, v3 VARCHAR(20), INDEX tt_5_pi0(v3, b2(9) ASC, ip_col) , INDEX tt_5_pi1(b0(19), ip_col, b2(10)) , INDEX tt_5_pi2(ip_col) , INDEX tt_5_pi3(v3, b0(12), i1 ASC)  ) ROW_FORMAT=COMPRESSED ENGINE=RocksDB PARTITION BY RANGE (ip_col)( PARTITION p0 VALUES LESS THAN (990), PARTITION p1 VALUES LESS THAN (1631), PARTITION p2 VALUES LESS THAN (1770), PARTITION p3 VALUES LESS THAN (1887), PARTITION p4 VALUES LESS THAN (3048), PARTITION p5 VALUES LESS THAN (3270), PARTITION p6 VALUES LESS THAN (3509), PARTITION p7 VALUES LESS THAN (3780), PARTITION p8 VALUES LESS THAN (4474), PARTITION p9 VALUES LESS THAN (4504)) ;
      
      INSERT INTO tt_5_p  ( ip_col ,b0 ,i1 ,b2 ,v3 ) VALUES( 1454, 'oAF1YaaF7BOEiosndZwgXgMevjiIA44YMe73Q1x2RsZ1cJIGrGlHhcsQCotFOUrTGTcvm0GesBrmoRUcAcnbYiaE7f5qGsofah2wxrja6SeEgCjGkx7EeOypzOI8ic', 1315, 'LhwlJOITDV2IIQSCukkrYiHQM9kawBN4whCj7uv53KYlnGz8l3h9SVKeiboRCyTjqhLGfsWWm9V4G4skyLpKPFQzFLXzuDh7dech834anv1z1fmsxjj7nEwZetAAi68GmqjCE2VIpqF997OIqhvXUGUDRrlzxeqbFVyR9i652ASWNODpBY5VIvxn7wsMlw2k1dakSjzFAE12vPG9fcK5DEoFHAXsGdasUXInZphhO41a4Nu8ozHKOIIUJPvHBLw', 'VCxF' ) ;
      
      INSERT INTO tt_5_p  ( ip_col ,b0 ,i1 ,b2 ,v3 ) VALUES( 1177, '9UMKc5HeJRIksuMwDhbPOQ5fw7W1QxckgdxRUzDi32Gm6VLGYdEstzKdmAvFIyO5b9VhH71ZF9YfrRfxfXpL29LMIrP1B1JCn1hYjDuL1yQ3oM2vtg8hDhmbNzw7LYnWCOeBJeEiY7c5e78qXtkSD', 848, 'kwkmem9dzVDHUHlzZBPqVlfDgK7RQ38WnpE1XoS74zcZKoG67Wz7NtAfZ4R8l3UeGMOemH4sFEGNvvn7bj2ACIaTcfLG6AbwhSBFhFkQoEzXE2QiiyLjhIp89crvTLOANaN1WLvFdhZsrA5lt1D3tY3VE7HRjmA94uhzU3apEk3hFTLAl0t', 'kNM5ME2tNkpypd8' ) ;
      
      INSERT INTO tt_5_p  ( ip_col ,b0 ,i1 ,b2 ,v3 ) VALUES( 269, 'ilRiiJS9CmEFEmEyaN', 2204, '72Rn7zh8AOM327cpMJV9ZYGlNwPxIhC2DML7JFptk3Tt4QwvetTRNliPnP', 'OjBR5ldSVEZmM1DU' ) ;
      
      ALTER TABLE tt_5_p DROP INDEX tt_5_pi2, LOCK=EXCLUSIVE, ALGORITHM=DEFAULT ;
      
      SELECT * FROM tt_5_p WHERE v3 >= '' AND v3 <= 'chHZ' ;
      

      Another way to repeat the crash:

      Step 1: Start MySQL server
      Step 2: Enable RocksDB
      ./bin/ps-admin --enable-rocksdb -uroot -S/tmp/mysql_22000.sock
      
      Step 3: Connect to client
      Step 4: source ~/rocksdb_bug.sql;

      Attachments

        Issue Links

          Activity

            People

              rinat.ibragimov Rinat Ibragimov
              mohit.joshi Mohit Joshi
              Votes:
              0 Vote for this issue
              Watchers:
              2 Start watching this issue

              Dates

                Created:
                Updated:
                Resolved:

                Time Tracking

                  Estimated:
                  Original Estimate - Not Specified
                  Not Specified
                  Remaining:
                  Remaining Estimate - Not Specified
                  Not Specified
                  Logged:
                  Time Spent - 1 hour, 15 minutes
                  1h 15m

                  Smart Checklist