The problem was apparently introduced by this change:

commit 016790403a4bb6182094870870ce1a1c3e2756dc Author: Sergei Petrunia <psergey@askmonty.org> Date: Tue May 31 17:59:04 2016 +0300 MDEV-9764: MariaDB does not limit memory used for range optimization ...

Stack trace from 10.1 commit ccdd63388a789f407d502848d12c618593a5ded1 debug build mysqld: /data/src/10.1/sql/mem_root_array.h:72: Element_type& Mem_root_array<Element_type, has_trivial_destructor>::at(size_t) [with Element_type = SEL_ARG*; bool has_trivial_destructor = true; size_t = long unsigned int]: Assertion `n < size()' failed. 160702 14:30:03 [ERROR] mysqld got signal 6 ; #6 0x00007f742b4e8266 in __assert_fail_base (fmt=0x7f742b621238 "%s%s%s:%u: %s%sAssertion `%s' failed.

%n", assertion=assertion@entry=0x7f742e6e5dac "n < size()", file=file@entry=0x7f742e6e5d88 "/data/src/10.1/sql/mem_root_array.h", line=line@entry=72, function=function@entry=0x7f742e6e5ec0 <Mem_root_array<SEL_ARG*, true>::at(unsigned long)::__PRETTY_FUNCTION__> "Element_type& Mem_root_array<Element_type, has_trivial_destructor>::at(size_t) [with Element_type = SEL_ARG*; bool has_trivial_destructor = true; size_t = long unsigned int]") at assert.c:92 #7 0x00007f742b4e8312 in __GI___assert_fail (assertion=0x7f742e6e5dac "n < size()", file=0x7f742e6e5d88 "/data/src/10.1/sql/mem_root_array.h", line=72, function=0x7f742e6e5ec0 <Mem_root_array<SEL_ARG*, true>::at(unsigned long)::__PRETTY_FUNCTION__> "Element_type& Mem_root_array<Element_type, has_trivial_destructor>::at(size_t) [with Element_type = SEL_ARG*; bool has_trivial_destructor = true; size_t = long unsigned int]") at assert.c:101 #8 0x00007f742e063e27 in Mem_root_array<SEL_ARG*, true>::at (this=0x7f7422454108, n=1) at /data/src/10.1/sql/mem_root_array.h:72 #9 0x00007f742e063805 in Mem_root_array<SEL_ARG*, true>::operator[] (this=0x7f7422454108, n=1) at /data/src/10.1/sql/mem_root_array.h:82 #10 0x00007f742e05d1d8 in get_index_range_tree (index=1, range_tree=0x7f7422454100, param=0x7f742ec457d0, param_idx=0x7f742ec43e00) at /data/src/10.1/sql/opt_range.cc:13057 #11 0x00007f742e05ba28 in get_best_group_min_max (param=0x7f742ec457d0, tree=0x7f7422454100, read_time=1.428462998102467) at /data/src/10.1/sql/opt_range.cc:12404 #12 0x00007f742e0453fc in SQL_SELECT::test_quick_select (this=0x7f74224e65e0, thd=0x7f74247fa070, keys_to_use=..., prev_tables=4611686018427387904, limit=1, force_quick_range=false, ordered_output=false, remove_false_parts_of_where=false) at /data/src/10.1/sql/opt_range.cc:2572 #13 0x00007f742dd44cf4 in make_join_select (join=0x7f74224e4280, select=0x7f74224e6408, cond=0x7f74224e36d8) at /data/src/10.1/sql/sql_select.cc:9917 #14 0x00007f742dd2cd38 in JOIN::optimize_inner (this=0x7f74224e4280) at /data/src/10.1/sql/sql_select.cc:1572 #15 0x00007f742dd2aeaa in JOIN::optimize (this=0x7f74224e4280) at /data/src/10.1/sql/sql_select.cc:1036 #16 0x00007f742dce1aad in st_select_lex::optimize_unflattened_subqueries (this=0x7f74247fe188, const_only=true) at /data/src/10.1/sql/sql_lex.cc:3763 #17 0x00007f742de78a17 in JOIN::optimize_constant_subqueries (this=0x7f74224e3bd0) at /data/src/10.1/sql/opt_subselect.cc:5085 #18 0x00007f742dd2b2f1 in JOIN::optimize_inner (this=0x7f74224e3bd0) at /data/src/10.1/sql/sql_select.cc:1146 #19 0x00007f742dd2aeaa in JOIN::optimize (this=0x7f74224e3bd0) at /data/src/10.1/sql/sql_select.cc:1036 #20 0x00007f742dd3351e in mysql_select (thd=0x7f74247fa070, rref_pointer_array=0x7f74247fe400, tables=0x0, wild_num=0, fields=..., conds=0x0, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2147748608, result=0x7f74224e3bb0, unit=0x7f74247fda88, select_lex=0x7f74247fe188) at /data/src/10.1/sql/sql_select.cc:3437 #21 0x00007f742dd29080 in handle_select (thd=0x7f74247fa070, lex=0x7f74247fd9c0, result=0x7f74224e3bb0, setup_tables_done_option=0) at /data/src/10.1/sql/sql_select.cc:384 #22 0x00007f742dcf941c in execute_sqlcom_select (thd=0x7f74247fa070, all_tables=0x7f74224e2f48) at /data/src/10.1/sql/sql_parse.cc:5894 #23 0x00007f742dcef329 in mysql_execute_command (thd=0x7f74247fa070) at /data/src/10.1/sql/sql_parse.cc:2960 #24 0x00007f742dcfcb54 in mysql_parse (thd=0x7f74247fa070, rawbuf=0x7f74224e2088 "SELECT 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 )", length=65, parser_state=0x7f742ec475e0) at /data/src/10.1/sql/sql_parse.cc:7314 #25 0x00007f742dceb57e in dispatch_command (command=COM_QUERY, thd=0x7f74247fa070, packet=0x7f742613e071 "SELECT 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 )", packet_length=65) at /data/src/10.1/sql/sql_parse.cc:1486 #26 0x00007f742dcea2b5 in do_command (thd=0x7f74247fa070) at /data/src/10.1/sql/sql_parse.cc:1107 #27 0x00007f742de1fddd in do_handle_one_connection (thd_arg=0x7f74247fa070) at /data/src/10.1/sql/sql_connect.cc:1350 #28 0x00007f742de1fb41 in handle_one_connection (arg=0x7f74247fa070) at /data/src/10.1/sql/sql_connect.cc:1262 #29 0x00007f742e101336 in pfs_spawn_thread (arg=0x7f742ac27ef0) at /data/src/10.1/storage/perfschema/pfs.cc:1860 #30 0x00007f742d3ea0a4 in start_thread (arg=0x7f742ec48b00) at pthread_create.c:309

Stack trace from 10.1.15 release build #2 <signal handler called> #3 get_sel_arg_for_keypart (cur_range=0x7f5f54cb6440, keypart_tree=0x4, field=0x7f5f49041058) at /home/buildbot/buildbot/build/sql/opt_range.cc:12843 #4 get_constant_key_infix (first_non_infix_part=<synthetic pointer>, key_infix_len=<synthetic pointer>, key_infix=0x7f5f54cb7910 "", last_part=0x7f5f490413a0, min_max_arg_part=0x0, first_non_group_part=0x7f5f49041380, index_range_tree=0x4, index_info=<optimized out>, thd=<optimized out>) at /home/buildbot/buildbot/build/sql/opt_range.cc:12935 #5 get_best_group_min_max (read_time=1.428462998102467, tree=0x7f5f4933c098, param=0x7f5f54cb65c0) at /home/buildbot/buildbot/build/sql/opt_range.cc:12409 #6 SQL_SELECT::test_quick_select (this=this@entry=0x7f5f49339478, thd=thd@entry=0x7f5f4dbfa008, keys_to_use=..., prev_tables=4611686018427387905, prev_tables@entry=4611686018427387904, limit=<optimized out>, force_quick_range=force_quick_range@entry=false, ordered_output=ordered_output@entry=false, remove_false_parts_of_where=remove_false_parts_of_where@entry=false) at /home/buildbot/buildbot/build/sql/opt_range.cc:2572 #7 0x00007f5f53edd952 in make_join_select (join=join@entry=0x7f5f49337218, select=0x7f5f49339320, cond=0x7f5f49336670) at /home/buildbot/buildbot/build/sql/sql_select.cc:9917 #8 0x00007f5f53fe5554 in JOIN::optimize_inner (this=this@entry=0x7f5f49337218) at /home/buildbot/buildbot/build/sql/sql_select.cc:1572 #9 0x00007f5f53fe7510 in JOIN::optimize (this=this@entry=0x7f5f49337218) at /home/buildbot/buildbot/build/sql/sql_select.cc:1036 #10 0x00007f5f53f8be38 in st_select_lex::optimize_unflattened_subqueries (this=0x7f5f4dbfdfe0, const_only=const_only@entry=true) at /home/buildbot/buildbot/build/sql/sql_lex.cc:3763 #11 0x00007f5f5408f44a in JOIN::optimize_constant_subqueries (this=this@entry=0x7f5f49336b68) at /home/buildbot/buildbot/build/sql/opt_subselect.cc:5085 #12 0x00007f5f53fe4798 in JOIN::optimize_inner (this=this@entry=0x7f5f49336b68) at /home/buildbot/buildbot/build/sql/sql_select.cc:1146 #13 0x00007f5f53fe7510 in JOIN::optimize (this=this@entry=0x7f5f49336b68) at /home/buildbot/buildbot/build/sql/sql_select.cc:1036 #14 0x00007f5f53fe766d in mysql_select (thd=thd@entry=0x7f5f4dbfa008, rref_pointer_array=rref_pointer_array@entry=0x7f5f4dbfe258, tables=<optimized out>, wild_num=<optimized out>, fields=..., conds=<optimized out>, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2147748608, result=result@entry=0x7f5f49336b48, unit=unit@entry=0x7f5f4dbfd8e0, select_lex=select_lex@entry=0x7f5f4dbfdfe0) at /home/buildbot/buildbot/build/sql/sql_select.cc:3437 #15 0x00007f5f53feafed in handle_select (thd=thd@entry=0x7f5f4dbfa008, lex=lex@entry=0x7f5f4dbfd818, result=result@entry=0x7f5f49336b48, setup_tables_done_option=setup_tables_done_option@entry=0) at /home/buildbot/buildbot/build/sql/sql_select.cc:384 #16 0x00007f5f53f8d2c2 in execute_sqlcom_select (thd=thd@entry=0x7f5f4dbfa008, all_tables=0x7f5f49335ee0) at /home/buildbot/buildbot/build/sql/sql_parse.cc:5894 #17 0x00007f5f53f9955e in mysql_execute_command (thd=thd@entry=0x7f5f4dbfa008) at /home/buildbot/buildbot/build/sql/sql_parse.cc:2960 #18 0x00007f5f53f9caad in mysql_parse (thd=0x7f5f4dbfa008, rawbuf=<optimized out>, length=<optimized out>, parser_state=0x7f5f54cba600) at /home/buildbot/buildbot/build/sql/sql_parse.cc:7314 #19 0x00007f5f53f9f6fb in dispatch_command (command=command@entry=COM_QUERY, thd=thd@entry=0x7f5f4dbfa008, packet=0x7f5f4cfcf009 "SELECT 1 IN ( SELECT COUNT( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 )", packet_length=1228099616, packet_length@entry=65) at /home/buildbot/buildbot/build/sql/sql_parse.cc:1486 #20 0x00007f5f53f9fcab in do_command (thd=0x7f5f4dbfa008) at /home/buildbot/buildbot/build/sql/sql_parse.cc:1107 #21 0x00007f5f5405bacf in do_handle_one_connection (thd_arg=thd_arg@entry=0x7f5f4dbfa008) at /home/buildbot/buildbot/build/sql/sql_connect.cc:1350 #22 0x00007f5f5405bc27 in handle_one_connection (arg=arg@entry=0x7f5f4dbfa008) at /home/buildbot/buildbot/build/sql/sql_connect.cc:1262 #23 0x00007f5f5449769d in pfs_spawn_thread (arg=0x7f5f5182ca08) at /home/buildbot/buildbot/build/storage/perfschema/pfs.cc:1860 #24 0x00007f5f52ec70a4 in start_thread (arg=0x7f5f54cbbb00) at pthread_create.c:309 #25 0x00007f5f525f087d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Test case CREATE TABLE t1 (f1 INT NOT NULL , f2 VARCHAR (3) NOT NULL , KEY (f1), KEY (f2, f1)); INSERT INTO t1 VALUES (0, 'foo' ),(1, 'bar' ); SELECT 1 IN ( SELECT COUNT ( DISTINCT f2 ) FROM t1 WHERE f1 <= 4 );

No crash on MySQL 5.7 as far as I can tell.