Skip to content

Commit

Permalink
[thread_pool] Call thd_wait_* callbacks for admitting new query in th…
Browse files Browse the repository at this point in the history
…read pool plugin

Summary:
1. Change the code to admit new query through `thd_wait_begin/end` callbacks using new `THD_WAIT_ADMIT` wait type.
2. Move yield counting to the callbacks to unify the code between the server and the plugin. `yield_cond` exposes the yield condition to check once the yield counter says that yield is needed.
3. Expose all wait events through `admission_control_wait_events` var. This is especially needed for `USER_LOCK` which still controls innodb thread concurrency yield.
4. Copy and fix up admission control tests to the thread_pool suite. Only the var and table names are changed to refer to thread_pool plugin. The results have been diffed with original tests to make sure they are identical (except for var/table names).

Reviewed By: lth

Differential Revision: D27593084

---------------------------------------------------------------------------------------------

use lambda instead of std::bind (facebook#1243)

Summary:
std::bind(yield_condition, table) will generate a functor which size is larger than std::function's local buf, thus std::function needs to new/delete memory to store the functor.

This PR use the lambda which just capture one pointer(table), which size can fit into std::function's local buf thus new/delete is not needed.

Pull Request resolved: facebook#1243

Reviewed By: lth

Differential Revision: D40858532

Pulled By: hermanlee

---------------------------------------------------------------------------------------------

Add yields to hash join iterator

Summary: Queries performing joins are lacking yields and monopolizing thread pool scheduler.

Differential Revision: D39948376

------------------------------------------------------------------------------

Add yield to union iterator

Summary:
This was detected by the stall detection (8.0.23 based callstack):
```
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431] Scheduler 12 stalled by worker 0x7f6367d5d640
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431] Stack trace for 1 thread(s) [1422783 mysqld]:
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 00000000001204d0 ppoll
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000289d67f vio_io_wait(Vio*, enum_vio_io_event, timeout_t)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000002952296 vio_socket_io_wait(Vio*, enum_vio_io_event)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000002385fb2 net_write_packet(NET*, unsigned char const*, unsigned long)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 00000000025b2c44 SELECT_LEX_UNIT::ExecuteIteratorQuery(THD*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000258d2db Sql_cmd_dml::execute(THD*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000254f179 mysql_execute_command(THD*, bool, unsigned long long*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000002545936 dispatch_sql_command(THD*, Parser_state*, unsigned long long*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000269d931 dispatch_command(THD*, COM_DATA const*, enum_server_command)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000269c3e0 do_command(THD*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000005c4ffa7 mysql::thread_pool::TpConnHandler::processEvent(void*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000005c5395f mysql::thread_pool::TpTask::execute()
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000005c5c299 mysql::thread_pool::TpWorkerPool::processWorker(void*)
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 0000000004867855 pfs_spawn_thread(void*) [clone .llvm.11916604799980067416]
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000009ac0e start_thread
W0205 14:38:40.101111 2319640 TpScheduler.cpp:431]     @ 000000000012d1db __clone3
```

Cleaned up existing yield code to use a common default yield condition.

Reviewed By: preritj24

Differential Revision: D43071441
  • Loading branch information
george-reynya authored and inikep committed May 21, 2024
1 parent 256b418 commit 0291698
Show file tree
Hide file tree
Showing 38 changed files with 2,237 additions and 61 deletions.
13 changes: 9 additions & 4 deletions include/mysql/service_thd_wait.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
class THD;
#define MYSQL_THD THD *

/*
/**
One should only report wait events that could potentially block for a
long time. A mutex wait is too short of an event to report. The reason
is that an event which is reported leads to a new thread starts
Expand All @@ -69,9 +69,13 @@ class THD;
holds true for global read locks, table locks and other meta data locks.
Another event of interest is going to sleep for an extended time.
Note that user-level locks no longer use THD_WAIT_USER_LOCK wait type.
@note User-level locks no longer use THD_WAIT_USER_LOCK wait type.
Since their implementation relies on metadata locks manager it uses
THD_WAIT_META_DATA_LOCK instead.
@note THD_WAIT_ADMIT is a fake wait type communicating that query has been
parsed and is ready for execution. The query attributes and sql command
are available at this point.
*/
typedef enum _thd_wait_type_e {
THD_WAIT_SLEEP = 1,
Expand All @@ -80,14 +84,15 @@ typedef enum _thd_wait_type_e {
THD_WAIT_GLOBAL_LOCK = 4,
THD_WAIT_META_DATA_LOCK = 5,
THD_WAIT_TABLE_LOCK = 6,
THD_WAIT_USER_LOCK = 7,
THD_WAIT_INNODB_CONC = 7,
THD_WAIT_BINLOG = 8,
THD_WAIT_GROUP_COMMIT = 9,
THD_WAIT_SYNC = 10,
THD_WAIT_FOR_HLC = 11,
THD_WAIT_NET_IO = 12,
THD_WAIT_YIELD = 13,
THD_WAIT_LAST = 14
THD_WAIT_ADMIT = 14,
THD_WAIT_LAST = 15
} thd_wait_type;

extern "C" struct thd_wait_service_st {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,17 @@ SET GLOBAL admission_control_wait_events='YIELD';
SELECT @@global.admission_control_wait_events;
@@global.admission_control_wait_events
YIELD
SET GLOBAL admission_control_wait_events='SLEEP,NET_IO,META_DATA_LOCK,ROW_LOCK,YIELD';
SET GLOBAL admission_control_wait_events='SLEEP,ROW_LOCK,META_DATA_LOCK,INNODB_CONC,NET_IO,YIELD';
SELECT @@global.admission_control_wait_events;
@@global.admission_control_wait_events
SLEEP,ROW_LOCK,META_DATA_LOCK,NET_IO,YIELD
SLEEP,ROW_LOCK,META_DATA_LOCK,INNODB_CONC,NET_IO,YIELD
SET GLOBAL admission_control_wait_events='SLEEP,SLEEP';
SELECT @@global.admission_control_wait_events;
@@global.admission_control_wait_events
SLEEP
SET GLOBAL admission_control_wait_events='YIELD,NONEXISTING_BIT';
ERROR 42000: Variable 'admission_control_wait_events' can't be set to the value of 'NONEXISTING_BIT'
SELECT @@global.admission_control_wait_events;
@@global.admission_control_wait_events
SLEEP,ROW_LOCK,META_DATA_LOCK,NET_IO,YIELD
SLEEP
set global admission_control_wait_events = @saved_admission_control_wait_events;
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ SELECT @@global.admission_control_wait_events;
SET GLOBAL admission_control_wait_events='YIELD';
SELECT @@global.admission_control_wait_events;

SET GLOBAL admission_control_wait_events='SLEEP,NET_IO,META_DATA_LOCK,ROW_LOCK,YIELD';
SET GLOBAL admission_control_wait_events='SLEEP,ROW_LOCK,META_DATA_LOCK,INNODB_CONC,NET_IO,YIELD';
SELECT @@global.admission_control_wait_events;

# check that repeating the same value is OK
SET GLOBAL admission_control_wait_events='SLEEP,SLEEP';
SELECT @@global.admission_control_wait_events;

# checking that setting variable to a non existing value raises error
Expand Down
160 changes: 160 additions & 0 deletions mysql-test/suite/thread_pool/r/admission_control.result
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
create database test_db;
create database test_db2;
create user 'test_user'@'localhost';
grant all on test_db.* to 'test_user'@'localhost';
grant all on test_db2.* to 'test_user'@'localhost';
grant all on test.* to 'test_user'@'localhost';
use test_db2;
use test_db;
set @start_max_running_queries= @@global.thread_pool_max_running_queries;
set @start_max_waiting_queries= @@global.thread_pool_max_waiting_queries;
set @@global.thread_pool_max_running_queries=10;
set @@global.thread_pool_max_waiting_queries=5;
create table t1(a int) engine=InnoDB;
lock table t1 write;
Threads waiting for admission will have appropriate state set in processlist.
Super user is exempted from admission control checks.
select * from t1;
a
set @@global.thread_pool_admission_control_filter = 'USE';
select @@global.thread_pool_admission_control_filter;
@@global.thread_pool_admission_control_filter
USE
Maximum waiting queries reached. So this would hit an error.
use test_db;
select * from t1||||
ERROR HY000: Maximum waiting queries 5 reached for database `test_db`
Maximum waiting queries reached. So this would hit an error.
use test_db2;
create table t1_test(aaa int);
insert into t1_test values (1);
select aaa from t1_test;
drop table t1_test;
use test_db;
select * from t1||||
aaa
1
ERROR HY000: Maximum waiting queries 5 reached for database `test_db`
use test_db;
select * from t1;
ERROR HY000: Maximum waiting queries 5 reached for database `test_db`
set @@global.thread_pool_admission_control_filter = '';
select @@global.thread_pool_admission_control_filter;
@@global.thread_pool_admission_control_filter

Check status variables
aborted_queries = 3
running_queries = 10
waiting_queries = 5
select * from information_schema.tp_admission_control_entities where schema_name like 'test_db%' order by schema_name;
SCHEMA_NAME WAITING_QUERIES RUNNING_QUERIES ABORTED_QUERIES TIMEOUT_QUERIES CONNECTIONS REJECTED_CONNECTIONS
test_db 5 10 3 0 15 0
test_db2 0 0 0 0 0 0
Filled up queues on one db doesn't affect queries on other db.
use test_db2;
select * from information_schema.tp_admission_control_entities where schema_name like 'test_db%' order by schema_name;
SCHEMA_NAME WAITING_QUERIES RUNNING_QUERIES ABORTED_QUERIES TIMEOUT_QUERIES CONNECTIONS REJECTED_CONNECTIONS
test_db 5 10 3 0 15 0
test_db2 0 1 0 0 1 0
set @@global.thread_pool_max_waiting_queries=6;
Kill a thread that is waiting for admission.
select count(*) from t1;
kill ID;
use test_db;
unlock tables;
Verify the waiting queries received wakeup signal.
select count(*) from t1;
count(*)
15
set @save_admission_control_by_trx = @@global.thread_pool_admission_control_by_trx;
select @save_admission_control_by_trx;
@save_admission_control_by_trx
0
set @@global.thread_pool_max_running_queries=5;
set @@global.thread_pool_max_waiting_queries=10;
# By default, open transaction has no effect on running queries
select count(*) from t1;
count(*)
15
# Test: open transactions will take slots in running queries,
# and will not be blocked
set @@global.thread_pool_admission_control_filter = 'BEGIN,COMMIT,ROLLBACK';
select @@global.thread_pool_admission_control_filter;
@@global.thread_pool_admission_control_filter
BEGIN,COMMIT,ROLLBACK
set @@global.thread_pool_admission_control_by_trx = true;
SELECT @@global.thread_pool_admission_control_by_trx;
@@global.thread_pool_admission_control_by_trx
1
Open transaction is able to continue running queries
connection con_max_wait;
New queries will be rejected (waiting queue is full)
select * from t1;
ERROR HY000: Maximum waiting queries 10 reached for database `test_db`
New transactions will be rejected (waiting queue is full)
begin;
select * from t1;
ERROR HY000: Maximum waiting queries 10 reached for database `test_db`
aborted_queries will increase by 2
Committing a transaction will free up the running query slots
The waiting queries will be unblocked
Check status variables
include/assert.inc [DB Admission control waiting queries should be zero]
include/assert.inc [DB Admission control running queries should be zero]
include/assert.inc [DB Admission control aborted queries should be five]
select * from information_schema.tp_admission_control_entities where schema_name like 'test_db%' order by schema_name;
SCHEMA_NAME WAITING_QUERIES RUNNING_QUERIES ABORTED_QUERIES TIMEOUT_QUERIES CONNECTIONS REJECTED_CONNECTIONS
test_db 0 0 5 0 13 0
test_db2 0 0 0 0 0 0
set @@global.thread_pool_admission_control_by_trx = @save_admission_control_by_trx;
select @@global.thread_pool_admission_control_by_trx;
@@global.thread_pool_admission_control_by_trx
0
set @@global.thread_pool_admission_control_filter = '';
select @@global.thread_pool_admission_control_filter;
@@global.thread_pool_admission_control_filter

# End of open transaction test
# Test admission_control_queue_timeout
use test_db;
set @@global.thread_pool_max_running_queries=1;
set @@global.thread_pool_max_waiting_queries=5;
set @save_admission_control_filter = @@global.thread_pool_admission_control_filter;
set @save_admission_control_queue_timeout = @@global.thread_pool_admission_control_queue_timeout;
set @@global.thread_pool_admission_control_queue_timeout = 100;
set @@global.thread_pool_admission_control_filter = 'BEGIN,COMMIT,ROLLBACK';
create table t2(a int primary key) engine=InnoDB;
begin;
insert into t2 values (1);
begin;
insert into t2 values (1);
insert into t2 values (2);
ERROR HY000: Got timeout while waiting on admission control queue for database `test_db`
rollback;
rollback;
drop table t2;
set @@global.thread_pool_admission_control_filter = @save_admission_control_filter;
set @@global.thread_pool_admission_control_queue_timeout = @save_admission_control_queue_timeout;
timeout_queries should be 1
timeout_queries = 1
waiting_queries should be 0
waiting_queries = 0
select * from information_schema.tp_admission_control_entities where schema_name like 'test_db%' order by schema_name;
SCHEMA_NAME WAITING_QUERIES RUNNING_QUERIES ABORTED_QUERIES TIMEOUT_QUERIES CONNECTIONS REJECTED_CONNECTIONS
test_db 0 0 5 1 15 0
test_db2 0 0 0 0 0 0
reset global.thread_pool_max_running_queries and global.thread_pool_max_waiting_queries
set @@global.thread_pool_max_running_queries=10;
set @@global.thread_pool_max_waiting_queries=5;
Run parallel load and drop the database.
set @@global.thread_pool_max_waiting_queries=0;
Cleanup.
Verify there are no waiting threads.
select count(*) from information_schema.processlist where state='waiting for admission';
count(*)
0
select * from information_schema.tp_admission_control_entities where schema_name like 'test_db%' order by schema_name;
SCHEMA_NAME WAITING_QUERIES RUNNING_QUERIES ABORTED_QUERIES TIMEOUT_QUERIES CONNECTIONS REJECTED_CONNECTIONS
set @@global.thread_pool_max_running_queries=@start_max_running_queries;
set @@global.thread_pool_max_waiting_queries=@start_max_waiting_queries;
drop user test_user@localhost;
93 changes: 93 additions & 0 deletions mysql-test/suite/thread_pool/r/admission_control_hang.result
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
create database test_db;
create user test_user@localhost;
grant all on test_db.* to test_user@localhost;
set @start_max_running_queries = @@global.max_running_queries;
set @@global.max_running_queries = 4;
set @start_innodb_lock_wait_timeout = @@global.innodb_lock_wait_timeout;
set @@global.innodb_lock_wait_timeout = 10000;
set @start_admission_control_filter = @@global.admission_control_filter;
set @@global.admission_control_filter = 'COMMIT';
create table t1 (a int) engine=innodb;
insert into t1 values(1);
begin;
update t1 set a=2 where a=1;
update t1 set a=2 where a=1;
update t1 set a=2 where a=1;
update t1 set a=2 where a=1;
update t1 set a=2 where a=1;
set @@global.admission_control_filter = 'USE';
select @@global.admission_control_filter;
@@global.admission_control_filter
USE
use test;
use test_db;
set @@global.admission_control_filter = 'ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,TRUNCATE,UPDATE,SHOW,ROLLBACK';
select @@global.admission_control_filter;
@@global.admission_control_filter
ALTER,BEGIN,COMMIT,CREATE,DELETE,DROP,INSERT,LOAD,SELECT,SET,REPLACE,ROLLBACK,TRUNCATE,UPDATE,SHOW
create table t2(a int) engine=innodb;
begin;
insert into t2 values(1);
update t2 set a=2 where a=1;
commit;
SHOW TABLES LIKE 't2';
Tables_in_test_db (t2)
t2
begin;
alter table t2 rename t3;
select * from t3;
a
2
delete from t3;
set @val = 1;
truncate table t3;
rollback;
drop table t3;
set @save_admission_control_by_trx = @@global.admission_control_by_trx;
select @save_admission_control_by_trx;
@save_admission_control_by_trx
0
# Turn on admission_control_by_trx
set @@global.admission_control_by_trx = true;
SELECT @@global.admission_control_by_trx;
@@global.admission_control_by_trx
1
create table t2(a int) engine=innodb;
begin;
insert into t2 values(1);
update t2 set a=2 where a=1;
commit;
SHOW TABLES LIKE 't2';
Tables_in_test_db (t2)
t2
begin;
alter table t2 rename t3;
select * from t3;
a
2
delete from t3;
set @val = 1;
truncate table t3;
rollback;
drop table t3;
set @@global.admission_control_filter = default;
select @@global.admission_control_filter;
@@global.admission_control_filter

select count(*) from t1;
count(*)
1
set @@global.admission_control_by_trx = @save_admission_control_by_trx;
select @@global.admission_control_by_trx;
@@global.admission_control_by_trx
0
set @@global.admission_control_filter = 'COMMIT';
select @@global.admission_control_filter;
@@global.admission_control_filter
COMMIT
commit;
set @@global.max_running_queries = @start_max_running_queries;
set @@global.innodb_lock_wait_timeout = @start_innodb_lock_wait_timeout;
set @@global.admission_control_filter = @start_admission_control_filter;
drop database test_db;
drop user test_user@localhost;
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
create database test_db;
create user test_user@localhost identified with 'mysql_native_password' BY '';
grant all on test_db.* to test_user@localhost;
grant all on test.* to test_user@localhost;
use test_db;
create table t1 (a int primary key, b int) engine=InnoDB;
drop database test_db;
drop user test_user@localhost;
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
create database test_db;
create user test_user@localhost identified with 'mysql_native_password' BY '';
grant all on test_db.* to test_user@localhost;
grant all on test.* to test_user@localhost;
use test_db;
create table t1 (a int primary key, b int) engine=InnoDB;
drop database test_db;
drop user test_user@localhost;
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
create database test_db;
create user test_user@localhost identified with 'mysql_native_password' BY '';
grant all on test_db.* to test_user@localhost;
grant all on test.* to test_user@localhost;
use test_db;
create table t1 (a int primary key, b int) engine=InnoDB;
drop database test_db;
drop user test_user@localhost;
Loading

0 comments on commit 0291698

Please sign in to comment.