In my MySQL DB (v5.7.31-34, using InnoDB), I'm trying to streamline the performance of the following query:
SELECT DISTINCT SQL_CALC_FOUND_ROWS `story_allocation_days`.*
FROM `story_allocation_days`
INNER JOIN `workspaces`
ON `workspaces`.`id` = `story_allocation_days`.`workspace_id`
INNER JOIN `participations`
ON `participations`.`workspace_id` = `workspaces`.`id`
INNER JOIN `assignments`
ON `assignments`.`id` = `story_allocation_days`.`assignment_id`
WHERE (story_allocation_days.deleted_at is null)
AND `workspaces`.`is_budgeted` = TRUE
AND (participations.account_id = 5071)
AND (participations.access_integer >= 30)
AND (participations.type = 'MavenParticipation' OR workspaces.account_id = 5071)
AND (assignments.assignee_id IS NOT NULL);
99822 rows in set (5.35 sec)
The EXPLAIN
on the above:
| id | select_type | table | partitions | type | possible_keys | key | key_len | ref | rows | filtered | Extra |
| 1 | SIMPLE | story_allocation_days | NULL | ALL | index_story_allocation_days_on_workspace_id,index_story_allocation_days_on_assignment_id_and_date,index_story_allocation_days_on_assignment_id | NULL | NULL | NULL | 430531 | 10.00 | Using where; Using temporary |
| 1 | SIMPLE | workspaces | NULL | eq_ref | PRIMARY,index_workspaces_on_account_id | PRIMARY | 4 | bm_rpm.story_allocation_days.workspace_id | 1 | 10.00 | Using where; Distinct |
| 1 | SIMPLE | assignments | NULL | eq_ref | PRIMARY,index_assignments_on_assignee_id | PRIMARY | 4 | bm_rpm.story_allocation_days.assignment_id | 1 | 50.00 | Using where; Distinct |
| 1 | SIMPLE | participations | NULL | ref | index_participations_on_account_id_and_workspace_id,index_participations_on_workspace_id_and_user_id,index_participations_for_sad_api_index,index_participations_on_account_id,index_participations_on_workspace_id | index_participations_for_sad_api_index | 10 | bm_rpm.story_allocation_days.workspace_id,const | 7 | 33.33 | Using where; Using index; Distinct |
Here's the FORMAT=JSON
version of the above:
| {
"query_block": {
"select_id": 1,
"cost_info": {
"query_cost": "153230.79"
},
"duplicates_removal": {
"using_temporary_table": true,
"using_filesort": false,
"nested_loop": [
{
"table": {
"table_name": "story_allocation_days",
"access_type": "ALL",
"possible_keys": [
"index_story_allocation_days_on_workspace_id",
"index_story_allocation_days_on_assignment_id_and_date",
"index_story_allocation_days_on_assignment_id"
],
"rows_examined_per_scan": 430268,
"rows_produced_per_join": 43026,
"filtered": "10.00",
"cost_info": {
"read_cost": "80548.24",
"eval_cost": "8605.36",
"prefix_cost": "89153.60",
"data_read_per_join": "3M"
},
"used_columns": [
"id",
"assignment_id",
"story_id",
"workspace_id",
"account_id",
"current",
"date",
"minutes",
"created_at",
"updated_at",
"deleted_at",
"cost_amount_in_cents",
"bill_amount_in_cents",
"cost_rate_in_cents",
"bill_rate_in_cents"
],
"attached_condition": "isnull(`bm_rpm`.`story_allocation_days`.`deleted_at`)"
}
},
{
"table": {
"table_name": "workspaces",
"access_type": "eq_ref",
"possible_keys": [
"PRIMARY",
"index_workspaces_on_account_id"
],
"key": "PRIMARY",
"used_key_parts": [
"id"
],
"key_length": "4",
"ref": [
"bm_rpm.story_allocation_days.workspace_id"
],
"rows_examined_per_scan": 1,
"rows_produced_per_join": 4302,
"filtered": "10.00",
"distinct": true,
"cost_info": {
"read_cost": "43026.80",
"eval_cost": "860.54",
"prefix_cost": "140785.76",
"data_read_per_join": "21M"
},
"used_columns": [
"id",
"is_budgeted",
"account_id"
],
"attached_condition": "(`bm_rpm`.`workspaces`.`is_budgeted` = TRUE)"
}
},
{
"table": {
"table_name": "assignments",
"access_type": "eq_ref",
"possible_keys": [
"PRIMARY",
"index_assignments_on_assignee_id"
],
"key": "PRIMARY",
"used_key_parts": [
"id"
],
"key_length": "4",
"ref": [
"bm_rpm.story_allocation_days.assignment_id"
],
"rows_examined_per_scan": 1,
"rows_produced_per_join": 2151,
"filtered": "50.00",
"distinct": true,
"cost_info": {
"read_cost": "4302.68",
"eval_cost": "430.27",
"prefix_cost": "145948.98",
"data_read_per_join": "134K"
},
"used_columns": [
"id",
"assignee_id"
],
"attached_condition": "(`bm_rpm`.`assignments`.`assignee_id` is not null)"
}
},
{
"table": {
"table_name": "participations",
"access_type": "ref",
"possible_keys": [
"index_participations_on_account_id_and_workspace_id",
"index_participations_on_workspace_id_and_user_id",
"index_participations_for_sad_api_index",
"index_participations_on_account_id",
"index_participations_on_workspace_id"
],
"key": "index_participations_for_sad_api_index",
"used_key_parts": [
"workspace_id",
"account_id"
],
"key_length": "10",
"ref": [
"bm_rpm.story_allocation_days.workspace_id",
"const"
],
"rows_examined_per_scan": 7,
"rows_produced_per_join": 5537,
"filtered": "33.33",
"using_index": true,
"distinct": true,
"cost_info": {
"read_cost": "3959.11",
"eval_cost": "1107.46",
"prefix_cost": "153230.79",
"data_read_per_join": "11M"
},
"used_columns": [
"id",
"workspace_id",
"type",
"account_id",
"access_integer"
],
"attached_condition": "((`bm_rpm`.`participations`.`access_integer` >= 30) and ((`bm_rpm`.`participations`.`type` = 'MavenParticipation') or (`bm_rpm`.`workspaces`.`account_id` = 5071)))"
}
}
]
}
}
} |
I tried finding an index which would change this from a full table scan to a more efficient type of search. Here are the indexes:
mysql> show indexes in story_allocation_days;
| Table | Non_unique | Key_name | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment |
| story_allocation_days | 0 | PRIMARY | 1 | id | A | 418853 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_workspace_id | 1 | workspace_id | A | 295 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_assignment_id_and_date | 1 | assignment_id | A | 42533 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_assignment_id_and_date | 2 | date | A | 418853 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_account_id | 1 | account_id | A | 41 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_story_id_and_date | 1 | story_id | A | 2519 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_story_id_and_date | 2 | date | A | 83246 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_date | 1 | date | A | 740 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_created_at | 1 | created_at | A | 6977 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_updated_at | 1 | updated_at | A | 7681 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_assignment_id | 1 | assignment_id | A | 43891 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | index_story_allocation_days_on_story_id | 1 | story_id | A | 2494 | NULL | NULL | | BTREE | | |
| story_allocation_days | 1 | sad_deleted_at | 1 | deleted_at | A | 225 | NULL | NULL | YES | BTREE | | |
I tried USE INDEX(index_story_allocation_days_on_assignment_id)
on story_allocation_days
, however this is actually marginally slower than the original query:
99822 rows in set (5.99 sec)
On a hunch, I thought maybe WHERE (story_allocation_days.deleted_at is null)
was tripping up the optimizer, so I tried creating the sad_deleted_at
index and updating all rows with NULL
for their deleted_at
column to 1970-01-01 00:00:00
, on the chance that the index performance was suffering due to the presence of NULL
values. However, this resulted in performance which was also worse than the original:
SELECT DISTINCT SQL_CALC_FOUND_ROWS `story_allocation_days`.*
FROM `story_allocation_days`
INNER JOIN `workspaces`
ON `workspaces`.`id` = `story_allocation_days`.`workspace_id`
INNER JOIN `participations`
ON `participations`.`workspace_id` = `workspaces`.`id`
INNER JOIN `assignments`
ON `assignments`.`id` = `story_allocation_days`.`assignment_id`
WHERE (story_allocation_days.deleted_at = '1970-01-01 00:00:00')
AND `workspaces`.`is_budgeted` = TRUE
AND (participations.account_id = 5071)
AND (participations.access_integer >= 30)
AND (participations.type = 'MavenParticipation' OR workspaces.account_id = 5071)
AND (assignments.assignee_id IS NOT NULL);
99822 rows in set (6.12 sec)
Here's the corresponding EXPLAIN
statement:
| 1 | SIMPLE | story_allocation_days | NULL | ref | index_story_allocation_days_on_workspace_id,index_story_allocation_days_on_assignment_id_and_date,index_story_allocation_days_on_assignment_id,sad_deleted_at | sad_deleted_at | 6 | const | 210391 | 100.00 | Using temporary |
| 1 | SIMPLE | workspaces | NULL | eq_ref | PRIMARY,index_workspaces_on_account_id | PRIMARY | 4 | bm_rpm.story_allocation_days.workspace_id | 1 | 10.00 | Using where; Distinct |
| 1 | SIMPLE | assignments | NULL | eq_ref | PRIMARY,index_assignments_on_assignee_id | PRIMARY | 4 | bm_rpm.story_allocation_days.assignment_id | 1 | 50.00 | Using where; Distinct |
| 1 | SIMPLE | participations | NULL | ref | index_participations_on_account_id_and_workspace_id,index_participations_on_workspace_id_and_user_id,index_participations_for_sad_api_index,index_participations_on_account_id,index_participations_on_workspace_id | index_participations_for_sad_api_index | 10 | bm_rpm.story_allocation_days.workspace_id,const | 7 | 33.33 | Using where; Using index; Distinct |
And again, the FORMAT=JSON
version of the above:
# NEW:
| {
"query_block": {
"select_id": 1,
"cost_info": {
"query_cost": "366895.66"
},
"duplicates_removal": {
"using_temporary_table": true,
"using_filesort": false,
"nested_loop": [
{
"table": {
"table_name": "story_allocation_days",
"access_type": "ref",
"possible_keys": [
"index_story_allocation_days_on_workspace_id",
"index_story_allocation_days_on_assignment_id_and_date",
"index_story_allocation_days_on_assignment_id",
"sad_deleted_at"
],
"key": "sad_deleted_at",
"used_key_parts": [
"deleted_at"
],
"key_length": "6",
"ref": [
"const"
],
"rows_examined_per_scan": 210552,
"rows_produced_per_join": 210552,
"filtered": "100.00",
"cost_info": {
"read_cost": "11223.00",
"eval_cost": "42110.40",
"prefix_cost": "53333.40",
"data_read_per_join": "16M"
},
"used_columns": [
"id",
"assignment_id",
"story_id",
"workspace_id",
"account_id",
"current",
"date",
"minutes",
"created_at",
"updated_at",
"deleted_at",
"cost_amount_in_cents",
"bill_amount_in_cents",
"cost_rate_in_cents",
"bill_rate_in_cents"
]
}
},
{
"table": {
"table_name": "workspaces",
"access_type": "eq_ref",
"possible_keys": [
"PRIMARY",
"index_workspaces_on_account_id"
],
"key": "PRIMARY",
"used_key_parts": [
"id"
],
"key_length": "4",
"ref": [
"bm_rpm.story_allocation_days.workspace_id"
],
"rows_examined_per_scan": 1,
"rows_produced_per_join": 21055,
"filtered": "10.00",
"distinct": true,
"cost_info": {
"read_cost": "210552.00",
"eval_cost": "4211.04",
"prefix_cost": "305995.80",
"data_read_per_join": "105M"
},
"used_columns": [
"id",
"is_budgeted",
"account_id"
],
"attached_condition": "(`bm_rpm`.`workspaces`.`is_budgeted` = TRUE)"
}
},
{
"table": {
"table_name": "assignments",
"access_type": "eq_ref",
"possible_keys": [
"PRIMARY",
"index_assignments_on_assignee_id"
],
"key": "PRIMARY",
"used_key_parts": [
"id"
],
"key_length": "4",
"ref": [
"bm_rpm.story_allocation_days.assignment_id"
],
"rows_examined_per_scan": 1,
"rows_produced_per_join": 10527,
"filtered": "50.00",
"distinct": true,
"cost_info": {
"read_cost": "21055.20",
"eval_cost": "2105.52",
"prefix_cost": "331262.04",
"data_read_per_join": "657K"
},
"used_columns": [
"id",
"assignee_id"
],
"attached_condition": "(`bm_rpm`.`assignments`.`assignee_id` is not null)"
}
},
{
"table": {
"table_name": "participations",
"access_type": "ref",
"possible_keys": [
"index_participations_on_account_id_and_workspace_id",
"index_participations_on_workspace_id_and_user_id",
"index_participations_for_sad_api_index",
"index_participations_on_account_id",
"index_participations_on_workspace_id"
],
"key": "index_participations_for_sad_api_index",
"used_key_parts": [
"workspace_id",
"account_id"
],
"key_length": "10",
"ref": [
"bm_rpm.story_allocation_days.workspace_id",
"const"
],
"rows_examined_per_scan": 7,
"rows_produced_per_join": 27096,
"filtered": "33.33",
"using_index": true,
"distinct": true,
"cost_info": {
"read_cost": "19373.95",
"eval_cost": "5419.35",
"prefix_cost": "366895.66",
"data_read_per_join": "55M"
},
"used_columns": [
"id",
"workspace_id",
"type",
"account_id",
"access_integer"
],
"attached_condition": "((`bm_rpm`.`participations`.`access_integer` >= 30) and ((`bm_rpm`.`participations`.`type` = 'MavenParticipation') or (`bm_rpm`.`workspaces`.`account_id` = 5071)))"
}
}
]
}
}
} |
I'm confused why the query time doesn't improve (and in fact gets marginally worse), even though the filter
column for story_allocation_days
goes from 10.0
to 100.0
and the # of rows examined goes from 400,000+ to 200,000-ish.
Can anyone see why the index isn't improving the query time?
EDIT:
I tried removing SQL_CALC_FOUND_ROWS
from the original query, but it still took 5.51 seconds for 99k rows. EXPLAIN
below, in JSON format (since I'm running out of characters for this question):
| {
"query_block": {
"select_id": 1,
"cost_info": {
"query_cost": "153230.79"
},
"duplicates_removal": {
"using_temporary_table": true,
"using_filesort": false,
"nested_loop": [
{
"table": {
"table_name": "story_allocation_days",
"access_type": "ALL",
"possible_keys": [
"index_story_allocation_days_on_workspace_id",
"index_story_allocation_days_on_assignment_id_and_date",
"index_story_allocation_days_on_assignment_id"
],
"rows_examined_per_scan": 430268,
"rows_produced_per_join": 43026,
"filtered": "10.00",
"cost_info": {
"read_cost": "80548.24",
"eval_cost": "8605.36",
"prefix_cost": "89153.60",
"data_read_per_join": "3M"
},
"used_columns": [
"id",
"assignment_id",
"story_id",
"workspace_id",
"account_id",
"current",
"date",
"minutes",
"created_at",
"updated_at",
"deleted_at",
"cost_amount_in_cents",
"bill_amount_in_cents",
"cost_rate_in_cents",
"bill_rate_in_cents"
],
"attached_condition": "isnull(`bm_rpm`.`story_allocation_days`.`deleted_at`)"
}
},
{
"table": {
"table_name": "workspaces",
"access_type": "eq_ref",
"possible_keys": [
"PRIMARY",
"index_workspaces_on_account_id"
],
"key": "PRIMARY",
"used_key_parts": [
"id"
],
"key_length": "4",
"ref": [
"bm_rpm.story_allocation_days.workspace_id"
],
"rows_examined_per_scan": 1,
"rows_produced_per_join": 4302,
"filtered": "10.00",
"distinct": true,
"cost_info": {
"read_cost": "43026.80",
"eval_cost": "860.54",
"prefix_cost": "140785.76",
"data_read_per_join": "21M"
},
"used_columns": [
"id",
"is_budgeted",
"account_id"
],
"attached_condition": "(`bm_rpm`.`workspaces`.`is_budgeted` = TRUE)"
}
},
{
"table": {
"table_name": "assignments",
"access_type": "eq_ref",
"possible_keys": [
"PRIMARY",
"index_assignments_on_assignee_id"
],
"key": "PRIMARY",
"used_key_parts": [
"id"
],
"key_length": "4",
"ref": [
"bm_rpm.story_allocation_days.assignment_id"
],
"rows_examined_per_scan": 1,
"rows_produced_per_join": 2151,
"filtered": "50.00",
"distinct": true,
"cost_info": {
"read_cost": "4302.68",
"eval_cost": "430.27",
"prefix_cost": "145948.98",
"data_read_per_join": "134K"
},
"used_columns": [
"id",
"assignee_id"
],
"attached_condition": "(``bm_rpm`.`assignments`.`assignee_id` is not null)"
}
},
{
"table": {
"table_name": "participations",
"access_type": "ref",
"possible_keys": [
"index_participations_on_account_id_and_workspace_id",
"index_participations_on_workspace_id_and_user_id",
"index_participations_for_sad_api_index",
"index_participations_on_account_id",
"index_participations_on_workspace_id"
],
"key": "index_participations_for_sad_api_index",
"used_key_parts": [
"workspace_id",
"account_id"
],
"key_length": "10",
"ref": [
"bm_rpm.story_allocation_days.workspace_id",
"const"
],
"rows_examined_per_scan": 7,
"rows_produced_per_join": 5537,
"filtered": "33.33",
"using_index": true,
"distinct": true,
"cost_info": {
"read_cost": "3959.11",
"eval_cost": "1107.46",
"prefix_cost": "153230.79",
"data_read_per_join": "11M"
},
"used_columns": [
"id",
"workspace_id",
"type",
"account_id",
"access_integer"
],
"attached_condition": "((`bm_rpm`.`participations`.`access_integer` >= 30) and ((`bm_rpm`.`participations`.`type` = 'MavenParticipation') or (`bm_rpm`.`workspaces`.`account_id` = 5071)))"
}
}
]
}
}
} |
EDIT #2:
Here's the SHOW CREATE TABLE
for story_allocation_days
:
| story_allocation_days | CREATE TABLE `story_allocation_days` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`assignment_id` int(11) NOT NULL,
`story_id` int(11) NOT NULL,
`workspace_id` int(11) NOT NULL,
`account_id` int(11) NOT NULL,
`current` tinyint(1) NOT NULL,
`date` date NOT NULL,
`minutes` int(11) NOT NULL DEFAULT '0',
`created_at` datetime NOT NULL,
`updated_at` datetime NOT NULL,
`deleted_at` datetime DEFAULT NULL,
`cost_amount_in_cents` bigint(20) DEFAULT NULL,
`bill_amount_in_cents` bigint(20) DEFAULT NULL,
`cost_rate_in_cents` bigint(20) DEFAULT NULL,
`bill_rate_in_cents` bigint(20) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `index_story_allocation_days_on_workspace_id` (`workspace_id`),
KEY `index_story_allocation_days_on_assignment_id_and_date` (`assignment_id`,`date`),
KEY `index_story_allocation_days_on_account_id` (`account_id`),
KEY `index_story_allocation_days_on_story_id_and_date` (`story_id`,`date`),
KEY `index_story_allocation_days_on_date` (`date`),
KEY `index_story_allocation_days_on_created_at` (`created_at`),
KEY `index_story_allocation_days_on_updated_at` (`updated_at`),
KEY `index_story_allocation_days_on_assignment_id` (`assignment_id`),
KEY `index_story_allocation_days_on_story_id` (`story_id`)
) ENGINE=InnoDB AUTO_INCREMENT=9646396 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci |
SQL_CALC_FOUND_ROWS
. This is known to have a high performance cost, and there's no reason you need it in the query you show.SQL_CALC_FOUND_ROWS
.show create table yourtablename
for all the tables in the querySHOW CREATE TABLE
for all 4 tables due to StackOverflow's 30,000 char limit on a question body. However, I was able to add such a statement for the main table,story_allocation_days
. Hopefully that's helpful.