Remove type filter from issues/mr/notes ES queries
What does this MR do and why?
This MR removes { "term": { "type": { "value": "<TYPE>" } from elasticsearch queries for issues, merge requests, and notes. This results in approximately 10% performance gain.
Benchmark using Hyperfine
$ hyperfine --warmup 3 './benchmark_issues_with_type.sh' './benchmark_issues_without_type.sh'
Benchmark #1: ./benchmark_issues_with_type.sh
Time (mean ± σ): 20.1 ms ± 2.2 ms [User: 5.0 ms, System: 2.5 ms]
Range (min … max): 14.0 ms … 26.4 ms 111 runs
Benchmark #2: ./benchmark_issues_without_type.sh
Time (mean ± σ): 18.2 ms ± 2.0 ms [User: 4.9 ms, System: 2.4 ms]
Range (min … max): 12.0 ms … 21.9 ms 139 runs
Summary
'./benchmark_issues_without_type.sh' ran
1.11 ± 0.17 times faster than './benchmark_issues_with_type.sh'$ hyperfine --warmup 3 './benchmark_mr_with_type.sh' './benchmark_mr_without_type.sh'
Benchmark #1: ./benchmark_mr_with_type.sh
Time (mean ± σ): 17.5 ms ± 2.2 ms [User: 5.1 ms, System: 2.3 ms]
Range (min … max): 11.8 ms … 23.1 ms 147 runs
Benchmark #2: ./benchmark_mr_without_type.sh
Time (mean ± σ): 16.1 ms ± 1.8 ms [User: 4.9 ms, System: 2.6 ms]
Range (min … max): 10.4 ms … 20.1 ms 165 runs
Summary
'./benchmark_mr_without_type.sh' ran
1.08 ± 0.18 times faster than './benchmark_mr_with_type.sh'$ hyperfine --warmup 3 './benchmark_notes_with_type.sh' './benchmark_notes_without_type.sh'
Benchmark #1: ./benchmark_notes_with_type.sh
Time (mean ± σ): 20.6 ms ± 3.1 ms [User: 4.8 ms, System: 3.2 ms]
Range (min … max): 16.3 ms … 36.0 ms 107 runs
Benchmark #2: ./benchmark_notes_without_type.sh
Time (mean ± σ): 18.4 ms ± 2.0 ms [User: 5.0 ms, System: 2.8 ms]
Range (min … max): 13.8 ms … 25.3 ms 139 runs
Summary
'./benchmark_notes_without_type.sh' ran
1.12 ± 0.21 times faster than './benchmark_notes_with_type.sh'Benchmark files
benchmark_issues_with_type.sh
#!/bin/sh
curl -H 'Content-type: application/json' -XGET "$CLUSTER_URL/gitlab-development-issues/doc/_search?from=0&routing=project_20&size=20" -d '{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"iid^3",
"title^2",
"description"
],
"query": "update",
"lenient": true,
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:issue",
"value": "issue"
}
}
},
{
"terms": {
"_name": "issue:authorized:project",
"project_id": [
20
]
}
}
]
}
},
"highlight": {
"fields": {
"iid": {},
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}'benchmark_issues_without_type.sh
#!/bin/sh
curl -H 'Content-type: application/json' -XGET "$CLUSTER_URL/gitlab-development-issues/doc/_search?from=0&routing=project_20&size=20" -d '{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "issue:match:search_terms",
"fields": [
"iid^3",
"title^2",
"description"
],
"query": "update",
"lenient": true,
"default_operator": "and"
}
}
],
"filter": [
{
"terms": {
"_name": "issue:authorized:project",
"project_id": [
20
]
}
}
]
}
},
"highlight": {
"fields": {
"iid": {},
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}'benchmark_mr_with_type.sh
#!/bin/sh
curl -H 'Content-type: application/json' -XGET "$CLUSTER_URL/gitlab-development-merge_requests/doc/_search?from=0&routing=project_20&size=20" -d '{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "merge_request:match:search_terms",
"fields": [
"iid^3",
"title^2",
"description"
],
"query": "update",
"lenient": true,
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:merge_request",
"value": "merge_request"
}
}
},
{
"terms": {
"_name": "merge_request:authorized:project",
"target_project_id": [
20
]
}
}
]
}
},
"highlight": {
"fields": {
"iid": {},
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}'benchmark_mr_without_type.sh
#!/bin/sh
curl -H 'Content-type: application/json' -XGET "$CLUSTER_URL/gitlab-development-merge_requests/doc/_search?from=0&routing=project_20&size=20" -d '{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "merge_request:match:search_terms",
"fields": [
"iid^3",
"title^2",
"description"
],
"query": "update",
"lenient": true,
"default_operator": "and"
}
}
],
"filter": [
{
"terms": {
"_name": "merge_request:authorized:project",
"target_project_id": [
20
]
}
}
]
}
},
"highlight": {
"fields": {
"iid": {},
"title": {},
"description": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}'benchmark_notes_with_type.sh
#!/bin/sh
curl -H 'Content-type: application/json' -XGET "$CLUSTER_URL/gitlab-development-notes/doc/_search?from=0&routing=project_20&size=20" -d '{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "note:match:search_terms",
"fields": [
"note"
],
"query": "update",
"lenient": true,
"default_operator": "and"
}
}
],
"filter": [
{
"term": {
"type": {
"_name": "doc:is_a:note",
"value": "note"
}
}
},
{
"bool": {
"_name": "note:authorized",
"should": [
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:Issue",
"value": "Issue"
}
}
},
{
"bool": {
"_name": "note:authorized:project:issues",
"filter": [
{
"terms": {
"_name": "note:authorized:project:issues:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:issues:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:MergeRequest",
"value": "MergeRequest"
}
}
},
{
"bool": {
"_name": "note:authorized:project:merge_requests",
"filter": [
{
"terms": {
"_name": "note:authorized:project:merge_requests:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:merge_requests:enabled_or_private",
"merge_requests_access_level": [
20,
10
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:Snippet",
"value": "Snippet"
}
}
},
{
"bool": {
"_name": "note:authorized:project:snippets",
"filter": [
{
"terms": {
"_name": "note:authorized:project:snippets:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:snippets:enabled_or_private",
"snippets_access_level": [
20,
10
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:Commit",
"value": "Commit"
}
}
},
{
"bool": {
"_name": "note:authorized:project:repository",
"filter": [
{
"terms": {
"_name": "note:authorized:project:repository:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:repository:enabled_or_private",
"repository_access_level": [
20,
10
]
}
}
]
}
}
]
}
}
]
}
}
]
}
},
"highlight": {
"fields": {
"note": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}'benchmark_notes_without_type.sh
#!/bin/sh
curl -H 'Content-type: application/json' -XGET "$CLUSTER_URL/gitlab-development-notes/doc/_search?from=0&routing=project_20&size=20" -d '{
"query": {
"bool": {
"must": [
{
"simple_query_string": {
"_name": "note:match:search_terms",
"fields": [
"note"
],
"query": "update",
"lenient": true,
"default_operator": "and"
}
}
],
"filter": [
{
"bool": {
"_name": "note:authorized",
"should": [
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:Issue",
"value": "Issue"
}
}
},
{
"bool": {
"_name": "note:authorized:project:issues",
"filter": [
{
"terms": {
"_name": "note:authorized:project:issues:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:issues:enabled_or_private",
"issues_access_level": [
20,
10
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:MergeRequest",
"value": "MergeRequest"
}
}
},
{
"bool": {
"_name": "note:authorized:project:merge_requests",
"filter": [
{
"terms": {
"_name": "note:authorized:project:merge_requests:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:merge_requests:enabled_or_private",
"merge_requests_access_level": [
20,
10
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:Snippet",
"value": "Snippet"
}
}
},
{
"bool": {
"_name": "note:authorized:project:snippets",
"filter": [
{
"terms": {
"_name": "note:authorized:project:snippets:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:snippets:enabled_or_private",
"snippets_access_level": [
20,
10
]
}
}
]
}
}
]
}
},
{
"bool": {
"must": [
{
"term": {
"noteable_type": {
"_name": "note:authorized:noteable:is_a:Commit",
"value": "Commit"
}
}
},
{
"bool": {
"_name": "note:authorized:project:repository",
"filter": [
{
"terms": {
"_name": "note:authorized:project:repository:membership:id",
"project_id": [
20
]
}
},
{
"terms": {
"_name": "note:authorized:project:repository:enabled_or_private",
"repository_access_level": [
20,
10
]
}
}
]
}
}
]
}
}
]
}
}
]
}
},
"highlight": {
"fields": {
"note": {}
},
"number_of_fragments": 0,
"pre_tags": [
"gitlabelasticsearch→"
],
"post_tags": [
"←gitlabelasticsearch"
]
}
}'
Screenshots or screen recordings
These are strongly recommended to assist reviewers and reduce the time to merge your change.
How to set up and validate locally
Numbered steps to set up and validate the change are strongly suggested.
- Enable Advanced Search integration
- Without switching to this MR branch: Open
/searchand search for issues or merge request - Check that you can see
termfilter bytypein Elasticsearch query (this can be done via performance bar) - Switch to this MR branch
- Refresh the search and ensure that
termfilter is no longer in the query
MR acceptance checklist
This checklist encourages us to confirm any changes have been analyzed to reduce risks in quality, performance, reliability, security, and maintainability.
-
I have evaluated the MR acceptance checklist for this MR.
Edited by Dmitry Gruzd