Fix N+1 queries for issues search
There are additional N+1 queries for issues search related to the namespace.route and project.project_features.
The spec ee/spec/features/search/elastic/global_search_spec.rb
should also be updated to set the query_count_multiplier
to 0.
Seen again while working on: !57284 (merged)
failing spec
Failures:
1) Global elastic search I do not overload the database searching issues behaves like an efficient database result avoids N+1 database queries
Failure/Error: expect { visit path }.not_to exceed_query_limit(control_count)
Expected a maximum of 36 queries, got 42:
SELECT "users".* FROM "users" WHERE "users"."id" = 1 ORDER BY "users"."id" ASC LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT 1 AS one FROM "identities" WHERE "identities"."user_id" = 1 AND (provider LIKE 'ldap%' AND extern_uid IS NOT NULL) LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "user_preferences".* FROM "user_preferences" WHERE "user_preferences"."user_id" = 1 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "projects"."id" FROM "projects" INNER JOIN "project_authorizations" ON "projects"."id" = "project_authorizations"."project_id" WHERE "project_authorizations"."user_id" = 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "projects"."id" FROM "projects" LEFT JOIN project_features ON projects.id = project_features.project_id WHERE "projects"."id" = 1 AND ("project_features"."issues_access_level" IS NULL OR "project_features"."issues_access_level" IN (20,30) OR ("project_features"."issues_access_level" = 10 AND EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 10)))) AND (EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 10)) OR projects.visibility_level IN (10,20)) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "projects"."id" FROM "projects" INNER JOIN "project_authorizations" ON "projects"."id" = "project_authorizations"."project_id" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.access_level >= 20) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "issues".* FROM "issues" WHERE "issues"."id" IN (5, 1, 2, 6, 3, 4) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "users".* FROM "users" WHERE "users"."id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "projects".* FROM "projects" WHERE "projects"."id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_type" = 'Project' AND "routes"."source_id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "label_links".* FROM "label_links" WHERE "label_links"."target_type" = 'Issue' AND "label_links"."target_id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "timelogs".* FROM "timelogs" WHERE "timelogs"."issue_id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "issue_assignees".* FROM "issue_assignees" WHERE "issue_assignees"."issue_id" IN (1, 2, 3, 4, 5, 6) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "project_features".* FROM "project_features" WHERE "project_features"."project_id" = 5 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "project_features".* FROM "project_features" WHERE "project_features"."project_id" = 1 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 1 AND "namespaces"."type" = 'Group' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT MAX("project_authorizations"."access_level") AS maximum_access_level, "project_authorizations"."user_id" AS project_authorizations_user_id FROM "project_authorizations" WHERE "project_authorizations"."project_id" = 1 AND "project_authorizations"."user_id" = 1 GROUP BY "project_authorizations"."user_id" /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "project_features".* FROM "project_features" WHERE "project_features"."project_id" = 2 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "project_features".* FROM "project_features" WHERE "project_features"."project_id" = 6 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "project_features".* FROM "project_features" WHERE "project_features"."project_id" = 3 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "project_features".* FROM "project_features" WHERE "project_features"."project_id" = 4 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id" DESC LIMIT 100 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 5 AND "routes"."source_type" = 'Namespace' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 1 AND "routes"."source_type" = 'Namespace' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 2 AND "routes"."source_type" = 'Namespace' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 6 AND "routes"."source_type" = 'Namespace' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 3 AND "routes"."source_type" = 'Namespace' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = 4 AND "routes"."source_type" = 'Namespace' LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "appearances".* FROM "appearances" ORDER BY "appearances"."id" ASC LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "user_statuses".* FROM "user_statuses" WHERE "user_statuses"."user_id" = 1 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT COUNT(*) FROM "projects" INNER JOIN "namespaces" ON "projects"."namespace_id" = "namespaces"."id" WHERE "namespaces"."owner_id" = 1 AND "namespaces"."type" IS NULL /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT 1 AS one FROM "users" WHERE "users"."id" = 1 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "users".* FROM "users" WHERE "users"."id" = 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT COUNT(*) FROM "issues" INNER JOIN "projects" ON "projects"."id" = "issues"."project_id" LEFT JOIN project_features ON projects.id = project_features.project_id WHERE (
issues.confidential IS NOT TRUE
OR (issues.confidential = TRUE
AND (issues.author_id = 1
OR EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = 1 AND issue_id = issues.id)
OR EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = issues.project_id) AND (project_authorizations.access_level >= 20))))) AND (EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 10)) OR projects.visibility_level IN (10,20)) AND ("project_features"."issues_access_level" IS NULL OR "project_features"."issues_access_level" IN (20,30) OR ("project_features"."issues_access_level" = 10 AND EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 10)))) AND ("issues"."state_id" IN (1)) AND (EXISTS (SELECT true FROM "issue_assignees" WHERE "issue_assignees"."user_id" IN (1) AND issue_id = issues.id)) AND "projects"."archived" = FALSE /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT COUNT(*) FROM "merge_requests" INNER JOIN "projects" ON "projects"."id" = "merge_requests"."target_project_id" LEFT JOIN project_features ON projects.id = project_features.project_id WHERE (EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 20)) OR projects.visibility_level IN (10,20)) AND ("project_features"."merge_requests_access_level" IS NULL OR "project_features"."merge_requests_access_level" IN (20,30) OR ("project_features"."merge_requests_access_level" = 10 AND EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 20)))) AND ("merge_requests"."state_id" IN (1)) AND (EXISTS (SELECT true FROM "merge_request_assignees" WHERE "merge_request_assignees"."user_id" IN (1) AND merge_request_id = merge_requests.id)) AND "projects"."archived" = FALSE /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "users".* FROM "users" WHERE "users"."id" = 1 LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT COUNT(*) FROM "merge_requests" INNER JOIN "projects" ON "projects"."id" = "merge_requests"."target_project_id" LEFT JOIN project_features ON projects.id = project_features.project_id WHERE (EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 20)) OR projects.visibility_level IN (10,20)) AND ("project_features"."merge_requests_access_level" IS NULL OR "project_features"."merge_requests_access_level" IN (20,30) OR ("project_features"."merge_requests_access_level" = 10 AND EXISTS (SELECT 1 FROM "project_authorizations" WHERE "project_authorizations"."user_id" = 1 AND (project_authorizations.project_id = projects.id) AND (project_authorizations.access_level >= 20)))) AND ("merge_requests"."state_id" IN (1)) AND "projects"."archived" = FALSE AND EXISTS (SELECT true FROM "merge_request_reviewers" WHERE merge_request_id = merge_requests.id AND "merge_request_reviewers"."user_id" = 1) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT COUNT(*) FROM "todos" WHERE "todos"."user_id" = 1 AND ("todos"."state" IN ('pending')) /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT 1 AS one FROM "geo_nodes" LIMIT 1 /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "broadcast_messages".* FROM "broadcast_messages" WHERE (ends_at > '2021-03-24 18:29:46.813230') AND "broadcast_messages"."broadcast_type" = 1 ORDER BY "broadcast_messages"."id" ASC /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
SELECT "broadcast_messages".* FROM "broadcast_messages" WHERE (ends_at > '2021-03-24 18:29:47.721294') AND "broadcast_messages"."broadcast_type" = 2 ORDER BY "broadcast_messages"."id" ASC /*application:test,correlation_id:ac532e5f-3095-4b81-968b-d105f71eaf49,endpoint_id:SearchController#show*/
Shared Example Group: "an efficient database result" called from ./ee/spec/features/search/elastic/global_search_spec.rb:50
# ./ee/spec/features/search/elastic/global_search_spec.rb:36:in `block (3 levels) in <main>'
# ./spec/spec_helper.rb:365:in `block (3 levels) in <main>'
# ./spec/support/sidekiq_middleware.rb:9:in `with_sidekiq_server_middleware'
# ./spec/spec_helper.rb:356:in `block (2 levels) in <main>'
# ./spec/spec_helper.rb:348:in `block (3 levels) in <main>'
# ./lib/gitlab/application_context.rb:31:in `with_raw_context'
# ./spec/spec_helper.rb:348:in `block (2 levels) in <main>'
# ./spec/support/sidekiq.rb:21:in `block (3 levels) in <main>'
# ./spec/support/sidekiq.rb:8:in `gitlab_sidekiq_inline'
# ./spec/support/sidekiq.rb:21:in `block (2 levels) in <main>'
# ./ee/spec/support/elastic.rb:21:in `block (2 levels) in <main>'
# -e:1:in `<main>'
The following discussion from !36941 (merged) should be addressed:
-
@tigerwnz started a discussion: (+2 comments) It looks like we should also preload the project's
route
andnamespace
, as each of these are N+1 too.scope :with_web_entity_associations, -> { preload(:author, project: [:route, :namespace] ) }
There is also an additional N+1 loading
namespace.route
. Ideally we would just addnamespace: :route
to the preloaded associations, but the single-table inheritance ofGroup
andNamespace
seems to prevent this working. A problem for another day perhaps?
Release Notes
Edited by John McGuire