Include route and hosted_plan to reduce n+1 queries in namespaces list api
What does this MR do?
Include namespace
's route and gitlab_subscription
's hosted_plan
in the namespaces list API, since
those are required for every namespace and gitlab_subscription.
This reduces N+1 queries by 3 per namespace.
-
namespaces.include_route
gets rid of the following queries:[2] SELECT "routes".* FROM "routes" WHERE "routes"."source_id" = $1 AND "routes"."source_type" = $2 LIMIT $3
-
Replacing
namespaces.include_gitlab_subscription
withnamespaces.include_gitlab_subscription_with_hosted_plan
gets rid of the following query:[1] SELECT "plans".* FROM "plans" WHERE "plans"."id" = $1 LIMIT $2
The remaining extra queries (as reported by the `exceed_all_query_limit` matcher) are:
[2] SELECT "personal_access_tokens".* FROM "personal_access_tokens" WHERE "personal_access_tokens"."token_digest" = 'abcd' LIMIT 1
[1] INSERT INTO "personal_access_tokens" ("user_id", "name", "expires_at", "created_at", "updated_at", "scopes", "token_digest") VALUES (1, 'PAT 2', '2020-10-12', '2020-10-07 12:13:40.865517', '2020-10-07 12:13:40.865517', '---
- api
', 'abcd') RETURNING "id"
[1] UPDATE "personal_access_tokens" SET "last_used_at" = '2020-10-07 12:13:40.997206' WHERE "personal_access_tokens"."id" = 2
[1] SELECT COUNT(*) FROM (SELECT 1 AS one FROM "namespaces" WHERE "namespaces"."id" IN (1, 3, 5, 4) LIMIT 10001) subquery_for_count
[1] SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" IN (1, 3, 5, 4) ORDER BY "namespaces"."id" ASC LIMIT 20 OFFSET 0
[1] SELECT "routes".* FROM "routes" WHERE "routes"."source_type" = 'Namespace' AND "routes"."source_id" IN (1, 3, 4, 5)
[1] SELECT "gitlab_subscriptions".* FROM "gitlab_subscriptions" WHERE "gitlab_subscriptions"."namespace_id" IN (1, 3, 4, 5)
[1] WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 3)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "namespaces".* FROM "base_and_ancestors" AS "namespaces" WHERE "namespaces"."parent_id" IS NULL LIMIT $1
[2] SELECT "saml_providers".* FROM "saml_providers" WHERE "saml_providers"."group_id" = $1 LIMIT $2
[1] SELECT "members".* FROM ((SELECT "members".* FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = 'GroupMember' AND "members"."source_type" = 'Namespace' AND "users"."state" = 'active' AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 3)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "id" FROM "base_and_ancestors" AS "namespaces"))
UNION
(WITH "group_group_links_cte" AS (SELECT "group_group_links".* FROM "group_group_links" WHERE "group_group_links"."shared_group_id" IN (WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 3)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "id" FROM "base_and_ancestors" AS "namespaces")) SELECT "members"."id", LEAST("group_group_links"."group_access", "members"."access_level") AS access_level, "members"."source_id", "members"."source_type", "members"."user_id", "members"."notification_level", "members"."type", "members"."created_at", "members"."updated_at", "members"."created_by_id", "members"."invite_email", "members"."invite_token", "members"."invite_accepted_at", "members"."requested_at", "members"."expires_at", "members"."ldap", "members"."override" FROM "members", "group_group_links_cte" AS "group_group_links" WHERE "members"."type" = 'GroupMember' AND "members"."source_type" = 'Namespace' AND "members"."requested_at" IS NULL AND "members"."source_id" = "group_group_links"."shared_with_group_id" AND "members"."source_type" = 'Namespace' AND (members.access_level > 5))) members WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "members"."user_id" = $3 ORDER BY "members"."access_level" DESC LIMIT $4
[1] SELECT "gitlab_subscriptions".* FROM "gitlab_subscriptions" WHERE "gitlab_subscriptions"."namespace_id" = $1 LIMIT $2
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_descendants" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 3)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_descendants" WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id")) SELECT "namespaces"."id" FROM "base_and_descendants" AS "namespaces")
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND ("users"."user_type" IS NULL OR "users"."user_type" != $4) AND "members"."source_id" IN (SELECT "projects"."id" FROM "projects" INNER JOIN "namespaces" ON "namespaces"."type" = $5 AND "namespaces"."id" = "projects"."namespace_id" AND "namespaces"."type" = $6 WHERE "projects"."namespace_id" IN (WITH RECURSIVE "base_and_descendants" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 3)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_descendants" WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id")) SELECT id FROM "base_and_descendants" AS "namespaces"))
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" INNER JOIN "group_group_links" ON "group_group_links"."shared_with_group_id" = "namespaces"."id" WHERE "namespaces"."type" = 'Group' AND "group_group_links"."shared_group_id" IN (WITH RECURSIVE "base_and_descendants" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 3)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_descendants" WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id")) SELECT "namespaces"."id" FROM "base_and_descendants" AS "namespaces"))
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "namespaces"."id" FROM "base_and_ancestors" AS "namespaces")
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" INNER JOIN "project_group_links" ON "project_group_links"."group_id" = "namespaces"."id" WHERE "namespaces"."type" = 'Group' AND "project_group_links"."project_id" IN (SELECT "projects"."id" FROM "projects" INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project' WHERE (rs.path LIKE 'group2/group1/%')))
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "namespaces"."id" FROM "base_and_ancestors" AS "namespaces")
[1] SELECT "members".* FROM ((SELECT "members".* FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = 'GroupMember' AND "members"."source_type" = 'Namespace' AND "users"."state" = 'active' AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND (members.access_level > 5) AND "members"."source_id" = 5)
UNION
(WITH "group_group_links_cte" AS (SELECT "group_group_links".* FROM "group_group_links" WHERE "group_group_links"."shared_group_id" = 5) SELECT "members"."id", LEAST("group_group_links"."group_access", "members"."access_level") AS access_level, "members"."source_id", "members"."source_type", "members"."user_id", "members"."notification_level", "members"."type", "members"."created_at", "members"."updated_at", "members"."created_by_id", "members"."invite_email", "members"."invite_token", "members"."invite_accepted_at", "members"."requested_at", "members"."expires_at", "members"."ldap", "members"."override" FROM "members", "group_group_links_cte" AS "group_group_links" WHERE "members"."type" = 'GroupMember' AND "members"."source_type" = 'Namespace' AND "members"."requested_at" IS NULL AND "members"."source_id" = "group_group_links"."shared_with_group_id" AND "members"."source_type" = 'Namespace' AND (members.access_level > 5))) members WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "members"."user_id" = $3 ORDER BY "members"."access_level" DESC LIMIT $4
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_descendants" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 5)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_descendants" WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id")) SELECT "namespaces"."id" FROM "base_and_descendants" AS "namespaces") AND (members.access_level > 10)
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND ("users"."user_type" IS NULL OR "users"."user_type" != $4) AND "members"."source_id" IN (SELECT "projects"."id" FROM "projects" INNER JOIN "namespaces" ON "namespaces"."type" = $5 AND "namespaces"."id" = "projects"."namespace_id" AND "namespaces"."type" = $6 WHERE "projects"."namespace_id" IN (WITH RECURSIVE "base_and_descendants" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 5)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_descendants" WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id")) SELECT id FROM "base_and_descendants" AS "namespaces")) AND (members.access_level > 10)
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" INNER JOIN "group_group_links" ON "group_group_links"."shared_with_group_id" = "namespaces"."id" WHERE "namespaces"."type" = 'Group' AND "group_group_links"."shared_group_id" IN (WITH RECURSIVE "base_and_descendants" AS ((SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = 5)
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_descendants" WHERE "namespaces"."type" = 'Group' AND "namespaces"."parent_id" = "base_and_descendants"."id")) SELECT "namespaces"."id" FROM "base_and_descendants" AS "namespaces") AND (group_access > 10))
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "namespaces"."id" FROM "base_and_ancestors" AS "namespaces") AND (members.access_level > 10)
[1] SELECT DISTINCT "members"."user_id" FROM "members" LEFT OUTER JOIN "users" ON "members"."user_id" = "users"."id" WHERE "members"."type" = $1 AND "members"."source_type" = $2 AND "users"."state" = $3 AND "members"."requested_at" IS NULL AND "members"."invite_token" IS NULL AND (members.access_level > 5) AND "members"."source_id" IN (WITH RECURSIVE "base_and_ancestors" AS ((SELECT "namespaces".* FROM "namespaces" INNER JOIN "project_group_links" ON "project_group_links"."group_id" = "namespaces"."id" WHERE "namespaces"."type" = 'Group' AND "project_group_links"."project_id" IN (SELECT "projects"."id" FROM "projects" INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project' WHERE (rs.path LIKE 'group3/%')) AND (group_access > 10))
UNION
(SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."type" = 'Group' AND "namespaces"."id" = "base_and_ancestors"."parent_id")) SELECT "namespaces"."id" FROM "base_and_ancestors" AS "namespaces") AND (members.access_level > 10)
Related issue: #121581
Related discussions: !22121 (comment 264034946) #121581 (comment 279757581)
Screenshots
Does this MR meet the acceptance criteria?
Conformity
- Changelog entry
- Documentation (if required)
-
Code review guidelines -
Merge request performance guidelines -
Style guides -
Database guides -
Separation of EE specific content
Availability and Testing
-
Review and add/update tests for this feature/bug. Consider all test levels. See the Test Planning Process. - Tested in all supported browsers
- Informed Infrastructure department of a default or new setting change, if applicable per definition of done
Security
If this MR contains changes to processing or storing of credentials or tokens, authorization and authentication methods and other items described in the security review guidelines:
- Label as security and @ mention
@gitlab-com/gl-security/appsec
- The MR includes necessary changes to maintain consistency between UI, API, email, or other methods
- Security reports checked/validated by a reviewer from the AppSec team
Edited by Reuben Pereira