Skip to content

Resolve "Ancestor group scopes with limits"

Alex Pooley requested to merge 346700-ancestor-group-scopes-with-limits into master

What does this MR do and why?

Wrap the ancestor group scope in a CTE. This allows proper handling of LIMIT and OFFSET queries.

First noticed at !74606 (comment 739551716)

For example, the current query of Group.where(id: [1,2,3]).limit(1).self_and_ancestors is translated to the SQL equivalent of Group.where(id: [1,2,3]).self_and_ancestors.limit(1).

This work sits behind a feature flag use_traversal_ids_for_ancestor_scopes currently 100% enabled.

Query Plan. https://postgres.ai/console/gitlab/gitlab-production-tunnel-pg12/sessions/7613/commands/27024

New Query

WITH "base_cte" AS MATERIALIZED (
  SELECT 
    "namespaces"."id", 
    "namespaces"."traversal_ids" 
  FROM 
    "namespaces" 
  WHERE 
    "namespaces"."type" = 'Group' 
    AND "namespaces"."id" IN (1, 2, 3) 
  LIMIT 
    1
), "ancestors_cte" AS MATERIALIZED (
  SELECT 
    unnest(traversal_ids) as ancestor_id 
  FROM 
    "base_cte"
) 
SELECT 
  DISTINCT "namespaces"."id", 
  "namespaces"."name", 
  "namespaces"."path", 
  "namespaces"."owner_id", 
  "namespaces"."created_at", 
  "namespaces"."updated_at", 
  "namespaces"."type", 
  "namespaces"."description", 
  "namespaces"."avatar", 
  "namespaces"."membership_lock", 
  "namespaces"."share_with_group_lock", 
  "namespaces"."visibility_level", 
  "namespaces"."request_access_enabled", 
  "namespaces"."ldap_sync_status", 
  "namespaces"."ldap_sync_error", 
  "namespaces"."ldap_sync_last_update_at", 
  "namespaces"."ldap_sync_last_successful_update_at", 
  "namespaces"."ldap_sync_last_sync_at", 
  "namespaces"."description_html", 
  "namespaces"."lfs_enabled", 
  "namespaces"."parent_id", 
  "namespaces"."shared_runners_minutes_limit", 
  "namespaces"."repository_size_limit", 
  "namespaces"."require_two_factor_authentication", 
  "namespaces"."two_factor_grace_period", 
  "namespaces"."cached_markdown_version", 
  "namespaces"."project_creation_level", 
  "namespaces"."runners_token", 
  "namespaces"."file_template_project_id", 
  "namespaces"."saml_discovery_token", 
  "namespaces"."runners_token_encrypted", 
  "namespaces"."custom_project_templates_group_id", 
  "namespaces"."auto_devops_enabled", 
  "namespaces"."extra_shared_runners_minutes_limit", 
  "namespaces"."last_ci_minutes_notification_at", 
  "namespaces"."last_ci_minutes_usage_notification_level", 
  "namespaces"."subgroup_creation_level", 
  "namespaces"."emails_disabled", 
  "namespaces"."max_pages_size", 
  "namespaces"."max_artifacts_size", 
  "namespaces"."mentions_disabled", 
  "namespaces"."default_branch_protection", 
  "namespaces"."unlock_membership_to_ldap", 
  "namespaces"."max_personal_access_token_lifetime", 
  "namespaces"."push_rule_id", 
  "namespaces"."shared_runners_enabled", 
  "namespaces"."allow_descendants_override_disabled_shared_runners", 
  "namespaces"."traversal_ids" 
FROM 
  "ancestors_cte", 
  "namespaces" 
WHERE 
  "namespaces"."type" = 'Group' 
  AND "namespaces"."id" = "ancestors_cte"."ancestor_id"

Old Query

SELECT 
  "namespaces"."id", 
  "namespaces"."name", 
  "namespaces"."path", 
  "namespaces"."owner_id", 
  "namespaces"."created_at", 
  "namespaces"."updated_at", 
  "namespaces"."type", 
  "namespaces"."description", 
  "namespaces"."avatar", 
  "namespaces"."membership_lock", 
  "namespaces"."share_with_group_lock", 
  "namespaces"."visibility_level", 
  "namespaces"."request_access_enabled", 
  "namespaces"."ldap_sync_status", 
  "namespaces"."ldap_sync_error", 
  "namespaces"."ldap_sync_last_update_at", 
  "namespaces"."ldap_sync_last_successful_update_at", 
  "namespaces"."ldap_sync_last_sync_at", 
  "namespaces"."description_html", 
  "namespaces"."lfs_enabled", 
  "namespaces"."parent_id", 
  "namespaces"."shared_runners_minutes_limit", 
  "namespaces"."repository_size_limit", 
  "namespaces"."require_two_factor_authentication", 
  "namespaces"."two_factor_grace_period", 
  "namespaces"."cached_markdown_version", 
  "namespaces"."project_creation_level", 
  "namespaces"."runners_token", 
  "namespaces"."file_template_project_id", 
  "namespaces"."saml_discovery_token", 
  "namespaces"."runners_token_encrypted", 
  "namespaces"."custom_project_templates_group_id", 
  "namespaces"."auto_devops_enabled", 
  "namespaces"."extra_shared_runners_minutes_limit", 
  "namespaces"."last_ci_minutes_notification_at", 
  "namespaces"."last_ci_minutes_usage_notification_level", 
  "namespaces"."subgroup_creation_level", 
  "namespaces"."emails_disabled", 
  "namespaces"."max_pages_size", 
  "namespaces"."max_artifacts_size", 
  "namespaces"."mentions_disabled", 
  "namespaces"."default_branch_protection", 
  "namespaces"."unlock_membership_to_ldap", 
  "namespaces"."max_personal_access_token_lifetime", 
  "namespaces"."push_rule_id", 
  "namespaces"."shared_runners_enabled", 
  "namespaces"."allow_descendants_override_disabled_shared_runners", 
  "namespaces"."traversal_ids" 
FROM 
  (
    SELECT 
      "namespaces"."id", 
      "namespaces"."name", 
      "namespaces"."path", 
      "namespaces"."owner_id", 
      "namespaces"."created_at", 
      "namespaces"."updated_at", 
      "namespaces"."type", 
      "namespaces"."description", 
      "namespaces"."avatar", 
      "namespaces"."membership_lock", 
      "namespaces"."share_with_group_lock", 
      "namespaces"."visibility_level", 
      "namespaces"."request_access_enabled", 
      "namespaces"."ldap_sync_status", 
      "namespaces"."ldap_sync_error", 
      "namespaces"."ldap_sync_last_update_at", 
      "namespaces"."ldap_sync_last_successful_update_at", 
      "namespaces"."ldap_sync_last_sync_at", 
      "namespaces"."description_html", 
      "namespaces"."lfs_enabled", 
      "namespaces"."parent_id", 
      "namespaces"."shared_runners_minutes_limit", 
      "namespaces"."repository_size_limit", 
      "namespaces"."require_two_factor_authentication", 
      "namespaces"."two_factor_grace_period", 
      "namespaces"."cached_markdown_version", 
      "namespaces"."project_creation_level", 
      "namespaces"."runners_token", 
      "namespaces"."file_template_project_id", 
      "namespaces"."saml_discovery_token", 
      "namespaces"."runners_token_encrypted", 
      "namespaces"."custom_project_templates_group_id", 
      "namespaces"."auto_devops_enabled", 
      "namespaces"."extra_shared_runners_minutes_limit", 
      "namespaces"."last_ci_minutes_notification_at", 
      "namespaces"."last_ci_minutes_usage_notification_level", 
      "namespaces"."subgroup_creation_level", 
      "namespaces"."emails_disabled", 
      "namespaces"."max_pages_size", 
      "namespaces"."max_artifacts_size", 
      "namespaces"."mentions_disabled", 
      "namespaces"."default_branch_protection", 
      "namespaces"."unlock_membership_to_ldap", 
      "namespaces"."max_personal_access_token_lifetime", 
      "namespaces"."push_rule_id", 
      "namespaces"."shared_runners_enabled", 
      "namespaces"."allow_descendants_override_disabled_shared_runners", 
      "namespaces"."traversal_ids" 
    FROM 
      "namespaces" 
    WHERE 
      "namespaces"."type" = 'Group' 
      AND "namespaces"."id" IN (
        SELECT 
          unnest(traversal_ids) 
        FROM 
          "namespaces" 
        WHERE 
          "namespaces"."type" = 'Group' 
          AND "namespaces"."id" IN (1, 2, 3) 
        LIMIT 
          1
      )
  ) namespaces 
WHERE 
  "namespaces"."type" = 'Group'

How to set up and validate locally

Feature.enable :use_traversal_ids
Feature.enable :use_traversal_ids_for_ancestor_scopes
Group.where(id: [1,2,3]).limit(1).self_and_ancestors

MR acceptance checklist

This checklist encourages us to confirm any changes have been analyzed to reduce risks in quality, performance, reliability, security, and maintainability.

Related to #346700 (closed)

Edited by Alex Pooley

Merge request reports