Improve CI usage quotas performance
Summary
It was highlighted via the grouputilization error budgets that the CI usage charts within the Usage Quotas page is under performing, causing grouputilization group to consume their monthly error budget.
In the last 21 days, we've had over 600 GraphQL queries for getCiMinutesUsageGroup
that exceed the 5s apdex (logs)
Sample JSON log
{
"_index": "pubsub-rails-inf-gprd-014676",
"_type": "_doc",
"_id": "fEXfM4UBOnt9WbS0AjZ6",
"_version": 1,
"_score": 1,
"_ignored": [
"json.params.value.keyword"
],
"_source": {
"@timestamp": "2022-12-21T08:48:50.476Z",
"kubernetes": {
"pod_name": "gitlab-webservice-api-68c8bdf648-nq4vj",
"container_image": "dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-webservice-ee:15-7-202212201620-e90e9805cff",
"region": "us-east1-d",
"container_name": "webservice",
"host": "gke-gprd-us-east1-d-generic-2-08188faa-h9jq",
"namespace_name": "gitlab",
"pod_ip": "10.68.31.74"
},
"json": {
"db_ci_duration_s": 0,
"redis_sessions_read_bytes": 186,
"db_ci_replica_wal_count": 0,
"user_id": 10111002,
"response_bytes": 140079,
"view_duration_s": 0.01457,
"db_ci_wal_count": 0,
"db_primary_count": 0,
"db_ci_replica_count": 24,
"db_primary_wal_count": 0,
"rack_attack_redis_duration_s": 0.000740360002964735,
"ua": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0",
"db_ci_replica_wal_cached_count": 0,
"redis_cache_write_bytes": 39436,
"params": [
{
"key": "_json",
"value": "[{\"operationName\"=>\"getCiMinutesUsageGroup\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"query getCiMinutesUsageGroup($namespaceId: NamespaceID, $first: Int, $last: Int, $after: String, $before: String) {\\n ciMinutesUsage(namespaceId: $namespaceId) {\\n nodes {\\n month\\n monthIso8601\\n minutes\\n sharedRunnersDuration\\n projects(first: $first, last: $last, after: $after, before: $before) {\\n nodes {\\n minutes\\n sharedRunnersDuration\\n project {\\n id\\n name\\n nameWithNamespace\\n avatarUrl\\n webUrl\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n ...PageInfo\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\\nfragment PageInfo on PageInfo {\\n hasNextPage\\n hasPreviousPage\\n startCursor\\n endCursor\\n __typename\\n}\\n\"}]"
},
{
"value": "{\"_json\"=>[{\"operationName\"=>\"getCiMinutesUsageGroup\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"query getCiMinutesUsageGroup($namespaceId: NamespaceID, $first: Int, $last: Int, $after: String, $before: String) {\\n ciMinutesUsage(namespaceId: $namespaceId) {\\n nodes {\\n month\\n monthIso8601\\n minutes\\n sharedRunnersDuration\\n projects(first: $first, last: $last, after: $after, before: $before) {\\n nodes {\\n minutes\\n sharedRunnersDuration\\n project {\\n id\\n name\\n nameWithNamespace\\n avatarUrl\\n webUrl\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n ...PageInfo\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\\nfragment PageInfo on PageInfo {\\n hasNextPage\\n hasPreviousPage\\n startCursor\\n endCursor\\n __typename\\n}\\n\"}]}",
"key": "graphql"
}
],
"mem_mallocs": 438428,
"cf_ray": "77cf5bb28fd49e3e-ATL",
"db_ci_count": 0,
"redis_allowed_cross_slot_calls": 1,
"correlation_id": "c409b0bad450ab1bdbd7a1fd2aecbf41",
"redis_cache_read_bytes": 3559,
"db_count": 2045,
"db_main_replica_wal_cached_count": 0,
"redis_write_bytes": 40892,
"meta.caller_id": "GraphqlController#execute",
"db_main_replica_count": 2021,
"redis_rate_limiting_write_bytes": 160,
"db_main_cached_count": 0,
"db_write_count": 0,
"db_ci_cached_count": 0,
"redis_read_bytes": 3749,
"redis_shared_state_duration_s": 0.000683,
"redis_rate_limiting_duration_s": 0.000523,
"tier": "sv",
"redis_calls": 291,
"rack_attack_redis_count": 1,
"redis_sessions_allowed_cross_slot_calls": 1,
"action": "execute",
"meta.user": "eric_tan",
"redis_sessions_calls": 4,
"redis_sessions_duration_s": 0.000911,
"request_urgency": "low",
"db_replica_duration_s": 0.913,
"meta.client_id": "user/10111002",
"db_main_replica_cached_count": 1696,
"queue_duration_s": 0.01687,
"db_replica_wal_cached_count": 0,
"method": "POST",
"component": "gitlab",
"worker_id": "puma_5",
"db_main_count": 0,
"gitaly_duration_s": 2.479788,
"controller": "GraphqlController",
"db_duration_s": 0.88624,
"status": 200,
"stage": "main",
"redis_cache_calls": 283,
"redis_cache_duration_s": 0.296575,
"db_ci_replica_duration_s": 0.435,
"db_replica_count": 2045,
"username": "eric_tan",
"db_main_duration_s": 0,
"db_main_replica_duration_s": 0.478,
"format": "*/*",
"redis_sessions_write_bytes": 1178,
"db_replica_cached_count": 1696,
"redis_shared_state_write_bytes": 118,
"meta.user_id": 10111002,
"cpu_s": 3.739482,
"db_main_replica_wal_count": 0,
"shard": "default",
"db_cached_count": 1696,
"redis_duration_s": 0.29869199999999996,
"duration_s": 6.58444,
"rate_limiting_gates": [],
"mem_objects": 1749250,
"db_main_wal_count": 0,
"type": "api",
"db_main_wal_cached_count": 0,
"redis_rate_limiting_read_bytes": 4,
"db_ci_wal_cached_count": 0,
"pid": 1162,
"time": "2022-12-21T08:48:27.676Z",
"target_duration_s": 5,
"db_primary_wal_cached_count": 0,
"feature_flag_states": [],
"meta.remote_ip": "203.123.4.58",
"tag": "api-rails.var.log.containers.gitlab-webservice-api-68c8bdf648-nq4vj_gitlab_webservice-15865865fb9dd5ac4f39394218898a440a0e11360814c33900c7c00e26018943.log",
"environment": "gprd",
"db_replica_wal_count": 0,
"db_primary_duration_s": 0,
"redis_shared_state_calls": 2,
"mem_bytes": 100871672,
"meta.feature_category": "subscription_cost_management",
"db_ci_replica_cached_count": 0,
"remote_ip": "203.123.4.58",
"subcomponent": "production_json",
"db_primary_cached_count": 0,
"gitaly_calls": 168,
"mem_total_bytes": 170841672,
"level": "info",
"path": "/api/graphql",
"graphql": [
{
"depth": 6,
"complexity": 28,
"used_fields": [
"CiMinutesNamespaceMonthlyUsage.month",
"CiMinutesNamespaceMonthlyUsage.monthIso8601",
"CiMinutesNamespaceMonthlyUsage.minutes",
"CiMinutesNamespaceMonthlyUsage.sharedRunnersDuration",
"CiMinutesProjectMonthlyUsage.minutes",
"CiMinutesProjectMonthlyUsage.sharedRunnersDuration",
"Project.id",
"Project.name",
"Project.nameWithNamespace",
"Project.avatarUrl",
"Project.webUrl",
"Project.__typename",
"CiMinutesProjectMonthlyUsage.project",
"CiMinutesProjectMonthlyUsage.__typename",
"CiMinutesProjectMonthlyUsageConnection.nodes",
"PageInfo.hasNextPage",
"PageInfo.hasPreviousPage",
"PageInfo.startCursor",
"PageInfo.endCursor",
"PageInfo.__typename",
"CiMinutesProjectMonthlyUsageConnection.pageInfo",
"CiMinutesProjectMonthlyUsageConnection.__typename",
"CiMinutesNamespaceMonthlyUsage.projects",
"CiMinutesNamespaceMonthlyUsage.__typename",
"CiMinutesNamespaceMonthlyUsageConnection.nodes",
"CiMinutesNamespaceMonthlyUsageConnection.__typename",
"Query.ciMinutesUsage"
],
"used_deprecated_fields": [],
"variables": "{\"namespaceId\"=>\"gid://gitlab/Group/8211820\", \"first\"=>20}",
"operation_name": "getCiMinutesUsageGroup"
}
],
"redis_rate_limiting_calls": 2
},
"type": "pubsubbeat-pubsub-rails-inf-gprd-5f5c54989f-cgktv",
"host": {
"name": "pubsubbeat-pubsub-rails-inf-gprd-5f5c54989f-cgktv"
},
"publish_time": "2022-12-21T08:48:50.352Z"
},
"fields": {
"json.type.keyword": [
"api"
],
"json.redis_sessions_allowed_cross_slot_calls": [
1
],
"json.db_replica_wal_cached_count": [
0
],
"json.mem_mallocs": [
438428
],
"json.redis_duration_s": [
0.298692
],
"host.name.keyword": [
"pubsubbeat-pubsub-rails-inf-gprd-5f5c54989f-cgktv"
],
"json.db_primary_cached_count": [
0
],
"json.params.key": [
"_json",
"graphql"
],
"json.db_replica_count": [
2045
],
"json.mem_total_bytes": [
170841672
],
"type": [
"pubsubbeat-pubsub-rails-inf-gprd-5f5c54989f-cgktv"
],
"json.db_main_wal_count": [
0
],
"json.redis_cache_read_bytes": [
3559
],
"json.mem_bytes": [
100871672
],
"json.redis_cache_write_bytes": [
39436
],
"publish_time": [
"2022-12-21T08:48:50.352Z"
],
"json.db_primary_wal_cached_count": [
0
],
"json.graphql.complexity.keyword": [
"28"
],
"json.params.value": [
"[{\"operationName\"=>\"getCiMinutesUsageGroup\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"query getCiMinutesUsageGroup($namespaceId: NamespaceID, $first: Int, $last: Int, $after: String, $before: String) {\\n ciMinutesUsage(namespaceId: $namespaceId) {\\n nodes {\\n month\\n monthIso8601\\n minutes\\n sharedRunnersDuration\\n projects(first: $first, last: $last, after: $after, before: $before) {\\n nodes {\\n minutes\\n sharedRunnersDuration\\n project {\\n id\\n name\\n nameWithNamespace\\n avatarUrl\\n webUrl\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n ...PageInfo\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\\nfragment PageInfo on PageInfo {\\n hasNextPage\\n hasPreviousPage\\n startCursor\\n endCursor\\n __typename\\n}\\n\"}]",
"{\"_json\"=>[{\"operationName\"=>\"getCiMinutesUsageGroup\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"query getCiMinutesUsageGroup($namespaceId: NamespaceID, $first: Int, $last: Int, $after: String, $before: String) {\\n ciMinutesUsage(namespaceId: $namespaceId) {\\n nodes {\\n month\\n monthIso8601\\n minutes\\n sharedRunnersDuration\\n projects(first: $first, last: $last, after: $after, before: $before) {\\n nodes {\\n minutes\\n sharedRunnersDuration\\n project {\\n id\\n name\\n nameWithNamespace\\n avatarUrl\\n webUrl\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n ...PageInfo\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\\nfragment PageInfo on PageInfo {\\n hasNextPage\\n hasPreviousPage\\n startCursor\\n endCursor\\n __typename\\n}\\n\"}]}"
],
"json.cf_ray": [
"77cf5bb28fd49e3e-ATL"
],
"json.target_duration_s": [
5
],
"json.meta.caller_id": [
"GraphqlController#execute"
],
"json.graphql.variables.keyword": [
"{\"namespaceId\"=>\"gid://gitlab/Group/8211820\", \"first\"=>20}"
],
"json.username": [
"eric_tan"
],
"json.subcomponent.keyword": [
"production_json"
],
"json.db_replica_cached_count": [
1696
],
"json.db_ci_replica_count": [
24
],
"json.user_id": [
"10111002"
],
"json.db_ci_cached_count": [
0
],
"json.request_urgency": [
"low"
],
"json.environment": [
"gprd"
],
"json.db_primary_duration_s": [
0
],
"json.graphql.operation_name.keyword": [
"getCiMinutesUsageGroup"
],
"json.queue_duration_s": [
0.01687
],
"json.rack_attack_redis_duration_s": [
0.00074036
],
"json.rack_attack_redis_count": [
1
],
"kubernetes.pod_ip": [
"10.68.31.74"
],
"json.type": [
"api"
],
"json.meta.feature_category": [
"subscription_cost_management"
],
"json.username.keyword": [
"eric_tan"
],
"json.db_main_wal_cached_count": [
0
],
"json.worker_id.keyword": [
"puma_5"
],
"json.environment.keyword": [
"gprd"
],
"json.meta.user": [
"eric_tan"
],
"json.duration_s": [
6.58444
],
"json.path.keyword": [
"/api/graphql"
],
"json.redis_read_bytes": [
3749
],
"json.meta.user.keyword": [
"eric_tan"
],
"json.gitaly_duration_s": [
2.479788
],
"json.action": [
"execute"
],
"json.gitaly_calls": [
168
],
"json.redis_sessions_duration_s": [
0.000911
],
"json.level.keyword": [
"info"
],
"json.meta.feature_category.keyword": [
"subscription_cost_management"
],
"kubernetes.namespace_name.keyword": [
"gitlab"
],
"json.redis_shared_state_calls": [
2
],
"json.redis_cache_duration_s": [
0.296575
],
"json.redis_calls": [
291
],
"json.redis_allowed_cross_slot_calls": [
1
],
"json.meta.client_id.keyword": [
"user/10111002"
],
"json.graphql.used_fields": [
"CiMinutesNamespaceMonthlyUsage.month",
"CiMinutesNamespaceMonthlyUsage.monthIso8601",
"CiMinutesNamespaceMonthlyUsage.minutes",
"CiMinutesNamespaceMonthlyUsage.sharedRunnersDuration",
"CiMinutesProjectMonthlyUsage.minutes",
"CiMinutesProjectMonthlyUsage.sharedRunnersDuration",
"Project.id",
"Project.name",
"Project.nameWithNamespace",
"Project.avatarUrl",
"Project.webUrl",
"Project.__typename",
"CiMinutesProjectMonthlyUsage.project",
"CiMinutesProjectMonthlyUsage.__typename",
"CiMinutesProjectMonthlyUsageConnection.nodes",
"PageInfo.hasNextPage",
"PageInfo.hasPreviousPage",
"PageInfo.startCursor",
"PageInfo.endCursor",
"PageInfo.__typename",
"CiMinutesProjectMonthlyUsageConnection.pageInfo",
"CiMinutesProjectMonthlyUsageConnection.__typename",
"CiMinutesNamespaceMonthlyUsage.projects",
"CiMinutesNamespaceMonthlyUsage.__typename",
"CiMinutesNamespaceMonthlyUsageConnection.nodes",
"CiMinutesNamespaceMonthlyUsageConnection.__typename",
"Query.ciMinutesUsage"
],
"json.db_ci_replica_wal_count": [
0
],
"json.redis_sessions_write_bytes": [
1178
],
"json.format": [
"*/*"
],
"json.db_primary_count": [
0
],
"json.db_main_replica_duration_s": [
0.478
],
"json.db_main_cached_count": [
0
],
"json.redis_rate_limiting_duration_s": [
0.000523
],
"json.ua.keyword": [
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0"
],
"json.db_write_count": [
0
],
"json.redis_shared_state_write_bytes": [
118
],
"json.redis_rate_limiting_write_bytes": [
160
],
"json.shard.keyword": [
"default"
],
"@timestamp": [
"2022-12-21T08:48:50.476Z"
],
"json.worker_id": [
"puma_5"
],
"json.db_ci_duration_s": [
0
],
"kubernetes.pod_name": [
"gitlab-webservice-api-68c8bdf648-nq4vj"
],
"json.db_replica_duration_s": [
0.913
],
"json.meta.remote_ip": [
"203.123.4.58"
],
"json.method.keyword": [
"POST"
],
"json.db_main_count": [
0
],
"json.graphql.complexity": [
"28"
],
"json.db_main_replica_wal_count": [
0
],
"json.redis_shared_state_duration_s": [
0.000683
],
"json.graphql.operation_name": [
"getCiMinutesUsageGroup"
],
"json.redis_rate_limiting_calls": [
2
],
"kubernetes.container_image.keyword": [
"dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-webservice-ee:15-7-202212201620-e90e9805cff"
],
"json.db_main_replica_wal_cached_count": [
0
],
"json.redis_cache_calls": [
283
],
"type.keyword": [
"pubsubbeat-pubsub-rails-inf-gprd-5f5c54989f-cgktv"
],
"kubernetes.container_image": [
"dev.gitlab.org:5005/gitlab/charts/components/images/gitlab-webservice-ee:15-7-202212201620-e90e9805cff"
],
"json.db_cached_count": [
1696
],
"host.name": [
"pubsubbeat-pubsub-rails-inf-gprd-5f5c54989f-cgktv"
],
"json.component.keyword": [
"gitlab"
],
"json.db_ci_wal_cached_count": [
0
],
"kubernetes.host.keyword": [
"gke-gprd-us-east1-d-generic-2-08188faa-h9jq"
],
"json.tier.keyword": [
"sv"
],
"json.redis_sessions_calls": [
4
],
"json.graphql.variables": [
"{\"namespaceId\"=>\"gid://gitlab/Group/8211820\", \"first\"=>20}"
],
"json.tag": [
"api-rails.var.log.containers.gitlab-webservice-api-68c8bdf648-nq4vj_gitlab_webservice-15865865fb9dd5ac4f39394218898a440a0e11360814c33900c7c00e26018943.log"
],
"json.cf_ray.keyword": [
"77cf5bb28fd49e3e-ATL"
],
"kubernetes.container_name": [
"webservice"
],
"json.level": [
"info"
],
"json.path": [
"/api/graphql"
],
"kubernetes.pod_ip.keyword": [
"10.68.31.74"
],
"json.redis_rate_limiting_read_bytes": [
4
],
"json.db_replica_wal_count": [
0
],
"json.response_bytes": [
140079
],
"kubernetes.container_name.keyword": [
"webservice"
],
"json.pid": [
1162
],
"json.ua": [
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0"
],
"json.db_primary_wal_count": [
0
],
"json.remote_ip": [
"203.123.4.58"
],
"json.stage": [
"main"
],
"json.db_count": [
2045
],
"json.graphql.used_fields.keyword": [
"CiMinutesNamespaceMonthlyUsage.month",
"CiMinutesNamespaceMonthlyUsage.monthIso8601",
"CiMinutesNamespaceMonthlyUsage.minutes",
"CiMinutesNamespaceMonthlyUsage.sharedRunnersDuration",
"CiMinutesProjectMonthlyUsage.minutes",
"CiMinutesProjectMonthlyUsage.sharedRunnersDuration",
"Project.id",
"Project.name",
"Project.nameWithNamespace",
"Project.avatarUrl",
"Project.webUrl",
"Project.__typename",
"CiMinutesProjectMonthlyUsage.project",
"CiMinutesProjectMonthlyUsage.__typename",
"CiMinutesProjectMonthlyUsageConnection.nodes",
"PageInfo.hasNextPage",
"PageInfo.hasPreviousPage",
"PageInfo.startCursor",
"PageInfo.endCursor",
"PageInfo.__typename",
"CiMinutesProjectMonthlyUsageConnection.pageInfo",
"CiMinutesProjectMonthlyUsageConnection.__typename",
"CiMinutesNamespaceMonthlyUsage.projects",
"CiMinutesNamespaceMonthlyUsage.__typename",
"CiMinutesNamespaceMonthlyUsageConnection.nodes",
"CiMinutesNamespaceMonthlyUsageConnection.__typename",
"Query.ciMinutesUsage"
],
"json.db_main_replica_count": [
2021
],
"kubernetes.region": [
"us-east1-d"
],
"json.request_urgency.keyword": [
"low"
],
"json.redis_write_bytes": [
40892
],
"json.meta.caller_id.keyword": [
"GraphqlController#execute"
],
"json.component": [
"gitlab"
],
"json.cpu_s": [
3.739482
],
"json.tag.keyword": [
"api-rails.var.log.containers.gitlab-webservice-api-68c8bdf648-nq4vj_gitlab_webservice-15865865fb9dd5ac4f39394218898a440a0e11360814c33900c7c00e26018943.log"
],
"json.graphql.depth": [
6
],
"json.params.key.keyword": [
"_json",
"graphql"
],
"hour": [
8
],
"json.controller": [
"GraphqlController"
],
"json.subcomponent": [
"production_json"
],
"json.time": [
"2022-12-21T08:48:27.676Z"
],
"json.redis_sessions_read_bytes": [
186
],
"json.db_main_duration_s": [
0
],
"kubernetes.host": [
"gke-gprd-us-east1-d-generic-2-08188faa-h9jq"
],
"json.db_main_replica_cached_count": [
1696
],
"kubernetes.pod_name.keyword": [
"gitlab-webservice-api-68c8bdf648-nq4vj"
],
"json.mem_objects": [
1749250
],
"json.shard": [
"default"
],
"json.stage.keyword": [
"main"
],
"json.correlation_id.keyword": [
"c409b0bad450ab1bdbd7a1fd2aecbf41"
],
"json.method": [
"POST"
],
"json.correlation_id": [
"c409b0bad450ab1bdbd7a1fd2aecbf41"
],
"json.db_ci_replica_cached_count": [
0
],
"kubernetes.region.keyword": [
"us-east1-d"
],
"json.db_ci_replica_wal_cached_count": [
0
],
"json.db_ci_count": [
0
],
"json.meta.remote_ip.keyword": [
"203.123.4.58"
],
"json.db_ci_replica_duration_s": [
0.435
],
"json.tier": [
"sv"
],
"ua_string_base": [
"Mozilla"
],
"kubernetes.namespace_name": [
"gitlab"
],
"json.db_duration_s": [
0.88624
],
"json.db_ci_wal_count": [
0
],
"json.remote_ip.keyword": [
"203.123.4.58"
],
"json.status": [
200
],
"json.action.keyword": [
"execute"
],
"json.meta.user_id": [
10111002
],
"json.format.keyword": [
"*/*"
],
"json.view_duration_s": [
0.01457
],
"json.controller.keyword": [
"GraphqlController"
],
"json.meta.client_id": [
"user/10111002"
]
},
"ignored_field_values": {
"json.params.value.keyword": [
"[{\"operationName\"=>\"getCiMinutesUsageGroup\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"query getCiMinutesUsageGroup($namespaceId: NamespaceID, $first: Int, $last: Int, $after: String, $before: String) {\\n ciMinutesUsage(namespaceId: $namespaceId) {\\n nodes {\\n month\\n monthIso8601\\n minutes\\n sharedRunnersDuration\\n projects(first: $first, last: $last, after: $after, before: $before) {\\n nodes {\\n minutes\\n sharedRunnersDuration\\n project {\\n id\\n name\\n nameWithNamespace\\n avatarUrl\\n webUrl\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n ...PageInfo\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\\nfragment PageInfo on PageInfo {\\n hasNextPage\\n hasPreviousPage\\n startCursor\\n endCursor\\n __typename\\n}\\n\"}]",
"{\"_json\"=>[{\"operationName\"=>\"getCiMinutesUsageGroup\", \"variables\"=>\"[FILTERED]\", \"query\"=>\"query getCiMinutesUsageGroup($namespaceId: NamespaceID, $first: Int, $last: Int, $after: String, $before: String) {\\n ciMinutesUsage(namespaceId: $namespaceId) {\\n nodes {\\n month\\n monthIso8601\\n minutes\\n sharedRunnersDuration\\n projects(first: $first, last: $last, after: $after, before: $before) {\\n nodes {\\n minutes\\n sharedRunnersDuration\\n project {\\n id\\n name\\n nameWithNamespace\\n avatarUrl\\n webUrl\\n __typename\\n }\\n __typename\\n }\\n pageInfo {\\n ...PageInfo\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n}\\n\\nfragment PageInfo on PageInfo {\\n hasNextPage\\n hasPreviousPage\\n startCursor\\n endCursor\\n __typename\\n}\\n\"}]}"
]
}
}
Apdex/Error budgets aside, this is likely not a great user experience for our customers.
Improvements
It appears as though when rendering the CI usage tab, we perform this GraphQL query and fetch all the CI usage data for all projects within the namespace.
For some namespaces, this can be a large amount of data which would explain why the query is slow for some requests but not all (~600/~8000).
The chart being rendered has dropdown options for year and month, so we could consider refactoring this query to only fetch the data for the selected year/month combination:
Involved components
If feasible, we'll need to :
- Modify the frontend to query for the selected params (date)
- Modify the backend GraphQL type to support the filter option by date
For 2, the underlying data appears to come from Ci::Namespace::MonthlyUsages
which has a date
column, so it should be possible to filter by that attribute.
Optional: Intended side effects
We should hopefully see a reduction in the number of queries that exceed the 5 second duration limit, thus reducing our error budget usage and improving the experience for our customers.
Some additional implementation details can be found in #386442 (comment 1226284518)