Scheduled group deletion fails with "Project 1234 can't be deleted" error when "Admin Mode" enabled and "Allowed to delete projects" set to Admins only
Summary
When "Admin mode" is enabled and "Allowed to delete projects" is set to Admins only, and an administrator deletes a group which is deferred for 7 days the adjourned_group_deletion_worker
sidekiq cron job is unable to delete the group's projects.
This occurs in multiple recent versions including 15.10.1, 15.8.3 and 15.7.6.
This issue was raised by a Premium customer in ZD internal link
A workaround is to temporarily disable admin mode and manually run the adjourned_group_deletion_worker
job.
Steps to reproduce
- enable "Admin mode" under Admin, Settings, General, Sign-in restrictions
- set "Allowed to delete projects" to "Administrators" under Admin, Settings, General, Visibility and access controls
- set "Deletion Protection" to "Keep deleted groups only for 7 days" under Admin, Settings, General, Visibility and access controls
- as an Admin user, create a group with a project, then remove the group
- update the group deletion schedule date to more than 7 days in the past, via the console:
pp GroupDeletionSchedule.all
gds=GroupDeletionSchedule.find(<group_id>)
gds.update!(marked_for_deletion_on: '2023-03-01')
- run the
adjourned_group_deletion_worker
sidekiq cron job by selecting "Enqueue Now" under Admin, Monitoring, Background Jobs, Cron - observe that the group is not deleted and the group deletion schedule remains
Example Project
What is the current bug behavior?
The deletion of the project within the group fails with a Project 1234 can't be deleted
error.
This is possibly triggered in the call to
return false unless can?(current_user, :remove_project, project)
in Projects::DestroyService
which is called by Groups::DestroyService
What is the expected correct behavior?
Groups should be deleted successfully when the above configuration settings are in effect.
Relevant logs and/or screenshots
{
"severity": "WARN",
"time": "2023-02-07T20:54:50.595Z",
"retry": 3,
"queue": "group_destroy",
"version": 0,
"backtrace": 5,
"class": "GroupDestroyWorker",
"args": [
"2113",
"6"
],
"jid": "b76f1595f4df1c77d0439137",
"created_at": "2023-02-07T20:53:49.279Z",
"correlation_id": "ee69e73595507d6b39cb7f9c139dfbd7",
"meta.caller_id": "AdjournedGroupDeletionWorker",
"meta.root_caller_id": "Cronjob",
"meta.feature_category": "subgroups",
"meta.user": "user1",
"meta.user_id": 6,
"meta.root_namespace": "group10",
"meta.client_id": "user/6",
"worker_data_consistency": "always",
"idempotency_key": "resque:gitlab:duplicate:group_destroy:1305bddc7c98eecb39c064fc41f7189c20e2c1b3763a8d46e61a04be808347bd",
"size_limiter": "validated",
"enqueued_at": "2023-02-07T20:54:50.537Z",
"error_message": "Project 1234 can't be deleted",
"error_class": "Groups::DestroyService::DestroyError",
"failed_at": "2023-02-07T20:53:49.327Z",
"retry_count": 1,
"error_backtrace": "eJzNkd0KwjAMhV9l7MYbbdwbt90MVLz0InCSwJfD\nySkHTxG0CZYL4F0HHikaiR40ub7zoNAHckMzzxmJutzVps0uwjp5y5LCB8o+\n4CpfL2gRW+UIRN8qi0C9GKB2baAKHl\nwbg2+bCoR/myUFWjBeTy+idnf8yuOE6E95EhLjmp/4IqPrzh/AS6/qdI\n",
"retried_at": "2023-02-07T20:54:17.408Z",
"job_size_bytes": 8,
"pid": 27,
"message": "GroupDestroyWorker JID-b76f1595f4df1c77d0439137: fail: 0.055712 sec",
"job_status": "fail",
"scheduling_latency_s": 0.001731,
"redis_calls": 1,
"redis_duration_s": 0.000393,
"redis_read_bytes": 1,
"redis_write_bytes": 129,
"redis_queues_calls": 1,
"redis_queues_duration_s": 0.000393,
"redis_queues_read_bytes": 1,
"redis_queues_write_bytes": 129,
"db_count": 11,
"db_write_count": 0,
"db_cached_count": 1,
"db_replica_count": 0,
"db_primary_count": 11,
"db_main_count": 11,
"db_main_replica_count": 0,
"db_replica_cached_count": 0,
"db_primary_cached_count": 1,
"db_main_cached_count": 1,
"db_main_replica_cached_count": 0,
"db_replica_wal_count": 0,
"db_primary_wal_count": 0,
"db_main_wal_count": 0,
"db_main_replica_wal_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_main_wal_cached_count": 0,
"db_main_replica_wal_cached_count": 0,
"db_replica_duration_s": 0,
"db_primary_duration_s": 0.021,
"db_main_duration_s": 0.021,
"db_main_replica_duration_s": 0,
"cpu_s": 0.030459,
"mem_objects": 11360,
"mem_bytes": 1384279,
"mem_mallocs": 3509,
"mem_total_bytes": 1838679,
"worker_id": "sidekiq_0",
"rate_limiting_gates": [],
"duration_s": 0.055712,
"completed_at": "2023-02-07T20:54:50.594Z",
"load_balancing_strategy": "primary",
"exception.class": "Groups::DestroyService::DestroyError",
"exception.message": "Project 1234 can't be deleted",
"exception.backtrace": [
"app/services/groups/destroy_service.rb:23:in `block in execute'",
"app/services/groups/destroy_service.rb:19:in `execute'",
"ee/app/services/ee/groups/destroy_service.rb:11:in `block in execute'",
"ee/app/services/ee/groups/destroy_service.rb:36:in `with_scheduling_epic_cache_update'",
"ee/app/services/ee/groups/destroy_service.rb:10:in `execute'",
"app/workers/group_destroy_worker.rb:22:in `perform'",
"lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb:26:in `call'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb:16:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb:44:in `perform'",
"lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb:8:in `call'",
"lib/gitlab/sidekiq_middleware/worker_context.rb:9:in `wrap_in_optional_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:19:in `block in call'",
"lib/gitlab/application_context.rb:115:in `block in use'",
"lib/gitlab/application_context.rb:115:in `use'",
"lib/gitlab/application_context.rb:55:in `with_context'",
"lib/gitlab/sidekiq_middleware/worker_context/server.rb:17:in `call'",
"lib/gitlab/sidekiq_status/server_middleware.rb:7:in `call'",
"lib/gitlab/sidekiq_versioning/middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:37:in `within'",
"lib/gitlab/sidekiq_middleware/query_analyzer.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/admin_mode/server.rb:14:in `call'",
"lib/gitlab/sidekiq_middleware/instrumentation_logger.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/batch_loader.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb:7:in `call'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:10:in `block in call'",
"lib/gitlab/with_request_store.rb:17:in `enabling_request_store'",
"lib/gitlab/with_request_store.rb:10:in `with_request_store'",
"lib/gitlab/sidekiq_middleware/request_store_middleware.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:76:in `block in call'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `block in instrument'",
"lib/gitlab/metrics/background_transaction.rb:33:in `run'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:103:in `instrument'",
"lib/gitlab/sidekiq_middleware/server_metrics.rb:75:in `call'",
"lib/gitlab/sidekiq_middleware/monitor.rb:10:in `block in call'",
"lib/gitlab/sidekiq_daemon/monitor.rb:46:in `within_job'",
"lib/gitlab/sidekiq_middleware/monitor.rb:9:in `call'",
"lib/gitlab/sidekiq_middleware/size_limiter/server.rb:13:in `call'",
"lib/gitlab/sidekiq_logging/structured_logger.rb:21:in `call'"
],
"db_duration_s": 0.02057
}
Output of checks
Results of GitLab environment info
Expand for output related to GitLab environment info
(For installations with omnibus-gitlab package run and paste the output of: `sudo gitlab-rake gitlab:env:info`) (For installations from source run and paste the output of: `sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production`)
Results of GitLab application Check
Expand for output related to the GitLab application check
(For installations with omnibus-gitlab package run and paste the output of:
sudo gitlab-rake gitlab:check SANITIZE=true
)(For installations from source run and paste the output of:
sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production SANITIZE=true
)(we will only investigate if the tests are passing)