When performing an upgrade an additional 'gitlab-ctl restart' may be required for migrations to be in place

Summary

Customer performed an upgrade with downtime from 13.8.2 to 13.11.5 on their Geo primary. Users received 500 errors when attempting to create MRs with error ActiveModel::UnknownAttributeError (unknown attribute 'diff_type' for MergeRequestDiff.

This column was added with v13.9, but this migrations shows as up in gitlab-rake db:migrate:status:

up     20210107105306  Add diff type to merge request diffs

After executing gitlab-ctl restart the issue was resolved.

This is very similar to gitlab-org/charts/gitlab#2529 (closed).

Steps to reproduce

1. apt-get install gitlab-ee=13.11.5
2. gitlab-ctl reconfigure

This does not appear to happen consistently. This customer typically does zero-downtime upgrades, but did not do so in this case so they could jump multiple minor versions.

What is the current bug behavior?

Some actions may fail after upgrading if a manual gitlab-ctl restart is not executed.

What is the expected correct behavior?

No manual restart required when performing an upgrade.

Relevant logs

Relevant logs

{
  "method": "POST",
  "path": "/root/test/-/merge_requests",
  "format": "html",
  "controller": "Projects::MergeRequests::CreationsController",
  "action": "create",
  "status": 500,
  "time": "2021-06-07T09:10:33.808Z",
  "params": [
    {
      "key": "utf8",
      "value": "✓"
    },
    {
      "key": "authenticity_token",
      "value": "[FILTERED]"
    },
    {
      "key": "merge_request",
      "value": {
        "title": "[FILTERED]",
        "description": "[FILTERED]",
        "assignee_ids": [
          "0"
        ],
        "reviewer_ids": [
          "0"
        ],
        "label_ids": [
          ""
        ],
        "update_blocking_merge_request_refs": "false",
        "approval_rules_attributes": [
          {
            "id": "",
            "approvals_required": "0",
            "name": "",
            "user_ids": [
              ""
            ],
            "group_ids": [
              ""
            ]
          }
        ],
        "force_remove_source_branch": "1",
        "squash": "0",
        "lock_version": "0",
        "source_project_id": "2322",
        "source_branch": "2021-06-07",
        "target_project_id": "2322",
        "target_branch": "master"
      }
    },
    {
      "key": "merge_request_diff_head_sha",
      "value": "8c97dd39b0b0d16096fefeef7d73427527140be5"
    },
    {
      "key": "namespace_id",
      "value": "root"
    },
    {
      "key": "project_id",
      "value": "test"
    }
  ],
  "remote_ip": "10.10.1.1",
  "user_id": 1,
  "username": "root",
  "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
  "correlation_id": "01F7JVSTBCD8S68SQDCN7ZEQ2R",
  "meta.user": "root",
  "meta.project": "root/test",
  "meta.root_namespace": "root",
  "meta.caller_id": "Projects::MergeRequests::CreationsController#create",
  "meta.remote_ip": "10.10.1.1",
  "meta.feature_category": "code_review",
  "meta.client_id": "user/1",
  "gitaly_calls": 1,
  "gitaly_duration_s": 0.023156,
  "redis_calls": 6,
  "redis_duration_s": 0.000998,
  "redis_read_bytes": 1194,
  "redis_write_bytes": 977,
  "redis_cache_calls": 5,
  "redis_cache_duration_s": 0.000847,
  "redis_cache_read_bytes": 1013,
  "redis_cache_write_bytes": 280,
  "redis_shared_state_calls": 1,
  "redis_shared_state_duration_s": 0.000151,
  "redis_shared_state_read_bytes": 181,
  "redis_shared_state_write_bytes": 697,
  "db_count": 50,
  "db_write_count": 8,
  "db_cached_count": 13,
  "cpu_s": 0.117363,
  "mem_objects": 79035,
  "mem_bytes": 12992420,
  "mem_mallocs": 45392,
  "queue_duration_s": 0.006298,
  "exception.class": "ActiveModel::UnknownAttributeError",
  "exception.message": "unknown attribute 'diff_type' for MergeRequestDiff.",
  "exception.backtrace": [
    "app/models/merge_request.rb:989:in `block in create_merge_request_diff'",
    "lib/gitlab/gitaly_client.rb:325:in `allow_n_plus_1_calls'",
    "app/models/merge_request.rb:988:in `create_merge_request_diff'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in create_merge_request_diff'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `create_merge_request_diff'",
    "app/models/merge_request.rb:981:in `ensure_merge_request_diff'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in ensure_merge_request_diff'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `ensure_merge_request_diff'",
    "lib/gitlab/database.rb:342:in `block in transaction'",
    "lib/gitlab/database.rb:341:in `transaction'",
    "app/services/issuable_base_service.rb:182:in `block in create'",
    "lib/gitlab/database.rb:342:in `block in transaction'",
    "lib/gitlab/database.rb:341:in `transaction'",
    "app/services/issuable_base_service.rb:181:in `create'",
    "app/services/merge_requests/base_service.rb:72:in `create'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in create'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `create'",
    "app/services/merge_requests/create_service.rb:29:in `create'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in create'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `create'",
    "app/services/merge_requests/create_service.rb:13:in `execute'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in execute'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `execute'",
    "app/controllers/projects/merge_requests/creations_controller.rb:22:in `create'",
    "ee/lib/gitlab/ip_address_state.rb:10:in `with'",
    "ee/app/controllers/ee/application_controller.rb:44:in `set_current_ip_address'",
    "app/controllers/application_controller.rb:485:in `set_current_admin'",
    "lib/gitlab/session.rb:11:in `with_session'",
    "app/controllers/application_controller.rb:476:in `set_session_storage'",
    "lib/gitlab/i18n.rb:73:in `with_locale'",
    "lib/gitlab/i18n.rb:79:in `with_user_locale'",
    "app/controllers/application_controller.rb:470:in `set_locale'",
    "app/controllers/application_controller.rb:463:in `block in set_current_context'",
    "lib/gitlab/application_context.rb:70:in `block in use'",
    "lib/gitlab/application_context.rb:70:in `use'",
    "lib/gitlab/application_context.rb:27:in `with_context'",
    "app/controllers/application_controller.rb:454:in `set_current_context'",
    "lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
    "lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
    "lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
    "lib/gitlab/metrics/transaction.rb:56:in `run'",
    "lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
    "lib/gitlab/request_profiler/middleware.rb:17:in `call'",
    "lib/gitlab/jira/middleware.rb:19:in `call'",
    "lib/gitlab/middleware/go.rb:20:in `call'",
    "lib/gitlab/etag_caching/middleware.rb:21:in `call'",
    "lib/gitlab/middleware/multipart.rb:172:in `call'",
    "lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
    "lib/gitlab/middleware/read_only.rb:18:in `call'",
    "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
    "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
    "lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
    "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
    "lib/gitlab/middleware/request_context.rb:21:in `call'",
    "config/initializers/fix_local_cache_middleware.rb:11:in `call'",
    "lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:21:in `call'",
    "lib/gitlab/metrics/requests_rack_middleware.rb:76:in `call'",
    "lib/gitlab/middleware/release_env.rb:12:in `call'"
  ],
  "db_duration_s": 0.08811,
  "view_duration_s": 0,
  "duration_s": 0.21094
}

Details of package version

Provide the package version installation details

gitlab-ee 13.11.5

Environment details

• Operating System: Ubuntu 18.04.1
• Installation Target, remove incorrect values:
  • VM: AWS
• Installation Type, remove incorrect values:
  • Upgrade from version 13.8.2-ee.0
• Is there any other software running on the machine: DataDog
• Is this a single or multiple node installation?
• Resources
  • CPU: 16, x86_64
  • Memory total: 29 GB

Configuration details

Provide the relevant sections of `/etc/gitlab/gitlab.rb`

external_url 'https://gitlab.example.com'
gitlab_rails['gitlab_email_enabled'] = true
gitlab_rails['gitlab_email_from'] = 'REDACTED EMAIL'
gitlab_rails['gitlab_email_display_name'] = 'Example GitLab'
gitlab_rails['gitlab_email_reply_to'] = 'REDACTED EMAIL'
gitlab_rails['omniauth_enabled'] = true
gitlab_rails['omniauth_allow_single_sign_on'] = ['saml']
gitlab_rails['omniauth_block_auto_created_users'] = false
gitlab_rails['omniauth_providers'] = [
  {
    name: 'saml',
    args: {
      assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback',
      idp_cert_fingerprint: 'a:b:c',
      idp_sso_target_url: 'https://example.onelogin.com/trust/saml2/http-post/sso/1',
      issuer: 'https://gitlab.example.com',
      name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:persistent'
    },
    label: 'OneLogin SAML SSO' # optional label for SAML login button, defaults to "Saml"
  }
]
gitlab_rails['auto_migrate'] = false
gitlab_rails['db_adapter'] = "postgresql"
gitlab_rails['db_encoding'] = "UNICODE"
gitlab_rails['db_database'] = "gitlabhq_production"
gitlab_rails['db_username'] = "gitlab"
gitlab_rails['db_password'] = "REDACTED PASSWORD"
gitlab_rails['db_host'] = "gitlab-rds.example.com"
gitlab_rails['smtp_enable'] = true
gitlab_rails['smtp_address'] = "email-smtp.us-east-1.amazonaws.com"
gitlab_rails['smtp_port'] = 587
gitlab_rails['smtp_user_name'] = "myuser"
gitlab_rails['smtp_password'] = "REDACTED PASSWORD"
gitlab_rails['smtp_domain'] = "example.com"
gitlab_rails['smtp_authentication'] = "login"
gitlab_rails['smtp_enable_starttls_auto'] = true
unicorn['enable'] = false
puma['enable'] = true
postgresql['enable'] = false
nginx['redirect_http_to_https'] = true
nginx['ssl_certificate'] = "/etc/gitlab/ssl/cert.cert"
nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/key.key"
logging['logrotate_dateformat'] = "-%Y-%m-%d" # use date extensions for rotated files rather than numbers e.g. a value of "-%Y-%m-%d" would give rotated files like production.log-2016-03-09.gz
gitlab_pages['enable'] = true
pages_external_url "https://gitlab.example.io"
gitlab_pages['external_http'] = ['10.10.1.1:8090']
gitlab_pages['access_control'] = true
pages_nginx['enable'] = false
postgres_exporter['enable'] = false
letsencrypt['enable'] = false
geo_primary_role['enable'] = true

Proposal

#6180 (comment 1074786793)

Locate areas of our documentation where we are documenting users manually run db:migrate or gitlab:db:configure, and ensure users know to restart puma and sidekiq processes.