Add a relabel_config to target __scheme__ for the kubernetes-pods job

What does this MR do?

Adds a relabel_config to target scheme in the kubernetes-pods scrape_config for Pod target discovery in Prometheus. This will set the target URI scheme in Prometheus to the value of the gitlab.com/prometheus_scheme annotation (if the value matches https?).

See the Notes in #3395 for additional overrides that will be required if setting the scheme to https.

Related issues

Closes #3395 (closed)

Checklist

See Definition of done.

For anything in this list which will not be completed, please provide a reason in the MR discussion.

Required

• Merge Request Title and Description are up to date, accurate, and descriptive
• MR targeting the appropriate branch
• MR has a green pipeline on GitLab.com

Expected (please provide an explanation if not completing)

• Test plan indicating conditions for success has been posted and passes
• Documentation created/updated N/A
• Tests added N/A
• Integration tests added to GitLab QA
• Equivalent MR/issue for omnibus-gitlab opened

Test plan

1. Deploy the chart with an gitlab.com/prometheus_scheme: https annotation applied to a metrics exporting pod (one that has the gitlab.com/prometheus_scrape: true annotation) E.g - sidekiq:

   helm upgrade --install gitlab . --timeout 600s --namespace=gitlab \
           --set global.hosts.domain=your_domain \
           --set global.image.pullPolicy=Always \
           --set certmanager-issuer.email=your_email@gitlab.com \
           --set gitlab.sidekiq.annotations."gitlab\.com/prometheus_scheme"="https" \
           --debug

2. Validate that the scrapeUrl is set to https:// - You should also have a lastError that indicates an error about getting an http response to an https request - e.g. "lastError": "Get \"https://10.42.1.157:3807/metrics\": http: server gave HTTP response to HTTPS client". That can be done from the toolbox with:

   kubectl exec <toolbox pod> --namespace=gitlab -- curl -L "http://gitlab-prometheus-server/api/v1/targets?state=active" | jq -r '.data.activeTargets | to_entries | .[] | select(.value.labels.app=="sidekiq")'

Example output from Step 2:

{
  "key": 8,
  "value": {
    "discoveredLabels": {
      "__address__": "10.42.1.157",
      "__meta_kubernetes_namespace": "gitlab",
      "__meta_kubernetes_pod_annotation_checksum_configmap": "1d6d4cbcda664c7faadcb64596b62723b883f1ec327246e9071d52ca3579b215",
      "__meta_kubernetes_pod_annotation_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
      "__meta_kubernetes_pod_annotation_gitlab_com_prometheus_path": "/metrics",
      "__meta_kubernetes_pod_annotation_gitlab_com_prometheus_port": "3807",
      "__meta_kubernetes_pod_annotation_gitlab_com_prometheus_scheme": "https",
      "__meta_kubernetes_pod_annotation_gitlab_com_prometheus_scrape": "true",
      "__meta_kubernetes_pod_annotation_prometheus_io_path": "/metrics",
      "__meta_kubernetes_pod_annotation_prometheus_io_port": "3807",
      "__meta_kubernetes_pod_annotation_prometheus_io_scrape": "true",
      "__meta_kubernetes_pod_annotationpresent_checksum_configmap": "true",
      "__meta_kubernetes_pod_annotationpresent_cluster_autoscaler_kubernetes_io_safe_to_evict": "true",
      "__meta_kubernetes_pod_annotationpresent_gitlab_com_prometheus_path": "true",
      "__meta_kubernetes_pod_annotationpresent_gitlab_com_prometheus_port": "true",
      "__meta_kubernetes_pod_annotationpresent_gitlab_com_prometheus_scheme": "true",
      "__meta_kubernetes_pod_annotationpresent_gitlab_com_prometheus_scrape": "true",
      "__meta_kubernetes_pod_annotationpresent_prometheus_io_path": "true",
      "__meta_kubernetes_pod_annotationpresent_prometheus_io_port": "true",
      "__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape": "true",
      "__meta_kubernetes_pod_container_init": "true",
      "__meta_kubernetes_pod_container_name": "dependencies",
      "__meta_kubernetes_pod_controller_kind": "ReplicaSet",
      "__meta_kubernetes_pod_controller_name": "gitlab-sidekiq-all-in-1-v2-5ff69bcc8f",
      "__meta_kubernetes_pod_host_ip": "172.19.0.4",
      "__meta_kubernetes_pod_ip": "10.42.1.157",
      "__meta_kubernetes_pod_label_app": "sidekiq",
      "__meta_kubernetes_pod_label_chart": "sidekiq-6.1.2",
      "__meta_kubernetes_pod_label_heritage": "Helm",
      "__meta_kubernetes_pod_label_pod_template_hash": "5ff69bcc8f",
      "__meta_kubernetes_pod_label_queue_pod_name": "all-in-1",
      "__meta_kubernetes_pod_label_release": "gitlab",
      "__meta_kubernetes_pod_labelpresent_app": "true",
      "__meta_kubernetes_pod_labelpresent_chart": "true",
      "__meta_kubernetes_pod_labelpresent_heritage": "true",
      "__meta_kubernetes_pod_labelpresent_pod_template_hash": "true",
      "__meta_kubernetes_pod_labelpresent_queue_pod_name": "true",
      "__meta_kubernetes_pod_labelpresent_release": "true",
      "__meta_kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-5ff69bcc8f-bbswz",
      "__meta_kubernetes_pod_node_name": "k3d-chartdev-agent-1",
      "__meta_kubernetes_pod_phase": "Running",
      "__meta_kubernetes_pod_ready": "true",
      "__meta_kubernetes_pod_uid": "e6a9997d-51b6-4caa-9df9-4e943ccfbe0d",
      "__metrics_path__": "/metrics",
      "__scheme__": "http",
      "__scrape_interval__": "1m",
      "__scrape_timeout__": "10s",
      "job": "kubernetes-pods"
    },
    "labels": {
      "app": "sidekiq",
      "chart": "sidekiq-6.1.2",
      "heritage": "Helm",
      "instance": "10.42.1.157:3807",
      "job": "kubernetes-pods",
      "kubernetes_namespace": "gitlab",
      "kubernetes_pod_name": "gitlab-sidekiq-all-in-1-v2-5ff69bcc8f-bbswz",
      "pod_template_hash": "5ff69bcc8f",
      "queue_pod_name": "all-in-1",
      "release": "gitlab"
    },
    "scrapePool": "kubernetes-pods",
    "scrapeUrl": "https://10.42.1.157:3807/metrics",
    "globalUrl": "https://10.42.1.157:3807/metrics",
    "lastError": "Get \"https://10.42.1.157:3807/metrics\": http: server gave HTTP response to HTTPS client",
    "lastScrape": "2022-07-13T16:32:51.169160724Z",
    "lastScrapeDuration": 0.001839669,
    "health": "down",
    "scrapeInterval": "1m",
    "scrapeTimeout": "10s"
  }
}