ci: Notify master failures in Slack

What does this MR do and why?

This changes the notify-pipeline-failure job to include more information similar to the native Slack integration for pipeline failure notifications.

The idea is to better control what and where failures are reported. For instance, Ruby 3 failures will now only go to the #f_ruby3 channel.

We'll now also:

• ignore child pipeline failures, as they're already reported in their parent one
• ignore failures for pipelines started by release-tools approver bot as they're known, non-actionable, and tracked by #332411
• not list failed jobs that are allowed to fail as they create noise in the notification (as they're non-actionable)

Related to https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/work_items/115987928.

TODO:

• Exclude child pipelines ("source": "parent_pipeline")
• Retrieve the pipeline's failed jobs (e.g. https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/pipelines/657528398/jobs?scope=failed), then exclude the one with "allow_failure": true
• Post the notification with links to failed jobs
• FOSS support: The CI_SLACK_WEBHOOK_URL variable is present at https://gitlab.com/gitlab-org/gitlab-foss/-/settings/ci_cd (although different than the one at https://gitlab.com/gitlab-org/gitlab/-/settings/ci_cd so we might want to ensure that notifications from FOSS comes in after this is merged)

Screenshots or screen recordings

Testing

Diff used for testing

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cfd2c88fd59b..5669a9b4d4b6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -43,6 +43,10 @@ workflow:
     - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby3/'
       variables:
         RUBY_VERSION: "3.0"
+    # TODO: Remove me before merge
+    - if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "switch-to-custom-slack-notification-for-master-broken"'
+      variables:
+        NOTIFY_PIPELINE_FAILURE_CHANNEL: "remy-master-broken-test"
     # For (detached) merge request pipelines.
     - if: '$CI_MERGE_REQUEST_IID'
     # For the maintenance scheduled pipelines, we set specific variables.
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index d7d27b91b135..5c19a2a055c1 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -23,6 +23,7 @@
     WEBPACK_COMPILE_LOG_PATH: "tmp/webpack-output.log"
   stage: prepare
   script:
+    - exit 1
     - yarn_install_script
     - export GITLAB_ASSETS_HASH=$(bin/rake gitlab:assets:hash_sum)
     - 'echo "CACHE_ASSETS_AS_PACKAGE: ${CACHE_ASSETS_AS_PACKAGE}"'
@@ -324,6 +325,7 @@ webpack-dev-server:
     WEBPACK_MEMORY_TEST: "true"
     WEBPACK_VENDOR_DLL: "true"
   script:
+    - exit 1
     - !reference [.yarn-install, script]
     - run_timed_command "retry yarn webpack-vendor"
     - run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js"
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 672d2301c3f3..0bb02c4432f7 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -12,6 +12,7 @@ setup-test-env:
   variables:
     SETUP_DB: "false"
   script:
+    - exit 1
     - echo $CI_MERGE_REQUEST_APPROVED
     - source scripts/gitlab_component_helpers.sh
     - run_timed_command "download_and_extract_gitlab_workhorse_package" || true
@@ -445,6 +446,7 @@ rspec:flaky-tests-report:
     - source scripts/utils.sh
     - source scripts/rspec_helpers.sh
   script:
+    - exit 1
     - generate_flaky_tests_reports
   artifacts:
     expire_in: 31d
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index 52ed85190ec9..cc05fc7a679a 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -10,6 +10,8 @@ code_quality:
     - .default-retry
     - .use-docker-in-docker
   stage: lint
+  script:
+    - exit 1
   artifacts:
     paths:
       - gl-code-quality-report.json  # GitLab-specific
@@ -36,6 +38,8 @@ code_quality:
 
 brakeman-sast:
   rules: !reference [".reports:rules:brakeman-sast", rules]
+  script:
+    - exit 1
 
 semgrep-sast:
   rules: !reference [".reports:rules:semgrep-sast", rules]
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index 7f9edd1650a5..32a7bd6d98db 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -122,6 +122,7 @@ detect-tests:
   variables:
     RSPEC_TESTS_MAPPING_ENABLED: "true"
   script:
+    - exit 1
     - source ./scripts/utils.sh
     - source ./scripts/rspec_helpers.sh
     - install_gitlab_gem
diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml
index 59ea665ae079..eb7a6a711f3b 100644
--- a/.gitlab/ci/static-analysis.gitlab-ci.yml
+++ b/.gitlab/ci/static-analysis.gitlab-ci.yml
@@ -31,6 +31,7 @@ static-analysis:
     - .static-analysis:rules:static-analysis
   parallel: 2
   script:
+    - exit 1
     - yarn_install_script
     - scripts/static-analysis
 
@@ -47,6 +48,7 @@ static-verification-with-database:
     - .static-analysis:rules:static-verification-with-database
     - .use-pg12
   script:
+    - exit 1
     - bundle exec rake lint:static_verification_with_database
   variables:
     SETUP_DB: "true"
@@ -62,6 +64,7 @@ generate-apollo-graphql-schema:
   variables:
     USE_BUNDLE_INSTALL: "false"
   script:
+    - exit 1
     - apollo client:download-schema --config=config/apollo.config.js ${GRAPHQL_SCHEMA_APOLLO_FILE}
   artifacts:
     name: graphql-schema-apollo
@@ -100,6 +103,7 @@ haml-lint:
     - .ruby-cache
     - .static-analysis:rules:haml-lint
   script:
+    - exit 1
     - run_timed_command "bundle exec haml-lint --parallel app/views"
   artifacts:
     expire_in: 31d
@@ -168,6 +172,7 @@ feature-flags-usage:
     - .rubocop-job-cache
     - .static-analysis:rules:rubocop
   script:
+    - exit 1
     # We need to disable the cache for this cop since it creates files under tmp/feature_flags/*.used,
     # the cache would prevent these files from being created.
     - run_timed_command "bundle exec rubocop --only Gitlab/MarkUsedFeatureFlags --cache false"

MR acceptance checklist

This checklist encourages us to confirm any changes have been analyzed to reduce risks in quality, performance, reliability, security, and maintainability.

• I have evaluated the MR acceptance checklist for this MR.