Skip to content

Improve flaky example reports

Albert Salim requested to merge caalberts-typo-example_id_reports into master

Improve flaky example reports

  • Fix a typo to check for :flaky_example_reports which fixed the report with example_id
  • Replace report by file name with report by example id to make finding of flaky example easier

Dry run report:

Top 20 flaky tests sorted by occurrences per file

./spec/features/merge_request/user_sees_merge_widget_spec.rb: 12
./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb: 10
./spec/support/shared_examples/features/discussion_comments_shared_example.rb: 9
./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb: 7
./spec/support/shared_examples/features/variable_list_shared_examples.rb: 7
./spec/features/issuables/markdown_references/jira_spec.rb: 7
./spec/features/search/user_uses_header_search_field_spec.rb: 5
./spec/support/shared_examples/features/reportable_note_shared_examples.rb: 4
./spec/features/merge_request/user_resolves_conflicts_spec.rb: 3
./spec/features/issues/user_creates_branch_and_merge_request_spec.rb: 3
./spec/features/issues/user_edits_issue_spec.rb: 3
./ee/spec/features/boards/swimlanes/epics_swimlanes_filtering_spec.rb: 3
./spec/features/merge_request/user_reverts_merge_request_spec.rb: 2
./ee/spec/features/merge_request/user_merges_immediately_spec.rb: 2
./spec/features/user_sees_revert_modal_spec.rb: 2
./spec/requests/api/merge_requests_spec.rb: 2
./spec/features/merge_request/user_posts_notes_spec.rb: 2
./ee/spec/features/promotion_spec.rb: 2
./spec/features/issues/create_issue_for_single_discussion_in_merge_request_spec.rb: 2
./ee/spec/features/issues/epic_in_issue_sidebar_spec.rb: 2

Number of flaky tests per type

features: 84
support: 37
spec: 31
requests: 5
models: 5
controllers: 4
lib: 3
finders: 1
workers: 1

Flaky tests per day for the last 20 days

2021-01-18: 27 files
	["./spec/controllers/concerns/renders_commits_spec.rb", "./spec/features/projects/show/user_sees_git_instructions_spec.rb", "./spec/features/projects/show/user_sees_collaboration_links_spec.rb", "./spec/features/security/project/public_access_spec.rb", "./spec/features/security/project/internal_access_spec.rb", "./spec/controllers/projects/merge_requests_controller_spec.rb", "./spec/models/group_spec.rb", "./spec/models/group_spec.rb", "./spec/support/shared_examples/features/snippets_shared_examples.rb", "./ee/spec/migrations/set_resolved_state_on_vulnerabilities_spec.rb", "./spec/features/user_can_display_performance_bar_spec.rb", "./ee/spec/services/ci/minutes/batch_reset_service_spec.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/controllers/projects/pipelines_controller_spec.rb", "./ee/spec/controllers/registrations/projects_controller_spec.rb", "./spec/features/issues/user_edits_issue_spec.rb", "./spec/features/issuables/markdown_references/jira_spec.rb", "./ee/spec/features/epics/shortcuts_epic_spec.rb", "./spec/controllers/projects/merge_requests_controller_spec.rb", "./spec/features/admin/admin_mode/login_spec.rb", "./spec/support/shared_examples/features/reportable_note_shared_examples.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./ee/spec/features/merge_request/user_merges_immediately_spec.rb", "./spec/features/projects/releases/user_views_releases_spec.rb", "./spec/features/issues/move_spec.rb", "./spec/support/shared_examples/features/wiki/file_attachments_shared_examples.rb"])
2021-01-17: 41 files
	["./ee/spec/lib/gem_extensions/elasticsearch/model/adapter/active_record/records_spec.rb", "./spec/support/shared_examples/models/concerns/bulk_insert_safe_shared_examples.rb", "./spec/features/projects/show/user_manages_notifications_spec.rb", "./ee/spec/support/shared_examples/features/protected_branches_access_control_shared_examples.rb", "./ee/spec/features/epics/epic_show_spec.rb", "./spec/features/security/project/private_access_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/features/projects/snippets/user_comments_on_snippet_spec.rb", "./spec/features/projects/snippets/create_snippet_spec.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/support/shared_examples/features/reportable_note_shared_examples.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./spec/support/shared_examples/models/concerns/bulk_insert_safe_shared_examples.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./ee/spec/controllers/registrations/projects_controller_spec.rb", "./spec/features/projects/pipelines/pipelines_spec.rb", "./spec/lib/gitlab/database/count/reltuples_count_strategy_spec.rb", "./spec/features/merge_request/batch_comments_spec.rb", "./spec/features/broadcast_messages_spec.rb", "./spec/features/search/user_uses_header_search_field_spec.rb", "./spec/features/issues/user_sorts_issue_comments_spec.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./spec/features/search/user_uses_header_search_field_spec.rb", "./ee/spec/features/epics/epic_show_spec.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./spec/features/issues/user_edits_issue_spec.rb", "./spec/features/issues/create_issue_for_single_discussion_in_merge_request_spec.rb", "./spec/features/merge_request/user_reverts_merge_request_spec.rb", "./spec/features/issues/user_creates_branch_and_merge_request_spec.rb", "./ee/spec/features/search/elastic/project_search_spec.rb", "./spec/lib/banzai/filter/commit_trailers_filter_spec.rb", "./spec/features/user_sees_revert_modal_spec.rb", "./spec/features/issues/issue_sidebar_spec.rb", "./spec/models/ci_platform_metric_spec.rb", "./spec/features/snippets/user_edits_snippet_spec.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb"])
2021-01-16: 40 files
	["./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/requests/git_http_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/features/callouts/registration_enabled_spec.rb", "./spec/features/merge_request/batch_comments_spec.rb", "./ee/spec/services/ci/minutes/batch_reset_service_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/models/clusters/applications/runner_spec.rb", "./spec/features/project_variables_spec.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./ee/spec/lib/gitlab/graphql/aggregations/vulnerabilities/lazy_user_notes_count_aggregate_spec.rb", "./spec/models/clusters/applications/runner_spec.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/features/merge_request/user_resolves_conflicts_spec.rb", "./spec/features/global_search_spec.rb", "./spec/requests/api/merge_requests_spec.rb", "./spec/requests/api/jobs_spec.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./ee/spec/features/boards/group_boards/user_edits_issues_spec.rb", "./spec/features/merge_request/user_edits_assignees_sidebar_spec.rb", "./spec/features/user_sees_revert_modal_spec.rb", "./ee/spec/features/promotion_spec.rb", "./ee/spec/features/merge_request/user_merges_immediately_spec.rb", "./spec/requests/api/merge_requests_spec.rb", "./spec/features/search/user_uses_header_search_field_spec.rb", "./spec/features/merge_request/user_awards_emoji_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/features/merge_request/user_accepts_merge_request_spec.rb", "./spec/features/merge_request/user_resolves_conflicts_spec.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./spec/features/issues/user_interacts_with_awards_spec.rb", "./ee/spec/features/search/user_searches_for_epics_spec.rb", "./spec/requests/api/groups_spec.rb", "./spec/features/merge_request/user_posts_notes_spec.rb", "./spec/features/merge_request/user_customizes_merge_commit_message_spec.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./spec/features/merge_request/user_reverts_merge_request_spec.rb"])
2021-01-15: 12 files
	["./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./ee/spec/features/search/user_searches_for_epics_spec.rb", "./spec/features/dashboard/projects_spec.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./spec/features/issues/create_issue_for_single_discussion_in_merge_request_spec.rb", "./ee/spec/features/promotion_spec.rb", "./spec/support/shared_examples/features/reportable_note_shared_examples.rb", "./spec/features/merge_request/user_posts_notes_spec.rb", "./ee/spec/migrations/migrate_scim_identities_to_saml_for_new_users_spec.rb", "./spec/support/shared_examples/services/container_registry_auth_service_shared_examples.rb"])
2021-01-14: 16 files
	["./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/support/shared_examples/features/resolving_discussions_in_issues_shared_examples.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./ee/spec/migrations/migrate_scim_identities_to_saml_for_new_users_spec.rb", "./spec/support/shared_examples/features/variable_list_shared_examples.rb", "./ee/spec/features/registrations/user_sees_new_onboarding_flow_spec.rb", "./spec/features/issuables/markdown_references/jira_spec.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./ee/spec/features/issues/issue_sidebar_spec.rb", "./spec/features/merge_request/user_sees_suggest_pipeline_spec.rb", "./ee/spec/features/projects/custom_projects_template_spec.rb", "./ee/spec/features/boards/swimlanes/epics_swimlanes_drag_issue_spec.rb", "./spec/features/issues/user_creates_branch_and_merge_request_spec.rb", "./spec/features/merge_request/user_resolves_conflicts_spec.rb"])
2021-01-13: 17 files
	["./ee/spec/features/boards/swimlanes/epics_swimlanes_filtering_spec.rb", "./spec/features/projects/commit/user_reverts_commit_spec.rb", "./spec/features/merge_request/user_rebases_merge_request_spec.rb", "./spec/features/issues/user_edits_issue_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/features/issues/create_issue_for_discussions_in_merge_request_spec.rb", "./spec/features/issuables/markdown_references/jira_spec.rb", "./spec/features/issues/form_spec.rb", "./spec/finders/pending_todos_finder_spec.rb", "./ee/spec/features/epics/todo_spec.rb", "./spec/features/search/user_uses_header_search_field_spec.rb", "./spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb", "./ee/spec/features/issues/epic_in_issue_sidebar_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects_spec.rb", "./ee/spec/features/issues/epic_in_issue_sidebar_spec.rb", "./spec/features/search/user_uses_header_search_field_spec.rb"])
2021-01-12: 15 files
	["./ee/spec/features/boards/swimlanes/epics_swimlanes_filtering_spec.rb", "./spec/features/merge_request/user_sees_merge_widget_spec.rb", "./ee/spec/features/boards/swimlanes/epics_swimlanes_filtering_spec.rb", "./spec/features/merge_request/user_sees_avatar_on_diff_notes_spec.rb", "./spec/features/issuables/markdown_references/jira_spec.rb", "./spec/features/issues/user_creates_branch_and_merge_request_spec.rb", "./spec/features/issuables/markdown_references/jira_spec.rb", "./spec/support/shared_examples/features/reportable_note_shared_examples.rb", "./spec/features/merge_request/user_creates_image_diff_notes_spec.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./spec/support/shared_examples/graphql/mutation_shared_examples.rb", "./spec/workers/ci/daily_build_group_report_results_worker_spec.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb", "./spec/features/issues/move_spec.rb", "./spec/support/shared_examples/features/project_upload_files_shared_examples.rb"])
2021-01-11: 3 files
	["./spec/features/issuables/markdown_references/jira_spec.rb", "./spec/features/issuables/markdown_references/jira_spec.rb", "./spec/support/shared_examples/features/discussion_comments_shared_example.rb"])

Top 20 flaky test examples sorted by number of reports

./spec/controllers/concerns/renders_commits_spec.rb[1:2:1]: 2738
./spec/features/projects/show/user_sees_git_instructions_spec.rb[1:1:3:2:1:1]: 2344
./spec/features/projects/show/user_sees_collaboration_links_spec.rb[1:1:1]: 1255
./spec/controllers/projects/merge_requests_controller_spec.rb[1:8:2:3:1:1:1]: 905
./spec/features/security/project/public_access_spec.rb[1:2:1]: 782
./spec/models/group_spec.rb[1:42:3:3:1]: 636
./spec/models/group_spec.rb[1:42:1:1:1]: 636
./spec/features/security/project/internal_access_spec.rb[1:2:1]: 554
./spec/features/projects/snippets/show_spec.rb[1:1:1:1]: 235
./spec/features/user_can_display_performance_bar_spec.rb[1:2:1:2:1]: 202
./ee/spec/lib/gem_extensions/elasticsearch/model/adapter/active_record/records_spec.rb[1:1:1]: 195
./spec/features/projects/show/user_manages_notifications_spec.rb[1:1]: 125
./ee/spec/features/boards/swimlanes/epics_swimlanes_filtering_spec.rb[1:1:3]: 86
./ee/spec/features/boards/swimlanes/epics_swimlanes_filtering_spec.rb[1:1:1]: 85
./ee/spec/features/epics/epic_show_spec.rb[1:1:3:1]: 79
./spec/models/ci_platform_metric_spec.rb[1:1:2:1:1]: 79
./spec/controllers/projects/merge_requests_controller_spec.rb[1:21:4:1:1]: 56
./spec/features/issuables/markdown_references/jira_spec.rb[1:1:1:1]: 50
./spec/features/security/project/private_access_spec.rb[1:2:1]: 39
./ee/spec/features/protected_branches_spec.rb[1:2:1:8]: 38

failureflaky-test issues

Edited by Albert Salim

Merge request reports