Add latest changes from gitlab-org/gitlab@master

gitlabhq · Oct 5, 2021 · 33a1d18 · 33a1d18
1 parent 8c6c213
commit 33a1d18
Show file tree

Hide file tree

Showing 27 changed files with 332 additions and 190 deletions.
diff --git a/app/assets/stylesheets/pages/issues.scss b/app/assets/stylesheets/pages/issues.scss
@@ -1,3 +1,51 @@
+.issue-token-link {
+  &[href] {
+    color: $blue-600;
+  }
+
+  &:hover,
+  &:focus {
+    outline: none;
+    text-decoration: none;
+  }
+}
+
+.issue-token-reference {
+  margin-right: 1px;
+  background-color: $gray-lighter;
+  transition: background $general-hover-transition-duration $general-hover-transition-curve, color $general-hover-transition-duration $general-hover-transition-curve;
+
+  .issue-token:hover &,
+  .issue-token-link:focus > & {
+    background-color: $gray-normal;
+    color: $blue-800;
+    text-decoration: none;
+  }
+}
+
+.issue-token-title {
+  background-color: $gray-normal;
+  transition: background $general-hover-transition-duration $general-hover-transition-curve;
+
+  .issue-token:hover &,
+  .issue-token-link:focus > & {
+    background-color: $border-gray-normal;
+  }
+}
+
+.issue-token-remove-button {
+  background-color: $gray-normal;
+  transition: background $general-hover-transition-duration $general-hover-transition-curve;
+
+  &:hover,
+  &:focus,
+  .issue-token:hover &,
+  .issue-token-link:focus + & {
+    background-color: $border-gray-normal;
+    outline: none;
+  }
+}
+
 .issue-realtime-pre-pulse {
   opacity: 0;
 }

diff --git a/app/controllers/profiles/two_factor_auths_controller.rb b/app/controllers/profiles/two_factor_auths_controller.rb
@@ -231,8 +231,6 @@ def groups_notification(groups)
   end
 
   def ensure_verified_primary_email
-    return unless Feature.enabled?(:ensure_verified_primary_email_for_2fa, default_enabled: :yaml)
-
     unless current_user.two_factor_enabled? || current_user.primary_email_verified?
       redirect_to profile_emails_path, notice: s_('You need to verify your primary email first before enabling Two-Factor Authentication.')
     end

diff --git a/app/services/ci/stuck_builds/drop_scheduled_service.rb b/app/services/ci/stuck_builds/drop_scheduled_service.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropScheduledService
+      include DropHelpers
+
+      BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
+
+      def execute
+        Gitlab::AppLogger.info "#{self.class}: Cleaning scheduled, timed-out builds"
+
+        drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
+      end
+
+      private
+
+      def scheduled_timed_out_builds
+        Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord
+          'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?',
+          BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago
+        )
+      end
+    end
+  end
+end
diff --git a/app/services/ci/stuck_builds/drop_service.rb b/app/services/ci/stuck_builds/drop_service.rb
@@ -6,7 +6,6 @@ class DropService
       include DropHelpers
 
       BUILD_PENDING_OUTDATED_TIMEOUT = 1.day
-      BUILD_SCHEDULED_OUTDATED_TIMEOUT = 1.hour
       BUILD_PENDING_STUCK_TIMEOUT = 1.hour
       BUILD_LOOKBACK = 5.days
 
@@ -18,8 +17,6 @@ def execute
           failure_reason: :stuck_or_timeout_failure
         )
 
-        drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
-
         drop_stuck(
           pending_builds(BUILD_PENDING_STUCK_TIMEOUT.ago),
           failure_reason: :stuck_or_timeout_failure
@@ -40,13 +37,6 @@ def pending_builds(timeout)
         end
       end
       # rubocop: enable CodeReuse/ActiveRecord
-
-      def scheduled_timed_out_builds
-        Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord
-          'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?',
-          BUILD_SCHEDULED_OUTDATED_TIMEOUT.ago
-        )
-      end
     end
   end
 end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
@@ -237,6 +237,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: cronjob:ci_stuck_builds_drop_scheduled
+  :worker_name: Ci::StuckBuilds::DropScheduledWorker
+  :feature_category: :continuous_integration
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:container_expiration_policy
   :worker_name: ContainerExpirationPolicyWorker
   :feature_category: :container_registry

diff --git a/app/workers/ci/stuck_builds/drop_scheduled_worker.rb b/app/workers/ci/stuck_builds/drop_scheduled_worker.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Ci
+  module StuckBuilds
+    class DropScheduledWorker
+      include ApplicationWorker
+      include ExclusiveLeaseGuard
+
+      idempotent!
+
+      # rubocop:disable Scalability/CronWorkerContext
+      # This is an instance-wide cleanup query, so there's no meaningful
+      # scope to consider this in the context of.
+      include CronjobQueue
+      # rubocop:enable Scalability/CronWorkerContext
+
+      data_consistency :always
+
+      feature_category :continuous_integration
+
+      def perform
+        try_obtain_lease do
+          Ci::StuckBuilds::DropScheduledService.new.execute
+        end
+      end
+
+      private
+
+      def lease_timeout
+        30.minutes
+      end
+    end
+  end
+end
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb
@@ -18,6 +18,7 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
 
   def perform
     Ci::StuckBuilds::DropRunningWorker.perform_in(20.minutes)
+    Ci::StuckBuilds::DropScheduledWorker.perform_in(40.minutes)
 
     return unless try_obtain_lease
 

diff --git a/config/feature_flags/development/ensure_verified_primary_email_for_2fa.yml b/config/feature_flags/development/ensure_verified_primary_email_for_2fa.yml
diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md
@@ -21,11 +21,11 @@ storage and retrieval. Gitaly can be:
 Gitaly implements a client-server architecture:
 
 - A Gitaly server is any node that runs Gitaly itself.
-- A Gitaly client is any node that runs a process that makes requests of the Gitaly server. These
-  include, but are not limited to:
-  - [GitLab Rails application](https://gitlab.com/gitlab-org/gitlab).
-  - [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell).
-  - [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse).
+- A Gitaly client is any node that runs a process that makes requests of the Gitaly server. Gitaly clients are also known as _Gitaly consumers_ and include:
+  - [GitLab Rails application](https://gitlab.com/gitlab-org/gitlab)
+  - [GitLab Shell](https://gitlab.com/gitlab-org/gitlab-shell)
+  - [GitLab Workhorse](https://gitlab.com/gitlab-org/gitlab-workhorse)
+  - [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer)
 
 Gitaly manages only Git repository access for GitLab. Other types of GitLab data aren't accessed
 using Gitaly.

diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md
@@ -3342,56 +3342,38 @@ to select a specific site profile and scanner profile.
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3515) in GitLab 11.5, you can control which failures to retry on.
 
-Use `retry` to configure how many times a job is retried in
-case of a failure.
+Use `retry` to configure how many times a job is retried if it fails.
+If not defined, defaults to `0` and jobs do not retry.
 
-When a job fails, the job is processed again,
-until the limit specified by the `retry` keyword is reached.
+When a job fails, the job is processed up to two more times, until it succeeds or
+reaches the maximum number of retries.
 
-If `retry` is set to `2`, and a job succeeds in a second run (first retry), it is not retried.
-The `retry` value must be a positive integer, from `0` to `2`
-(two retries maximum, three runs in total).
+By default, all failure types cause the job to be retried. Use [`retry:when`](#retrywhen)
+to select which failures to retry on.
 
-The following example retries all failure cases:
-
-```yaml
-test:
-  script: rspec
-  retry: 2
-```
-
-By default, a job is retried on all failure cases. To have better control
-over which failures to retry, `retry` can be a hash with the following keys:
+**Keyword type**: Job keyword. You can use it only as part of a job or in the
+[`default:` section](#custom-default-keyword-values).
 
-- `max`: The maximum number of retries.
-- `when`: The failure cases to retry.
+**Possible inputs**: `0` (default), `1`, or `2`.
 
-To retry only runner system failures at maximum two times:
+**Example of `retry`**:
 
 ```yaml
 test:
   script: rspec
-  retry:
-    max: 2
-    when: runner_system_failure
+  retry: 2
 ```
 
-If there is another failure, other than a runner system failure, the job
-is not retried.
+#### `retry:when`
 
-To retry on multiple failure cases, `when` can also be an array of failures:
+Use `retry:when` with `retry:max` to retry jobs for only specific failure cases.
+`retry:max` is the maximum number of retries, like [`retry`](#retry), and can be
+`0`, `1`, or `2`.
 
-```yaml
-test:
-  script: rspec
-  retry:
-    max: 2
-    when:
-      - runner_system_failure
-      - stuck_or_timeout_failure
-```
+**Keyword type**: Job keyword. You can use it only as part of a job or in the
+[`default:` section](#custom-default-keyword-values).
 
-Possible values for `when` are:
+**Possible inputs**: A single failure type, or an array of one or more failure types:
 
 <!--
   If you change any of the values below, make sure to update the `RETRY_WHEN_IN_DOCUMENTATION`
@@ -3415,7 +3397,34 @@ Possible values for `when` are:
 - `scheduler_failure`: Retry if the scheduler failed to assign the job to a runner.
 - `data_integrity_failure`: Retry if there is a structural integrity problem detected.
 
-You can specify the number of [retry attempts for certain stages of job execution](../runners/configure_runners.md#job-stages-attempts) using variables.
+**Example of `retry:when`** (single failure type):
+
+```yaml
+test:
+  script: rspec
+  retry:
+    max: 2
+    when: runner_system_failure
+```
+
+If there is a failure other than a runner system failure, the job is not retried.
+
+**Example of `retry:when`** (array of failure types):
+
+```yaml
+test:
+  script: rspec
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+```
+
+**Related topics**:
+
+You can specify the number of [retry attempts for certain stages of job execution](../runners/configure_runners.md#job-stages-attempts)
+using variables.
 
 ### `timeout`
 

diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md
@@ -37,14 +37,14 @@ FLAG:
 
 | If the feature is...     | Use this text |
 |--------------------------|---------------|
-| Available                | `On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Unavailable              | `On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Available, per-group     | `On self-managed GitLab, by default this feature is available. To hide the feature per group, ask an administrator to [disable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Unavailable, per-group   | `On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Available, per-project   | `On self-managed GitLab, by default this feature is available. To hide the feature per project or for your entire instance, ask an administrator to [disable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Unavailable, per-project | `On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Available, per-user      | `On self-managed GitLab, by default this feature is available. To hide the feature per user, ask an administrator to [disable the <flag name> flag](<path to>/administration/feature_flags.md).` |
-| Unavailable, per-user    | `On self-managed GitLab, by default this feature is not available. To make it available per user, ask an administrator to [enable the <flag name> flag](<path to>/administration/feature_flags.md).` |
+| Available                | `On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Unavailable              | `On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Available, per-group     | `On self-managed GitLab, by default this feature is available. To hide the feature per group, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Unavailable, per-group   | `On self-managed GitLab, by default this feature is not available. To make it available per group, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Available, per-project   | `On self-managed GitLab, by default this feature is available. To hide the feature per project or for your entire instance, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Unavailable, per-project | `On self-managed GitLab, by default this feature is not available. To make it available per project or for your entire instance, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Available, per-user      | `On self-managed GitLab, by default this feature is available. To hide the feature per user, ask an administrator to [disable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
+| Unavailable, per-user    | `On self-managed GitLab, by default this feature is not available. To make it available per user, ask an administrator to [enable the feature flag](<path to>/administration/feature_flags.md) named <flag name>.` |
 
 ### GitLab.com availability information
 
@@ -67,7 +67,7 @@ When the state of a flag changes (for example, disabled by default to enabled by
 Possible version history entries are:
 
 ```markdown
-> - [Introduced](issue-link) in GitLab X.X. [Deployed behind the <flag name> flag](../../administration/feature_flags.md), disabled by default.
+> - [Introduced](issue-link) in GitLab X.X [with a flag](../../administration/feature_flags.md) named <flag name>. Disabled by default.
 > - [Enabled on GitLab.com](issue-link) in GitLab X.X.
 > - [Enabled on GitLab.com](issue-link) in GitLab X.X. Available to GitLab.com administrators only.
 > - [Enabled on self-managed](issue-link) in GitLab X.X.
@@ -80,31 +80,31 @@ Possible version history entries are:
 The following examples show the progression of a feature flag.
 
 ```markdown
-> Introduced in GitLab 13.7. [Deployed behind the `forti_token_cloud` flag](../../administration/feature_flags.md), disabled by default.
+> Introduced in GitLab 13.7 [with a flag](../../administration/feature_flags.md) named `forti_token_cloud`. Disabled by default.
 
 FLAG:
 On self-managed GitLab, by default this feature is not available. To make it available,
-ask an administrator to [enable the `forti_token_cloud` flag](../administration/feature_flags.md).`
+ask an administrator to [enable the featured flag](../administration/feature_flags.md) named `forti_token_cloud`.
 The feature is not ready for production use.
 ```
 
 When the feature is enabled in production, you can update the version history:
 
 ```markdown
-> - Introduced in GitLab 13.7. [Deployed behind the `forti_token_cloud` flag](../../administration/feature_flags.md), disabled by default.
+> - Introduced in GitLab 13.7 [with a flag](../../administration/feature_flags.md) named `forti_token_cloud`. Disabled by default.
 > - [Enabled on self-managed](https://gitlab.com/issue/etc) GitLab 13.8.
 
 FLAG:
 On self-managed GitLab, by default this feature is available. To hide the feature per user,
-ask an administrator to [disable the `forti_token_cloud` flag](../administration/feature_flags.md).
+ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `forti_token_cloud`.
 ```
 
 And, when the feature is done and fully available to all users:
 
 ```markdown
-> - Introduced in GitLab 13.7. [Deployed behind the `forti_token_cloud` flag](../../administration/feature_flags.md), disabled by default.
+> - Introduced in GitLab 13.7 [with a flag](../../administration/feature_flags.md) named `forti_token_cloud`. Disabled by default.
 > - [Enabled on self-managed](https://gitlab.com/issue/etc) GitLab 13.8.
 > - [Enabled on GitLab.com](https://gitlab.com/issue/etc) in GitLab 13.9.
-> - [Feature flag `forti_token_cloud`](https://gitlab.com/issue/etc) removed in GitLab 14.0.
+> - [Feature flag removed](https://gitlab.com/issue/etc) in GitLab 14.0.
 > - [Generally available](issue-link) in GitLab 14.0.
 ```
diff --git a/doc/development/service_ping/implement.md b/doc/development/service_ping/implement.md
@@ -1031,9 +1031,9 @@ Example metrics persistence:
 class UsageData
   def count_secure_pipelines(time_period)
     ...
-    relation = ::Security::Scan.latest_successful_by_build.by_scan_types(scan_type).where(security_scans: time_period)
+    relation = ::Security::Scan.by_scan_types(scan_type).where(time_period)
 
-    pipelines_with_secure_jobs['dependency_scanning_pipeline'] = estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
+    pipelines_with_secure_jobs['dependency_scanning_pipeline'] = estimate_batch_distinct_count(relation, :pipeline_id, batch_size: 1000, start: start_id, finish: finish_id) do |result|
       ::Gitlab::Usage::Metrics::Aggregates::Sources::PostgresHll
         .save_aggregated_metrics(metric_name: 'dependency_scanning_pipeline', recorded_at_timestamp: recorded_at, time_period: time_period, data: result)
     end