Add latest changes from gitlab-org/gitlab@master

.rubocop_manual_todo.yml

@@ -2557,7 +2557,6 @@ Rails/IncludeUrlHelper:
 # TODO issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344279
 Style/OpenStructUse:
   Exclude:
-    - 'app/finders/snippets_finder.rb'
     - 'app/helpers/application_settings_helper.rb'
     - 'ee/spec/db/production/license_spec.rb'
     - 'ee/spec/features/projects/new_project_spec.rb'

app/controllers/projects/tags_controller.rb

@@ -14,18 +14,24 @@ class Projects::TagsController < Projects::ApplicationController
 
   # rubocop: disable CodeReuse/ActiveRecord
   def index
-    params[:sort] = params[:sort].presence || sort_value_recently_updated
+    begin
+      params[:sort] = params[:sort].presence || sort_value_recently_updated
 
-    @sort = params[:sort]
+      @sort = params[:sort]
 
-    @tags, @tags_loading_error = TagsFinder.new(@repository, params).execute
+      @tags = TagsFinder.new(@repository, params).execute
 
-    @tags = Kaminari.paginate_array(@tags).page(params[:page])
-    tag_names = @tags.map(&:name)
-    @tags_pipelines = @project.ci_pipelines.latest_successful_for_refs(tag_names)
+      @tags = Kaminari.paginate_array(@tags).page(params[:page])
+      tag_names = @tags.map(&:name)
+      @tags_pipelines = @project.ci_pipelines.latest_successful_for_refs(tag_names)
 
-    @releases = project.releases.where(tag: tag_names)
-    @tag_pipeline_statuses = Ci::CommitStatusesFinder.new(@project, @repository, current_user, @tags).execute
+      @releases = project.releases.where(tag: tag_names)
+      @tag_pipeline_statuses = Ci::CommitStatusesFinder.new(@project, @repository, current_user, @tags).execute
+
+    rescue Gitlab::Git::CommandError => e
+      @tags = []
+      @tags_loading_error = e
+    end
 
     respond_to do |format|
       status = @tags_loading_error ? :service_unavailable : :ok

app/controllers/projects/tree_controller.rb

@@ -19,6 +19,7 @@ class Projects::TreeController < Projects::ApplicationController
     push_frontend_feature_flag(:lazy_load_commits, @project, default_enabled: :yaml)
     push_frontend_feature_flag(:paginated_tree_graphql_query, @project, default_enabled: :yaml)
     push_frontend_feature_flag(:new_dir_modal, @project, default_enabled: :yaml)
+    push_frontend_feature_flag(:refactor_blob_viewer, @project, default_enabled: :yaml)
   end
 
   feature_category :source_code_management

app/controllers/projects_controller.rb

@@ -293,7 +293,11 @@ def refs
     end
 
     if find_tags && @repository.tag_count.nonzero?
-      tags, _ = TagsFinder.new(@repository, params).execute
+      tags = begin
+        TagsFinder.new(@repository, params).execute
+      rescue Gitlab::Git::CommandError
+        []
+      end
 
       options['Tags'] = tags.take(100).map(&:name)
     end

app/finders/snippets_finder.rb

@@ -42,12 +42,11 @@ class SnippetsFinder < UnionFinder
   include FinderMethods
   include Gitlab::Utils::StrongMemoize
 
-  attr_accessor :current_user, :params
-  delegate :explore, :only_personal, :only_project, :scope, :sort, to: :params
+  attr_reader :current_user, :params
 
   def initialize(current_user = nil, params = {})
     @current_user = current_user
-    @params = OpenStruct.new(params)
+    @params = params
 
     if project && author
       raise(
@@ -77,9 +76,9 @@ def execute
   private
 
   def init_collection
-    if explore
+    if explore?
       snippets_for_explore
-    elsif only_personal
+    elsif only_personal?
       personal_snippets
     elsif project
       snippets_for_a_single_project
@@ -110,7 +109,7 @@ def snippets_for_explore
   # over the resulting SQL query.
   def snippets_for_personal_and_multiple_projects
     queries = []
-    queries << personal_snippets unless only_project
+    queries << personal_snippets unless only_project?
 
     if Ability.allowed?(current_user, :read_cross_project)
       queries << snippets_of_visible_projects
@@ -171,7 +170,7 @@ def snippets_for_author
   end
 
   def visibility_from_scope
-    case scope.to_s
+    case params[:scope].to_s
     when 'are_private'
       Snippet::PRIVATE
     when 'are_internal'
@@ -206,7 +205,19 @@ def project
   end
 
   def sort_param
-    sort.presence || 'id_desc'
+    params[:sort].presence || 'id_desc'
+  end
+
+  def explore?
+    params[:explore].present?
+  end
+
+  def only_personal?
+    params[:only_personal].present?
+  end
+
+  def only_project?
+    params[:only_project].present?
   end
 
   def prepared_union(queries)

app/finders/tags_finder.rb

@@ -8,8 +8,6 @@ def initialize(repository, params)
   def execute
     tags = repository.tags_sorted_by(sort)
 
-    [by_search(tags), nil]
-  rescue Gitlab::Git::CommandError => e
-    [[], e]
+    by_search(tags)
   end
 end

app/models/namespace.rb

@@ -17,6 +17,9 @@ class Namespace < ApplicationRecord
   include EachBatch
 
   ignore_column :delayed_project_removal, remove_with: '14.1', remove_after: '2021-05-22'
+  # Temporary column used for back-filling project namespaces.
+  # Remove it once the back-filling of all project namespaces is done.
+  ignore_column :tmp_project_id, remove_with: '14.7', remove_after: '2022-01-22'
 
   # Tells ActiveRecord not to store the full class name, in order to save some space
   # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69794

app/views/admin/users/_users.html.haml

@@ -3,48 +3,38 @@
     = sprite_icon('chevron-lg-left', size: 12)
   .fade-right
     = sprite_icon('chevron-lg-right', size: 12)
-  %ul.nav-links.nav.nav-tabs.scrolling-tabs
-    = nav_link(html_options: { class: active_when(params[:filter].nil?) }) do
-      = link_to admin_users_path do
-        = s_('AdminUsers|Active')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.active_without_ghosts)
-    = nav_link(html_options: { class: active_when(params[:filter] == 'admins') }) do
-      = link_to admin_users_path(filter: "admins") do
-        = s_('AdminUsers|Admins')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.admins)
-    = nav_link(html_options: { class: "#{active_when(params[:filter] == 'two_factor_enabled')} filter-two-factor-enabled" }) do
-      = link_to admin_users_path(filter: 'two_factor_enabled') do
-        = s_('AdminUsers|2FA Enabled')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.with_two_factor)
-    = nav_link(html_options: { class: "#{active_when(params[:filter] == 'two_factor_disabled')} filter-two-factor-disabled" }) do
-      = link_to admin_users_path(filter: 'two_factor_disabled') do
-        = s_('AdminUsers|2FA Disabled')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.without_two_factor)
-    = nav_link(html_options: { class: active_when(params[:filter] == 'external') }) do
-      = link_to admin_users_path(filter: 'external') do
-        = s_('AdminUsers|External')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.external)
-    = nav_link(html_options: { class: active_when(params[:filter] == 'blocked') }) do
-      = link_to admin_users_path(filter: "blocked") do
-        = s_('AdminUsers|Blocked')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.blocked)
+  = gl_tabs_nav({ class: 'scrolling-tabs nav-links gl-display-flex gl-flex-grow-1' }) do
+    = gl_tab_link_to admin_users_path, { item_active: active_when(params[:filter].nil?), class: 'gl-border-0!' } do
+      = s_('AdminUsers|Active')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.active_without_ghosts))
+    = gl_tab_link_to admin_users_path(filter: "admins"), { item_active: active_when(params[:filter] == 'admins'), class: 'gl-border-0!' } do
+      = s_('AdminUsers|Admins')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.admins))
+    = gl_tab_link_to admin_users_path(filter: 'two_factor_enabled'), { item_active: active_when(params[:filter] == 'two_factor_enabled'), class: 'filter-two-factor-enabled gl-border-0!' } do
+      = s_('AdminUsers|2FA Enabled')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.with_two_factor))
+    = gl_tab_link_to admin_users_path(filter: 'two_factor_disabled'), { item_active: active_when(params[:filter] == 'two_factor_disabled'), class: 'filter-two-factor-disabled gl-border-0!' } do
+      = s_('AdminUsers|2FA Disabled')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.without_two_factor))
+    = gl_tab_link_to admin_users_path(filter: 'external'), { item_active: active_when(params[:filter] == 'external'), class: 'gl-border-0!' } do
+      = s_('AdminUsers|External')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.external))
+    = gl_tab_link_to admin_users_path(filter: "blocked"), { item_active: active_when(params[:filter] == 'blocked'), class: 'gl-border-0!' } do
+      = s_('AdminUsers|Blocked')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.blocked))
     - if ban_feature_available?
-      = nav_link(html_options: { class: active_when(params[:filter] == 'banned') }) do
-        = link_to admin_users_path(filter: "banned") do
-          = s_('AdminUsers|Banned')
-          %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.banned)
-    = nav_link(html_options: { class: "#{active_when(params[:filter] == 'blocked_pending_approval')} filter-blocked-pending-approval" }) do
-      = link_to admin_users_path(filter: "blocked_pending_approval"), data: { qa_selector: 'pending_approval_tab' } do
-        = s_('AdminUsers|Pending approval')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.blocked_pending_approval)
-    = nav_link(html_options: { class: active_when(params[:filter] == 'deactivated') }) do
-      = link_to admin_users_path(filter: "deactivated") do
-        = s_('AdminUsers|Deactivated')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.deactivated)
-    = nav_link(html_options: { class: active_when(params[:filter] == 'wop') }) do
-      = link_to admin_users_path(filter: "wop") do
-        = s_('AdminUsers|Without projects')
-        %small.badge.gl-tab-counter-badge.badge-muted.badge-pill.gl-badge.sm= limited_counter_with_delimiter(User.without_projects)
+      = gl_tab_link_to admin_users_path(filter: "banned"), { item_active: active_when(params[:filter] == 'banned'), class: 'gl-border-0!' } do
+        = s_('AdminUsers|Banned')
+        = gl_tab_counter_badge(limited_counter_with_delimiter(User.banned))
+    = gl_tab_link_to admin_users_path(filter: "blocked_pending_approval"), { item_active: active_when(params[:filter] == 'blocked_pending_approval'), class: 'filter-blocked-pending-approval gl-border-0!', data: { qa_selector: 'pending_approval_tab' } } do
+      = s_('AdminUsers|Pending approval')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.blocked_pending_approval))
+    = gl_tab_link_to admin_users_path(filter: "deactivated"), { item_active: active_when(params[:filter] == 'deactivated'), class: 'gl-border-0!' } do
+      = s_('AdminUsers|Deactivated')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.deactivated))
+    = gl_tab_link_to admin_users_path(filter: "wop"), { item_active: active_when(params[:filter] == 'wop'), class: 'gl-border-0!' } do
+      = s_('AdminUsers|Without projects')
+      = gl_tab_counter_badge(limited_counter_with_delimiter(User.without_projects))
   .nav-controls
     = render_if_exists 'admin/users/admin_email_users'
     = render_if_exists 'admin/users/admin_export_user_permissions'

app/views/projects/blame/show.html.haml

@@ -20,7 +20,7 @@
       %span.legend-box.legend-box-9
       %span.right-label Older
 
-    .table-responsive.file-content.blame.code.js-syntax-highlight
+    .table-responsive.file-content.blame.code{ class: user_color_scheme }
       %table
         - current_line = 1
         - @blame.groups.each do |blame_group|

config/feature_flags/development/jupyter_clean_diffs.yml

@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/343433
 milestone: '14.5'
 type: development
 group: group::incubation
-default_enabled: false
+default_enabled: true

data/deprecations/14-5-deprecation-vsa-announce-deprecation-of-vsa-filtering-calculation.yml

@@ -0,0 +1,13 @@
+- name: "Value Stream Analytics filtering calculation change" # The name of the feature to be deprecated
+  announcement_milestone: "14.5" # The milestone when this feature was first announced as deprecated.
+  announcement_date: "2021-11-22" # The date of the milestone release when this feature was first announced as deprecated. This should almost always be the 22nd of a month (YYYY-MM-22), unless you did an out of band blog post.
+  removal_milestone: "15.0" # The milestone when this feature is planned to be removed
+  body: | # Do not modify this line, instead modify the lines below.
+    We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. 
+    
+    If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change.
+  stage: manage # (optional - may be required in the future) String value of the stage that the feature was created in. e.g., Growth
+  tiers: # (optional - may be required in the future) An array of tiers that the feature is available in currently.  e.g., [Free, Silver, Gold, Core, Premium, Ultimate]
+  issue_url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/343210' # (optional) This is a link to the deprecation issue in GitLab
+  documentation_url: "https://docs.gitlab.com/ee/user/analytics/value_stream_analytics.html#filter-value-stream-analytics-data" # (optional) This is a link to the current documentation page
+  image_url: "vsa_warning.png" # (optional) This is a link to a thumbnail image depicting the feature

db/migrate/20211018101034_add_tmp_project_id_column_to_namespaces.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddTmpProjectIdColumnToNamespaces < Gitlab::Database::Migration[1.0]
+  enable_lock_retries!
+
+  def change
+    # this is a temporary column to be able to batch insert records into namespaces table and then be able to link these
+    # to projects table.
+    add_column :namespaces, :tmp_project_id, :integer # rubocop: disable Migration/AddColumnsToWideTables
+  end
+end

db/post_migrate/20211018101552_add_index_to_tmp_project_id_column_on_namespaces_table.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddIndexToTmpProjectIdColumnOnNamespacesTable < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'tmp_index_on_tmp_project_id_on_namespaces'
+
+  def up
+    add_concurrent_index :namespaces, :tmp_project_id, name: INDEX_NAME, unique: true
+  end
+
+  def down
+    remove_concurrent_index_by_name :namespaces, INDEX_NAME
+  end
+end

db/post_migrate/20211018101852_add_fk_to_tmp_project_id_column_on_namespaces_table.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddFkToTmpProjectIdColumnOnNamespacesTable < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :namespaces, :projects, column: :tmp_project_id
+  end
+
+  def down
+    remove_foreign_key :namespaces, column: :tmp_project_id
+  end
+end

db/post_migrate/20211018102252_add_index_to_group_id_column_on_webhooks_table.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class AddIndexToGroupIdColumnOnWebhooksTable < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_on_group_id_on_webhooks'
+
+  def up
+    add_concurrent_index :web_hooks, :group_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :web_hooks, INDEX_NAME
+  end
+end

db/schema_migrations/20211018101034

@@ -0,0 +1 @@
+1cadc3a932d5b62cfeafcd4090eddc37b44997dbbd0b34da1c7c87a5774bb683

db/schema_migrations/20211018101552

@@ -0,0 +1 @@
+9a62f0ec43ab295619d82494090c38539cb16408c8971bdde86bb8d02546f558

db/schema_migrations/20211018101852

@@ -0,0 +1 @@
+30e9632877d3ad33528be0f56962c0ab57f5eee3889183d9638cbaea903a3d82

db/schema_migrations/20211018102252

@@ -0,0 +1 @@
+14bb815cbdad2db56dafb7eaaff893de96116a1a9e8d6c5ed95f4bef9b9717fc

db/structure.sql

@@ -16375,7 +16375,8 @@ CREATE TABLE namespaces (
     push_rule_id bigint,
     shared_runners_enabled boolean DEFAULT true NOT NULL,
     allow_descendants_override_disabled_shared_runners boolean DEFAULT false NOT NULL,
-    traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL
+    traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL,
+    tmp_project_id integer
 );
 
 CREATE SEQUENCE namespaces_id_seq
@@ -26591,6 +26592,8 @@ CREATE INDEX index_oauth_openid_requests_on_access_grant_id ON oauth_openid_requ
 
 CREATE UNIQUE INDEX index_on_deploy_keys_id_and_type_and_public ON keys USING btree (id, type) WHERE (public = true);
 
+CREATE INDEX index_on_group_id_on_webhooks ON web_hooks USING btree (group_id);
+
 CREATE INDEX index_on_identities_lower_extern_uid_and_provider ON identities USING btree (lower((extern_uid)::text), provider);
 
 CREATE UNIQUE INDEX index_on_instance_statistics_recorded_at_and_identifier ON analytics_usage_trends_measurements USING btree (identifier, recorded_at);
@@ -27769,6 +27772,8 @@ CREATE INDEX tmp_index_namespaces_empty_traversal_ids_with_child_namespaces ON n
 
 CREATE INDEX tmp_index_namespaces_empty_traversal_ids_with_root_namespaces ON namespaces USING btree (id) WHERE ((parent_id IS NULL) AND (traversal_ids = '{}'::integer[]));
 
+CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USING btree (tmp_project_id);
+
 CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2);
 
 CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name);
@@ -29012,6 +29017,9 @@ ALTER TABLE ONLY application_settings
 ALTER TABLE ONLY merge_requests
     ADD CONSTRAINT fk_6a5165a692 FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY namespaces
+    ADD CONSTRAINT fk_6a77f66919 FOREIGN KEY (tmp_project_id) REFERENCES projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY geo_event_log
     ADD CONSTRAINT fk_6ada82d42a FOREIGN KEY (container_repository_updated_event_id) REFERENCES geo_container_repository_updated_events(id) ON DELETE CASCADE;
 

doc/update/deprecations.md

@@ -157,6 +157,14 @@ In milestone 15.0, we will remove the feature flag entirely. Moving forward, you
 
 Announced: 2021-11-22
 
+### Value Stream Analytics filtering calculation change
+
+We are changing how the date filter works in Value Stream Analytics. Instead of filtering by the time that the issue or merge request was created, the date filter will filter by the end event time of the given stage. This will result in completely different figures after this change has rolled out. 
+
+If you monitor Value Stream Analytics metrics and rely on the date filter, to avoid losing data, you must save the data prior to this change.
+
+Announced: 2021-11-22
+
 ### `AuthenticationType` for `[runners.cache.s3]` must be explicitly assigned
 
 In GitLab 15.0 and later, to access the AWS S3 cache, you must specify the `AuthenticationType` for [`[runners.cache.s3]`](https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runnerscaches3-section). The `AuthenticationType` must be `IAM` or `credentials`.

lib/api/tags.rb

@@ -24,7 +24,7 @@ class Tags < ::API::Base
         use :pagination
       end
       get ':id/repository/tags', feature_category: :source_code_management do
-        tags, _ = ::TagsFinder.new(user_project.repository,
+        tags = ::TagsFinder.new(user_project.repository,
                                 sort: "#{params[:order_by]}_#{params[:sort]}",
                                 search: params[:search]).execute
 
@@ -35,6 +35,9 @@ class Tags < ::API::Base
         else
           present paginated_tags, with: Entities::Tag, project: user_project
         end
+
+      rescue Gitlab::Git::CommandError
+        service_unavailable!
       end
 
       desc 'Get a single repository tag' do