Merge pull request #66622 from flokli/19.03-gitlab-12.1.6

[19.03] gitlab updates

nixos/modules/services/misc/gitlab.nix

@@ -52,7 +52,7 @@ let
     gitlab_url = "http+unix://${pathUrlQuote gitlabSocket}";
     http_settings.self_signed_cert = false;
     repos_path = "${cfg.statePath}/repositories";
-    secret_file = "${cfg.statePath}/config/gitlab_shell_secret";
+    secret_file = "${cfg.statePath}/gitlab_shell_secret";
     log_file = "${cfg.statePath}/log/gitlab-shell.log";
     custom_hooks_dir = "${cfg.statePath}/custom_hooks";
     redis = {
@@ -109,7 +109,7 @@ let
       gitlab_shell = {
         path = "${cfg.packages.gitlab-shell}";
         hooks_path = "${cfg.statePath}/shell/hooks";
-        secret_file = "${cfg.statePath}/config/gitlab_shell_secret";
+        secret_file = "${cfg.statePath}/gitlab_shell_secret";
         upload_pack = true;
         receive_pack = true;
       };
@@ -132,14 +132,9 @@ let
     HOME = "${cfg.statePath}/home";
     UNICORN_PATH = "${cfg.statePath}/";
     GITLAB_PATH = "${cfg.packages.gitlab}/share/gitlab/";
-    GITLAB_STATE_PATH = cfg.statePath;
-    GITLAB_UPLOADS_PATH = "${cfg.statePath}/uploads";
     SCHEMA = "${cfg.statePath}/db/schema.rb";
+    GITLAB_UPLOADS_PATH = "${cfg.statePath}/uploads";
     GITLAB_LOG_PATH = "${cfg.statePath}/log";
-    GITLAB_SHELL_PATH = "${cfg.packages.gitlab-shell}";
-    GITLAB_SHELL_CONFIG_PATH = "${cfg.statePath}/shell/config.yml";
-    GITLAB_SHELL_SECRET_PATH = "${cfg.statePath}/config/gitlab_shell_secret";
-    GITLAB_SHELL_HOOKS_PATH = "${cfg.statePath}/shell/hooks";
     GITLAB_REDIS_CONFIG_FILE = pkgs.writeText "redis.yml" (builtins.toJSON redisConfig);
     prometheus_multiproc_dir = "/run/gitlab";
     RAILS_ENV = "production";
@@ -174,6 +169,8 @@ let
      '';
   };
 
+  extraGitlabRb = pkgs.writeText "extra-gitlab.rb" cfg.extraGitlabRb;
+
   smtpSettings = pkgs.writeText "gitlab-smtp-settings.rb" ''
     if Rails.env.production?
       Rails.application.config.action_mailer.delivery_method = :smtp
@@ -280,6 +277,26 @@ in {
         description = "Extra configuration in config/database.yml.";
       };
 
+      extraGitlabRb = mkOption {
+        type = types.str;
+        default = "";
+        example = ''
+          if Rails.env.production?
+            Rails.application.config.action_mailer.delivery_method = :sendmail
+            ActionMailer::Base.delivery_method = :sendmail
+            ActionMailer::Base.sendmail_settings = {
+              location: "/run/wrappers/bin/sendmail",
+              arguments: "-i -t"
+            }
+          end
+        '';
+        description = ''
+          Extra configuration to be placed in config/extra-gitlab.rb. This can
+          be used to add configuration not otherwise exposed through this module's
+          options.
+        '';
+      };
+
       host = mkOption {
         type = types.str;
         default = config.networking.hostName;
@@ -480,23 +497,44 @@ in {
     systemd.tmpfiles.rules = [
       "d /run/gitlab 0755 ${cfg.user} ${cfg.group} -"
       "d ${gitlabEnv.HOME} 0750 ${cfg.user} ${cfg.group} -"
+      "z ${gitlabEnv.HOME}/.ssh/authorized_keys 0600 ${cfg.user} ${cfg.group} -"
       "d ${cfg.backupPath} 0750 ${cfg.user} ${cfg.group} -"
+      "d ${cfg.statePath} 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/builds 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/config 0750 ${cfg.user} ${cfg.group} -"
+      "D ${cfg.statePath}/config/initializers 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/db 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/log 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/repositories 2770 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/shell 0750 ${cfg.user} ${cfg.group} -"
+      "d ${cfg.statePath}/tmp 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/tmp/pids 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/tmp/sockets 0750 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/uploads 0700 ${cfg.user} ${cfg.group} -"
+      "d ${cfg.statePath}/custom_hooks 0700 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/custom_hooks/pre-receive.d 0700 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/custom_hooks/post-receive.d 0700 ${cfg.user} ${cfg.group} -"
       "d ${cfg.statePath}/custom_hooks/update.d 0700 ${cfg.user} ${cfg.group} -"
+      "d ${gitlabConfig.production.shared.path} 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/artifacts 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/lfs-objects 0750 ${cfg.user} ${cfg.group} -"
       "d ${gitlabConfig.production.shared.path}/pages 0750 ${cfg.user} ${cfg.group} -"
-    ];
+      "L+ ${cfg.statePath}/lib - - - - ${cfg.packages.gitlab}/share/gitlab/lib"
+      "L+ /run/gitlab/config - - - - ${cfg.statePath}/config"
+      "L+ /run/gitlab/log - - - - ${cfg.statePath}/log"
+      "L+ /run/gitlab/tmp - - - - ${cfg.statePath}/tmp"
+      "L+ /run/gitlab/uploads - - - - ${cfg.statePath}/uploads"
+
+      "L+ /run/gitlab/shell-config.yml - - - - ${pkgs.writeText "config.yml" (builtins.toJSON gitlabShellConfig)}"
+
+      "L+ ${cfg.statePath}/config/gitlab.yml - - - - ${pkgs.writeText "gitlab.yml" (builtins.toJSON gitlabConfig)}"
+      "L+ ${cfg.statePath}/config/database.yml - - - - ${pkgs.writeText "database.yml" (builtins.toJSON databaseConfig)}"
+      "L+ ${cfg.statePath}/config/secrets.yml - - - - ${pkgs.writeText "secrets.yml" (builtins.toJSON secretsConfig)}"
+      "L+ ${cfg.statePath}/config/unicorn.rb - - - - ${./defaultUnicornConfig.rb}"
+
+      "L+ ${cfg.statePath}/config/initializers/extra-gitlab.rb - - - - ${extraGitlabRb}"
+    ] ++ optional cfg.smtp.enable
+      "L+ ${cfg.statePath}/config/initializers/smtp_settings.rb - - - - ${smtpSettings}" ;
 
     systemd.services.gitlab-sidekiq = {
       after = [ "network.target" "redis.service" "gitlab.service" ];
@@ -588,39 +626,14 @@ in {
         gnupg
       ];
       preStart = ''
-        cp -rf ${cfg.packages.gitlab}/share/gitlab/db/* ${cfg.statePath}/db
-        rm -rf ${cfg.statePath}/config
-        mkdir ${cfg.statePath}/config
-        if [ -e ${cfg.statePath}/lib ]; then
-          rm ${cfg.statePath}/lib
-        fi
+        ${pkgs.sudo}/bin/sudo -u ${cfg.user} cp -f ${cfg.packages.gitlab}/share/gitlab/VERSION ${cfg.statePath}/VERSION
+        ${pkgs.sudo}/bin/sudo -u ${cfg.user} rm -rf ${cfg.statePath}/db/*
+        ${pkgs.sudo}/bin/sudo -u ${cfg.user} cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config
+        ${pkgs.sudo}/bin/sudo -u ${cfg.user} cp -rf --no-preserve=mode ${cfg.packages.gitlab}/share/gitlab/db/* ${cfg.statePath}/db
 
-        ln -sf ${cfg.packages.gitlab}/share/gitlab/lib ${cfg.statePath}/lib
-        [ -L /run/gitlab/config ] || ln -sf ${cfg.statePath}/config /run/gitlab/config
-        [ -L /run/gitlab/log ] || ln -sf ${cfg.statePath}/log /run/gitlab/log
-        [ -L /run/gitlab/tmp ] || ln -sf ${cfg.statePath}/tmp /run/gitlab/tmp
-        [ -L /run/gitlab/uploads ] || ln -sf ${cfg.statePath}/uploads /run/gitlab/uploads
-        cp ${cfg.packages.gitlab}/share/gitlab/VERSION ${cfg.statePath}/VERSION
-        cp -rf ${cfg.packages.gitlab}/share/gitlab/config.dist/* ${cfg.statePath}/config
-        ${optionalString cfg.smtp.enable ''
-          ln -sf ${smtpSettings} ${cfg.statePath}/config/initializers/smtp_settings.rb
-        ''}
-        ${pkgs.openssl}/bin/openssl rand -hex 32 > ${cfg.statePath}/config/gitlab_shell_secret
-
-        # JSON is a subset of YAML
-        ln -sf ${pkgs.writeText "gitlab.yml" (builtins.toJSON gitlabConfig)} ${cfg.statePath}/config/gitlab.yml
-        ln -sf ${pkgs.writeText "database.yml" (builtins.toJSON databaseConfig)} ${cfg.statePath}/config/database.yml
-        ln -sf ${pkgs.writeText "secrets.yml" (builtins.toJSON secretsConfig)} ${cfg.statePath}/config/secrets.yml
-        ln -sf ${./defaultUnicornConfig.rb} ${cfg.statePath}/config/unicorn.rb
-
-        # Install the shell required to push repositories
-        ln -sf ${pkgs.writeText "config.yml" (builtins.toJSON gitlabShellConfig)} /run/gitlab/shell-config.yml
-        [ -L ${cfg.statePath}/shell/hooks ] ||  ln -sf ${cfg.packages.gitlab-shell}/hooks ${cfg.statePath}/shell/hooks
-        ${cfg.packages.gitlab-shell}/bin/install
-
-        chown -R ${cfg.user}:${cfg.group} ${cfg.statePath}/
-        chmod -R ug+rwX,o-rwx+X ${cfg.statePath}/
-        chown -R ${cfg.user}:${cfg.group} /run/gitlab
+        ${pkgs.openssl}/bin/openssl rand -hex 32 > ${cfg.statePath}/gitlab_shell_secret
+
+        ${pkgs.sudo}/bin/sudo -u ${cfg.user} ${cfg.packages.gitlab-shell}/bin/install
 
         if ! test -e "${cfg.statePath}/db-created"; then
           if [ "${cfg.databaseHost}" = "127.0.0.1" ]; then
@@ -633,7 +646,7 @@ in {
 
           ${pkgs.sudo}/bin/sudo -u ${cfg.user} -H ${gitlab-rake}/bin/gitlab-rake db:schema:load
 
-          touch "${cfg.statePath}/db-created"
+          ${pkgs.sudo}/bin/sudo -u ${cfg.user} touch "${cfg.statePath}/db-created"
         fi
 
         # Always do the db migrations just to be sure the database is up-to-date
@@ -642,22 +655,13 @@ in {
         if ! test -e "${cfg.statePath}/db-seeded"; then
           ${pkgs.sudo}/bin/sudo -u ${cfg.user} ${gitlab-rake}/bin/gitlab-rake db:seed_fu \
             GITLAB_ROOT_PASSWORD='${cfg.initialRootPassword}' GITLAB_ROOT_EMAIL='${cfg.initialRootEmail}'
-          touch "${cfg.statePath}/db-seeded"
+          ${pkgs.sudo}/bin/sudo -u ${cfg.user} touch "${cfg.statePath}/db-seeded"
         fi
 
-        # The gitlab:shell:create_hooks task seems broken for fixing links
-        # so we instead delete all the hooks and create them anew
-        rm -f ${cfg.statePath}/repositories/**/*.git/hooks
-        ${pkgs.sudo}/bin/sudo -u ${cfg.user} -H ${gitlab-rake}/bin/gitlab-rake gitlab:shell:create_hooks
+        # We remove potentially broken links to old gitlab-shell versions
+        rm -Rf ${cfg.statePath}/repositories/**/*.git/hooks
 
         ${pkgs.sudo}/bin/sudo -u ${cfg.user} -H ${pkgs.git}/bin/git config --global core.autocrlf "input"
-
-        # Change permissions in the last step because some of the
-        # intermediary scripts like to create directories as root.
-        chmod -R u+rwX,go-rwx+X ${gitlabEnv.HOME}
-        chmod -R ug+rwX,o-rwx ${cfg.statePath}/repositories
-        chmod -R ug-s ${cfg.statePath}/repositories
-        find ${cfg.statePath}/repositories -type d -print0 | xargs -0 chmod g+s
       '';
 
       serviceConfig = {

pkgs/applications/version-management/gitlab/data.json

@@ -1,32 +1,32 @@
 {
   "ce": {
-    "version": "11.10.8",
-    "repo_hash": "1ygwkajkwhr2vzkzljfj6l1ypvmmzj9ps8ijha5m9qglkzjz0gsn",
-    "deb_hash": "1b66yw0i795pahainx3rpqaliffmn5py0ws8is8f5hr7cghchln0",
-    "deb_url": "https://packages.gitlab.com/gitlab/gitlab-ce/packages/debian/stretch/gitlab-ce_11.10.8-ce.0_amd64.deb/download.deb",
+    "version": "12.1.6",
+    "repo_hash": "0zyqxzyb1m8qa94iiyqf5ivqxrg60y378r94fsixy17fbh4y7sa6",
+    "deb_hash": "14bwcmwar44pjnjr6fdn2h9y92vyfva0kb76d4pz8pqc9pfm2hmg",
+    "deb_url": "https://packages.gitlab.com/gitlab/gitlab-ce/packages/debian/stretch/gitlab-ce_12.1.6-ce.0_amd64.deb/download.deb",
     "owner": "gitlab-org",
     "repo": "gitlab-ce",
-    "rev": "v11.10.8",
+    "rev": "v12.1.6",
     "passthru": {
-      "GITALY_SERVER_VERSION": "1.34.3",
-      "GITLAB_PAGES_VERSION": "1.5.0",
-      "GITLAB_SHELL_VERSION": "9.0.0",
-      "GITLAB_WORKHORSE_VERSION": "8.5.2"
+      "GITALY_SERVER_VERSION": "1.53.2",
+      "GITLAB_PAGES_VERSION": "1.7.1",
+      "GITLAB_SHELL_VERSION": "9.3.0",
+      "GITLAB_WORKHORSE_VERSION": "8.7.0"
     }
   },
   "ee": {
-    "version": "11.10.8",
-    "repo_hash": "1vw0d99w5bvagbl9xia5ik3754s7jgkh01b3wm77snfkcg31psb6",
-    "deb_hash": "05fsjxlr56zv4wc90r6ns81n5h1ykafjsi8vq6h22gzxjjvaay7m",
-    "deb_url": "https://packages.gitlab.com/gitlab/gitlab-ee/packages/debian/stretch/gitlab-ee_11.10.8-ee.0_amd64.deb/download.deb",
+    "version": "12.1.6",
+    "repo_hash": "1lxci1hwcccgw567c5733yl95xsxr4gnw41m48igdw9y8p6g7zbc",
+    "deb_hash": "0yjilhf4bjhl1a596x6n4cvjklfbrwhymslhx977nfjazfb5slx3",
+    "deb_url": "https://packages.gitlab.com/gitlab/gitlab-ee/packages/debian/stretch/gitlab-ee_12.1.6-ee.0_amd64.deb/download.deb",
     "owner": "gitlab-org",
     "repo": "gitlab-ee",
-    "rev": "v11.10.8-ee",
+    "rev": "v12.1.6-ee",
     "passthru": {
-      "GITALY_SERVER_VERSION": "1.34.3",
-      "GITLAB_PAGES_VERSION": "1.5.0",
-      "GITLAB_SHELL_VERSION": "9.0.0",
-      "GITLAB_WORKHORSE_VERSION": "8.5.2"
+      "GITALY_SERVER_VERSION": "1.53.2",
+      "GITLAB_PAGES_VERSION": "1.7.1",
+      "GITLAB_SHELL_VERSION": "9.3.0",
+      "GITLAB_WORKHORSE_VERSION": "8.7.0"
     }
   }
 }

pkgs/applications/version-management/gitlab/default.nix

@@ -1,14 +1,29 @@
 { stdenv, lib, fetchurl, fetchFromGitLab, bundlerEnv
-, ruby, tzdata, git, nettools
+, ruby, tzdata, git, nettools, nixosTests
 , gitlabEnterprise ? false
 }:
 
 let
-  rubyEnv = bundlerEnv {
+  rubyEnv = bundlerEnv rec {
     name = "gitlab-env-${version}";
     inherit ruby;
     gemdir = ./rubyEnv- + "${if gitlabEnterprise then "ee" else "ce"}";
-    groups = [ "default" "unicorn" "ed25519" "metrics" ];
+    gemset =
+      let x = import (gemdir + "/gemset.nix");
+      in x // {
+        # grpc expects the AR environment variable to contain `ar rpc`. See the
+        # discussion in nixpkgs #63056.
+        grpc = x.grpc // {
+          patches = [ ./fix-grpc-ar.patch ];
+          dontBuild = false;
+        };
+      };
+    groups = [
+      "default" "unicorn" "ed25519" "metrics" "development" "puma" "test"
+    ];
+    # N.B. omniauth_oauth2_generic and apollo_upload_server both provide a
+    # `console` executable.
+    ignoreCollisions = true;
   };
 
   flavour = if gitlabEnterprise then "ee" else "ce";
@@ -61,6 +76,7 @@ stdenv.mkDerivation rec {
     # Work around unpacking deb containing binary with suid bit
     tar -f gitlab-deb-data.tar --delete ./opt/gitlab/embedded/bin/ksu
     tar -xf gitlab-deb-data.tar
+    rm gitlab-deb-data.tar
 
     mv -v opt/gitlab/embedded/service/gitlab-rails/public/assets public
     rm -rf opt # only directory in data.tar.gz
@@ -92,6 +108,9 @@ stdenv.mkDerivation rec {
     GITLAB_PAGES_VERSION = data.passthru.GITLAB_PAGES_VERSION;
     GITLAB_SHELL_VERSION = data.passthru.GITLAB_SHELL_VERSION;
     GITLAB_WORKHORSE_VERSION = data.passthru.GITLAB_WORKHORSE_VERSION;
+    tests = {
+      nixos-test-passes = nixosTests.gitlab;
+    };
   };
 
   meta = with lib; {

pkgs/applications/version-management/gitlab/fix-grpc-ar.patch

@@ -0,0 +1,10 @@
+--- a/src/ruby/ext/grpc/extconf.rb
++++ b/src/ruby/ext/grpc/extconf.rb
+@@ -27,6 +27,7 @@ ENV['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
+ if ENV['AR'].nil? || ENV['AR'].size == 0
+     ENV['AR'] = RbConfig::CONFIG['AR'] + ' rcs'
+ end
++ENV['AR'] = ENV['AR'] + ' rcs'
+ if ENV['CC'].nil? || ENV['CC'].size == 0
+     ENV['CC'] = RbConfig::CONFIG['CC']
+ end

pkgs/applications/version-management/gitlab/gitaly/Gemfile

@@ -1,29 +1,32 @@
 source 'https://rubygems.org'
 
 # Require bundler >= 1.16.5 to avoid this bug: https://github.com/bundler/bundler/issues/6537
-gem 'bundler', '>= 1.16.5'
+gem 'bundler', '>= 1.17.3'
 
 gem 'rugged', '~> 0.28'
 gem 'github-linguist', '~> 6.1', require: 'linguist'
 gem 'gitlab-markup', '~> 1.7.0'
-gem 'gitaly-proto', '~> 1.22.0'
-gem 'activesupport', '~> 5.0.2'
+gem 'activesupport', '~> 5.1.7'
+gem 'gitaly-proto', '~> 1.36.0'
 gem 'rdoc', '~> 4.2'
 gem 'gitlab-gollum-lib', '~> 4.2.7.7', require: false
 gem 'gitlab-gollum-rugged_adapter', '~> 0.4.4.2', require: false
-gem 'grpc', '~> 1.15.0'
+gem 'grpc', '~> 1.19.0'
 gem 'sentry-raven', '~> 2.9.0', require: false
 gem 'faraday', '~> 0.12'
 gem 'rbtrace', require: false
 
+# Labkit provides observability functionality
+gem 'gitlab-labkit', '~> 0.3.0'
+
 # Detects the open source license the repository includes
 # This version needs to be in sync with GitLab CE/EE
 gem 'licensee', '~> 8.9.0'
 
-gem 'google-protobuf', '~> 3.6'
+gem 'google-protobuf', '~> 3.7.1'
 
 group :development, :test do
-  gem 'rubocop', '~> 0.50', require: false
+  gem 'rubocop', '~> 0.69', require: false
   gem 'rspec', require: false
   gem 'rspec-parameterized', require: false
   gem 'timecop', require: false