Merge branch 'hh/sqitch-upgrade'

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: git://github.com/opscode/omnibus-software.git
-  revision: f37700aaa3752d98d2e3d38cb6704bd850efc8ba
+  revision: 5ee2eae4a159f3c20dccf89e64a3532eadba2eff
   branch: master
   specs:
     omnibus-software (0.0.1)
@@ -11,14 +11,14 @@ GEM
     addressable (2.3.5)
     arr-pm (0.0.8)
       cabin (> 0)
-    backports (3.3.5)
+    backports (3.5.0)
     cabin (0.6.1)
-    childprocess (0.3.9)
+    childprocess (0.4.0)
       ffi (~> 1.0, >= 1.0.11)
     clamp (0.6.3)
     ffi (1.9.3)
     ffi (1.9.3-x86-mingw32)
-    fpm (1.0.0)
+    fpm (1.0.2)
       arr-pm (~> 0.0.8)
       backports (>= 2.6.2)
       cabin (>= 0.6.0)
@@ -27,7 +27,7 @@ GEM
       ffi
       ftw (~> 0.0.30)
       json (>= 1.7.7)
-    ftw (0.0.37)
+    ftw (0.0.39)
       addressable
       backports (>= 2.6.2)
       cabin (> 0)
@@ -60,7 +60,7 @@ GEM
       rake (>= 0.9)
       thor (>= 0.16.0)
       uber-s3
-    rake (10.1.0)
+    rake (10.1.1)
     systemu (2.5.2)
     thor (0.18.1)
     uber-s3 (0.2.4)
@@ -73,8 +73,7 @@ GEM
     windows-pr (1.2.2)
       win32-api (>= 1.4.5)
       windows-api (>= 0.3.0)
-    yajl-ruby (1.1.0)
-    yajl-ruby (1.1.0-x86-mingw32)
+    yajl-ruby (1.2.0)
 
 PLATFORMS
   ruby

config/projects/chef-server-core.rb

@@ -41,6 +41,7 @@
 dependency "chef-solr"
 dependency "chef-expander"
 dependency "bookshelf" # S3 API compatible object store
+dependency "chef-server-schema"
 
 # the front-end services
 dependency "erchef" # the actual Chef Server REST API

config/projects/chef-server.rb

@@ -42,6 +42,7 @@
 dependency "chef-solr"
 dependency "chef-expander"
 dependency "bookshelf" # S3 API compatible object store
+dependency "chef-server-schema"
 
 # the front-end services
 dependency "erchef" # the actual Chef Server REST API

config/software/chef-server-cookbooks.rb

@@ -24,4 +24,9 @@
 build do
   command "mkdir -p #{install_dir}/embedded/cookbooks"
   command "#{install_dir}/embedded/bin/rsync --delete -a ./ #{install_dir}/embedded/cookbooks/"
+  block do
+    File.open("#{install_dir}/embedded/cookbooks/pre_upgrade_setup.json", "w") do |f|
+      f.puts "{\"run_list\": [ \"recipe[#{project.name}::pre_11.1_upgrade_setup]\" ]}"
+    end
+  end
 end

config/software/chef-server-schema.rb

@@ -0,0 +1,11 @@
+name "chef-server-schema"
+version "1.0.4"
+
+source :git => "git@github.com:opscode/chef-server-schema.git"
+
+dependency "sqitch"
+
+build do
+  command "mkdir -p #{install_dir}/embedded/service/#{name}"
+  command "#{install_dir}/embedded/bin/rsync -a --delete --exclude=.git/*** --exclude=.gitignore ./ #{install_dir}/embedded/service/#{name}/"
+end

config/software/sqitch.rb

@@ -0,0 +1,29 @@
+name "sqitch"
+version "0.973"
+
+dependency "perl"
+dependency "postgresql"
+
+source :url => "http://www.cpan.org/authors/id/D/DW/DWHEELER/App-Sqitch-#{version}.tar.gz",
+       :md5 => "0994e9f906a7a4a2e97049c8dbaef584"
+
+relative_path "App-Sqitch-#{version}"
+
+env = {
+  "PATH" => "#{install_dir}/embedded/bin:#{ENV["PATH"]}"
+}
+
+# Ensure we install with the properly configured embedded `cpan` client
+omnibus_cpan_client = "#{install_dir}/embedded/bin/cpan -j #{cache_dir}/cpan/OmnibusConfig.pm"
+
+# See https://github.com/theory/sqitch for more
+build do
+  command "perl Build.PL", :env => env
+  command "./Build installdeps --cpan_client '#{omnibus_cpan_client}'", :env => env
+  command "./Build", :env => env
+  command "./Build test", :env => env
+  command "./Build install", :env => env
+
+  # We're using PostgreSQL as our database engine, so we need the right driver
+  command "yes | #{omnibus_cpan_client} DBD::Pg", :env => env
+end

files/chef-server-cookbooks/chef-server/files/default/sql/widen-cookbook-version.sql

@@ -0,0 +1,100 @@
+-- Copyright 2013 Opscode, Inc. All Rights Reserved.
+--
+-- This file is provided to you under the Apache License,
+-- Version 2.0 (the "License"); you may not use this file
+-- except in compliance with the License.  You may obtain
+-- a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+-- @author Oliver Ferrigni <oliver@opscode.com>
+-- @author Ho-Sheng Hsiao <hosh@opscode.com>
+-- See: https://github.com/opscode/chef_db/commit/389bb2c55f46f2148cbb976dac8dfbb08a1d0d5f
+
+START TRANSACTION;
+
+DROP VIEW cookbook_versions_by_rank;
+DROP VIEW cookbook_version_dependencies;
+DROP VIEW joined_cookbook_version;
+
+ALTER TABLE cookbook_versions
+  ALTER major SET DATA TYPE bigint,
+  ALTER minor SET DATA TYPE bigint,
+  ALTER patch SET DATA TYPE bigint;
+
+CREATE OR REPLACE VIEW cookbook_versions_by_rank(
+        -- Cookbook Version fields
+        major, -- these 3 are needed for version information (duh)
+        minor,
+        patch,
+        version, -- concatenated string of the complete version
+        serialized_object, -- needed to access recipe manifest
+        -- Cookbook fields
+        org_id, -- used for filtering
+        name, -- both version and recipe queries require the cookbook name
+        -- View-specific fields
+        -- (also used for filtering)
+        rank) AS
+SELECT v.major,
+       v.minor,
+       v.patch,
+       v.major || '.' || v.minor || '.' || v.patch,
+       v.serialized_object,
+       c.org_id,
+       c.name,
+       rank() OVER (PARTITION BY v.cookbook_id
+                              ORDER BY v.major DESC, v.minor DESC, v.patch DESC)
+FROM cookbooks AS c
+JOIN cookbook_versions AS v
+  ON c.id = v.cookbook_id;
+
+CREATE OR REPLACE VIEW joined_cookbook_version(
+        -- Cookbook Version fields
+        major, -- these 3 are needed for version information (duh)
+        minor,
+        patch,
+        version, -- concatenated string of the complete version
+        serialized_object, -- needed to access recipe manifest
+        id, -- used for retrieving environment-filtered recipes
+        -- Cookbook fields
+        org_id, -- used for filtering
+        name) -- both version and recipe queries require the cookbook name
+AS
+SELECT v.major,
+       v.minor,
+       v.patch,
+       v.major || '.' || v.minor || '.' || v.patch,
+       v.serialized_object,
+       v.id,
+       c.org_id,
+       c.name
+FROM cookbooks AS c
+JOIN cookbook_versions AS v
+  ON c.id = v.cookbook_id;
+
+CREATE OR REPLACE VIEW cookbook_version_dependencies(
+        org_id, -- for filtering
+        name, -- cookbook name
+        major,
+        minor,
+        patch,
+        dependencies) -- version dependency JSON blob; needed for depsolver
+AS
+SELECT c.org_id,
+       c.name,
+       v.major,
+       v.minor,
+       v.patch,
+       v.meta_deps
+FROM cookbooks AS c
+JOIN cookbook_versions AS v
+  ON c.id = v.cookbook_id;
+
+COMMIT;

files/chef-server-cookbooks/chef-server/libraries/helper.rb

@@ -24,6 +24,13 @@ def initialize(node)
     @node = node
   end
 
+  def run_command(cmd, options)
+    cmd = Mixlib::ShellOut.new(cmd, options)
+    cmd.run_command
+    cmd
+  end
+
+
   def is_running?
     OmnibusHelper.service_up?("postgresql")
   end
@@ -34,14 +41,32 @@ def database_exists?(db_name)
               "| grep #{db_name}"])
   end
 
+  def managed_by_sqitch?
+    cmd = run_command "sqitch --db-user #{db_user} verify",
+      :cwd     => '/opt/chef-server/embedded/service/chef-server-schema',
+      :returns => [0, 1, 2]
+    cmd.exitstatus == 0
+  end
+
+  def needs_schema_update?
+    cmd = run_command "sqitch --db-user #{db_user} status",
+      :cwd     => '/opt/chef-server/embedded/service/chef-server-schema',
+      :returns => [0, 1, 2]
+    cmd.stdout !~ /^Nothing to deploy \(up-to-date\)$/
+  end
+
+  def db_user
+    node['chef_server']['postgresql']['username']
+  end
+
   def sql_user_exists?
     user_exists?(node['chef_server']['postgresql']['sql_user'])
   end
 
   def sql_ro_user_exists?
     user_exists?(node['chef_server']['postgresql']['sql_ro_user'])
   end
-  
+
   def user_exists?(db_user)
     psql_cmd(["-d 'template1'",
               "-c 'select usename from pg_user' -x",

files/chef-server-cookbooks/chef-server/recipes/postgresql.rb

@@ -145,12 +145,12 @@
   user pg_user
   not_if { !pg_helper.is_running? || pg_helper.database_exists?(db_name) }
   retries 30
-  notifies :run, "execute[migrate_database]", :immediately
+  notifies :run, 'execute[install_schema]', :immediately
 end
 
-execute "migrate_database" do
-  command "#{bin_dir}/psql #{db_name} --port #{pg_port} < priv/pgsql_schema.sql"
-  cwd chef_db_dir
+execute "install_schema" do
+  command "sqitch --db-user #{pg_user} deploy --verify" # same as preflight
+  cwd "/opt/chef-server/embedded/service/chef-server-schema"
   user pg_user
   action :nothing
 end

files/chef-server-cookbooks/chef-server/recipes/pre_11.1_upgrade_setup.rb

@@ -0,0 +1,43 @@
+# Brings up an existing database for sqitch. We assume you installed the schema
+# via omnibus. If this does not work, it would be better to use knife to backup
+# the chef server and then restore it after installing the new version of the chef-server.
+# We don't expect this to be needed in the future, since Chef Server 11.1+ will have
+# sqitch-managed databases.
+
+pg_helper = PgHelper.new(node)
+pg_user  = pg_helper.db_user
+cookbook = run_context.cookbook_collection['chef-server']
+sql_file = cookbook.preferred_filename_on_disk_location(node, :files, 'sql/widen-cookbook-version.sql', nil)
+
+# Make sure postgresql is running so we can check the db schema on file.
+execute '/opt/chef-server/bin/chef-server-ctl start postgresql' do
+  retries 20
+end
+
+ruby_block "check-for-sqitch" do
+  block { } # no op
+  only_if { pg_helper.is_running? && !pg_helper.managed_by_sqitch? }
+  notifies :run, "execute[apply-widen-cookbook-version]", :immediately
+end
+
+execute 'apply-widen-cookbook-version' do
+  command "psql -U #{pg_user} -d opscode_chef < #{sql_file}"
+  user pg_user
+  action :nothing
+  notifies :run, "execute[sqitchfy_database]", :immediately
+end
+
+execute 'sqitchfy_database' do
+  command "sqitch --db-user #{pg_user} deploy --log-only --to-target @1.0.0"
+  cwd '/opt/chef-server/embedded/service/chef-server-schema'
+  user pg_user
+  action :nothing
+  notifies :run, "execute[migrate_database]", :immediately
+end
+
+execute "migrate_database" do
+  command "sqitch --db-user #{pg_user} deploy --verify"
+  cwd "/opt/chef-server/embedded/service/chef-server-schema"
+  user pg_user
+  action :nothing
+end

files/chef-server-ctl-commands/upgrade.rb

@@ -0,0 +1,19 @@
+#
+# Copyright:: Copyright (c) 2014 Opscode, Inc.
+#
+# All Rights Reserved
+#
+
+add_command "upgrade", "Upgrade your private chef installation. Add the '--no-op' flag to see what *would* be upgraded", 1 do
+  use_why_run_mode = ARGV.include?("--no-op")
+
+  # Our upgrade process is really just a special chef run
+  command = ["chef-solo",
+             "--config #{base_path}/embedded/cookbooks/solo.rb",
+             "--json-attributes #{base_path}/embedded/cookbooks/pre_upgrade_setup.json",
+             "--log_level fatal"] # yes, that's an underscore in log_level
+  command << "--why-run" if use_why_run_mode
+  status = run_command(command.join(" "))
+  exit! 1 unless status.success?
+  reconfigure(false)
+end