Merge branch 'jan_2013_external_sources'

Conflicts:
	.gitignore
	Rakefile

.gitignore

@@ -12,4 +12,5 @@ log
 pdf_output
 pdf_source
 puppetdocs-latest.tar.gz*
-.bundle
+.bundle
+externalsources

Rakefile

@@ -19,18 +19,86 @@ end
 $LOAD_PATH.unshift File.expand_path('lib')
 
 references = %w(configuration function indirection metaparameter report type developer)
+top_dir = Dir.pwd
+
+namespace :externalsources do
+
+  # For now, we're using things in the _config.yml, just... because it's there I guess.
+  def load_externalsources
+    require 'yaml'
+    all_config = YAML.load(File.open("source/_config.yml"))
+    return all_config['externalsources']
+  end
+
+  def repo_name(repo_url)
+    repo_url.split('/')[-1].sub(/\.git$/, '')
+  end
+
+  # "Update all working copies defined in source/_config.yml"
+  task :update do
+    Rake::Task['externalsources:clone'].invoke
+    externalsources = load_externalsources
+    Dir.chdir("externalsources") do
+      externalsources.each do |name, info|
+        unless File.directory?(name)
+          puts "Making new working directory for #{name}"
+          system ("#{top_dir}/vendor/bin/git-new-workdir #{repo_name(info['repo'])} #{name} #{info['commit']}")
+        end
+        Dir.chdir(name) do
+          puts "Updating #{name}"
+          system ("git fetch origin && git checkout --force #{info['commit']} && git clean --force .")
+        end
+      end
+    end
+  end
+
+  # "Clone any external documentation repos (from externalsources in source/_config.yml) that don't yet exist"
+  task :clone do
+    externalsources = load_externalsources
+    repos = []
+    externalsources.each do |name, info|
+      repos << info['repo']
+    end
+    Dir.chdir("externalsources") do
+      repos.uniq.each do |repo|
+        system ("git clone #{repo}") unless File.directory?("#{repo_name(repo)}")
+      end
+    end
+  end
+
+  # "Symlink external documentation into place in the source directory"
+  task :link do
+    Rake::Task['externalsources:clean'].invoke # Bad things happen if any of these symlinks already exist, and Jekyll will run FOREVER
+    Rake::Task['externalsources:clean'].reenable
+    externalsources = load_externalsources
+    externalsources.each do |name, info|
+      # Have to use absolute paths for the source, since we have no idea how deep in the hierarchy info['url'] is (and thus how many ../..s it would need).
+      FileUtils.ln_sf "#{top_dir}/externalsources/#{name}/#{info['subdirectory']}", "source#{info['url']}"
+    end
+  end
+
+  # "Clean up any external source symlinks from the source directory" # In the current implementation, all external sources are symlinks and there are no other symlinks in the source. This means we can naively kill all symlinks in ./source. 
+  task :clean do
+    system("find ./source -type l -print0 | xargs -0 rm")
+  end
+end
 
 desc "Generate the documentation"
 task :generate do
+  Rake::Task['externalsources:update'].invoke # Create external sources if necessary, and check out the required working directories
+  Rake::Task['externalsources:link'].invoke # Link docs folders from external sources into the source at the appropriate places. 
+
   system("mkdir -p output")
   system("rm -rf output/*")
   system("mkdir output/references")
-  Dir.chdir("source")
-  system("bundle exec jekyll  ../output")
+  Dir.chdir("source") do
+    system("bundle exec jekyll  ../output")
+  end
+
   Rake::Task['references:symlink'].invoke
-  Dir.chdir("..")
-  puts Dir.pwd
-
+
+  Rake::Task['externalsources:clean'].invoke # The opposite of externalsources:link. Delete all symlinks in the source.
+  Rake::Task['externalsources:clean'].reenable
 end
 
 
@@ -50,55 +118,56 @@ task :generate_pdf do
   system("cp -rf source pdf_source")
   system("cp -rf pdf_mask/* pdf_source") # Copy in and/or overwrite differing files
   # The point being, this way we don't have to maintain separate copies of the actual source files, and it's clear which things are actually different for the PDF version of the page.
-  Dir.chdir("pdf_source")
-  system("bundle exec jekyll ../pdf_output")
+  Dir.chdir("pdf_source") do
+    system("bundle exec jekyll ../pdf_output")
+  end
   Rake::Task['references:symlink:for_pdf'].invoke
-  Dir.chdir("../pdf_output")
-  pdf_targets = YAML.load(File.open("../pdf_mask/pdf_targets.yaml"))
-  pdf_targets.each do |target, pages|
-    system("cat #{pages.join(' ')} > #{target}")
-    if target == 'puppetdb1.html'
-      content = File.read('puppetdb1.html')
-      content.gsub!('-puppetdb-1-install_from_source-html-step-3-option-b-manually-create-a-keystore-and-truststore', '-puppetdb-1-install_from_source-html-step-3-option-b-manuallu-create-a-keystore-and-truststore')
-      File.open('puppetdb1.html', "w") {|pdd1| pdd1.print(content)}
-      # Yeah, so, I found the magic string that, when used as an element ID and then
-      # linked to from elsewhere in the document, causes wkhtmltopdf to think an
-      # unthinkable thought and corrupt the output file.
-      # Your guess is as good as mine. #doomed #sorcery #wat
-      # >:|
-      # -NF
+  Dir.chdir("../pdf_output") do
+    pdf_targets = YAML.load(File.open("../pdf_mask/pdf_targets.yaml"))
+    pdf_targets.each do |target, pages|
+      system("cat #{pages.join(' ')} > #{target}")
+      if target == 'puppetdb1.html'
+        content = File.read('puppetdb1.html')
+        content.gsub!('-puppetdb-1-install_from_source-html-step-3-option-b-manually-create-a-keystore-and-truststore', '-puppetdb-1-install_from_source-html-step-3-option-b-manuallu-create-a-keystore-and-truststore')
+        File.open('puppetdb1.html', "w") {|pdd1| pdd1.print(content)}
+        # Yeah, so, I found the magic string that, when used as an element ID and then
+        # linked to from elsewhere in the document, causes wkhtmltopdf to think an
+        # unthinkable thought and corrupt the output file.
+        # Your guess is as good as mine. #doomed #sorcery #wat
+        # >:|
+        # -NF
+      end
     end
   end
 #   system("cat `cat ../pdf_source/page_order.txt` > rebuilt_index.html")
 #   system("mv index.html original_index.html")
 #   system("mv rebuilt_index.html index.html")
   puts "Remember to run rake serve_pdf"
   puts "Remember to run rake compile_pdf (while serving on localhost:9292)"
-  Dir.chdir("..")
 end
 
 desc "Temporary task for debugging PDF compile failures"
 task :reshuffle_pdf do
   require 'yaml'
-  Dir.chdir("pdf_output")
-  pdf_targets = YAML.load(File.open("../pdf_mask/pdf_targets.yaml"))
-  pdf_targets.each do |target, pages|
-    system("cat #{pages.join(' ')} > #{target}")
-    if target == 'puppetdb1.html'
-      content = File.read('puppetdb1.html')
-      content.gsub!('-puppetdb-1-install_from_source-html-step-3-option-b-manually-create-a-keystore-and-truststore', '-puppetdb-1-install_from_source-html-step-3-option-b-manuallu-create-a-keystore-and-truststore')
-      File.open('puppetdb1.html', "w") {|pdd1| pdd1.print(content)}
-      # Yeah, so, I found the magic string that, when used as an element ID and then
-      # linked to from elsewhere in the document, causes wkhtmltopdf to think an
-      # unthinkable thought and corrupt the output file.
-      # Your guess is as good as mine. #doomed #sorcery #wat
-      # >:|
-      # -NF
+  Dir.chdir("pdf_output") do
+    pdf_targets = YAML.load(File.open("../pdf_mask/pdf_targets.yaml"))
+    pdf_targets.each do |target, pages|
+      system("cat #{pages.join(' ')} > #{target}")
+      if target == 'puppetdb1.html'
+        content = File.read('puppetdb1.html')
+        content.gsub!('-puppetdb-1-install_from_source-html-step-3-option-b-manually-create-a-keystore-and-truststore', '-puppetdb-1-install_from_source-html-step-3-option-b-manuallu-create-a-keystore-and-truststore')
+        File.open('puppetdb1.html', "w") {|pdd1| pdd1.print(content)}
+        # Yeah, so, I found the magic string that, when used as an element ID and then
+        # linked to from elsewhere in the document, causes wkhtmltopdf to think an
+        # unthinkable thought and corrupt the output file.
+        # Your guess is as good as mine. #doomed #sorcery #wat
+        # >:|
+        # -NF
+      end
     end
   end
   puts "Remember to run rake serve_pdf"
   puts "Remember to run rake compile_pdf (while serving on localhost:9292)"
-  Dir.chdir("..")
 end
 
 
@@ -121,10 +190,10 @@ end
 desc "Create tarball of documentation"
 task :tarball do
   tarball_name = "puppetdocs-latest.tar.gz"
-  FileUtils.cd 'output'
-  sh "tar -czf #{tarball_name} *"
-  FileUtils.mv tarball_name, '..'
-  FileUtils.cd '..'
+  FileUtils.cd('output') do
+    sh "tar -czf #{tarball_name} *"
+    FileUtils.mv tarball_name, '..'
+  end
   sh "git rev-parse HEAD > #{tarball_name}.version" if File.directory?('.git') # Record the version of this tarball, but only if we're in a git repo.
 end
 
@@ -141,19 +210,19 @@ namespace :references do
 
   namespace :symlink do
 
-    desc "Show the versions that will be symlinked"
+    # "Show the versions that will be symlinked"
     task :versions do
       require 'puppet_docs'
       PuppetDocs::Reference.special_versions.each do |name, (version, source)|
         puts "#{name}: #{version}"
       end
     end
 
-    desc "Symlink the latest & stable directories when generating a flat page for PDFing"
+    # "Symlink the latest & stable directories when generating a flat page for PDFing"
     task :for_pdf do
       require 'puppet_docs'
       PuppetDocs::Reference.special_versions.each do |name, (version, source)|
-        Dir.chdir '../pdf_output/references' do
+        Dir.chdir 'pdf_output/references' do
           FileUtils.ln_sf version.to_s, name.to_s
         end
       end
@@ -162,11 +231,11 @@ namespace :references do
 
   end
 
-  desc "Symlink the latest & stable directories"
+  # "Symlink the latest & stable directories"
   task :symlink do
     require 'puppet_docs'
     PuppetDocs::Reference.special_versions.each do |name, (version, source)|
-      Dir.chdir '../output/references' do
+      Dir.chdir 'output/references' do
         FileUtils.ln_sf version.to_s, name.to_s
       end
     end
@@ -175,7 +244,7 @@ namespace :references do
   namespace :puppetdoc do
 
     references.each do |name|
-      desc "Write references/VERSION/#{name}"
+      # "Write references/VERSION/#{name}"
       task name => 'references:check_version' do
         require 'puppet_docs'
         PuppetDocs::Reference::Generator.new(ENV['VERSION'], name).generate
@@ -189,7 +258,7 @@ namespace :references do
 
   namespace :index do
 
-    desc "Generate a stub index for VERSION"
+    # "Generate a stub index for VERSION"
     task :stub => 'references:check_version' do
       filename = Pathname.new('source/references') + ENV['VERSION'] + 'index.markdown'
       filename.parent.mkpath
@@ -218,9 +287,9 @@ namespace :references do
   end
 
   task :fetch_tags do
-    Dir.chdir("vendor/puppet")
-    sh "git fetch --tags"
-    Dir.chdir("../..")
+    Dir.chdir("vendor/puppet") do
+      sh "git fetch --tags"
+    end
   end
 
   desc "Update the contents of source/man/{app}.markdown" # Note that the index must be built manually if new applications are added. Also, let's not ever have a `puppet index` command.
@@ -257,10 +326,10 @@ end
 
 task :default => :spec
 
-require 'rdoc/task'
-Rake::RDocTask.new do |rdoc|
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "puppet-docs"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+# require 'rdoc/task'
+# Rake::RDocTask.new do |rdoc|
+#   rdoc.rdoc_dir = 'rdoc'
+#   rdoc.title = "puppet-docs"
+#   rdoc.rdoc_files.include('README*')
+#   rdoc.rdoc_files.include('lib/**/*.rb')
+# end

source/_config.yml

@@ -18,4 +18,21 @@ defaultnav:
   /hiera/1: hiera1.html
 destination: ../output
 url: "http://docs.puppetlabs.com"
+externalsources:
+  puppetdb_1.1:
+    url: /puppetdb/1.1
+    repo: git://github.com/puppetlabs/puppetdb.git
+    # Change this to origin/1.1.x once puppetdb team cuts a branch
+    commit: origin/master
+    subdirectory: documentation
+  puppetdb_master:
+    url: /puppetdb/master
+    repo: git://github.com/puppetlabs/puppetdb.git
+    commit: origin/master
+    subdirectory: documentation
+  marionette-collective:
+    url: /mcollective
+    repo: git://github.com/puppetlabs/marionette-collective.git
+    commit: origin/master
+    subdirectory: website
 ---