Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

support for teleport --infer, which creates a sample Telfile

  • Loading branch information...
commit 199d0cc8c97e2f57de645ca3d883626a989dd7f7 1 parent f1fad43
Adam Doppelt authored August 10, 2011
1  lib/teleport.rb
@@ -4,4 +4,5 @@
4 4
 require "teleport/config"
5 5
 require "teleport/mirror"
6 6
 require "teleport/install"
  7
+require "teleport/infer"
7 8
 require "teleport/main"
318  lib/teleport/infer.rb
... ...
@@ -0,0 +1,318 @@
  1
+require "set"
  2
+
  3
+# Many, many thanks to Blueprint!
  4
+# https://github.com/devstructure/blueprint
  5
+
  6
+module Teleport
  7
+  class Infer
  8
+    include Util
  9
+    
  10
+    MD5SUMS = {
  11
+      '/etc/adduser.conf' => ['/usr/share/adduser/adduser.conf'],
  12
+      '/etc/apparmor.d/tunables/home.d/ubuntu' =>
  13
+      ['2a88811f7b763daa96c20b20269294a4'],
  14
+      '/etc/apt/apt.conf.d/00CDMountPoint' =>
  15
+      ['cb46a4e03f8c592ee9f56c948c14ea4e'],
  16
+      '/etc/apt/apt.conf.d/00trustcdrom' =>
  17
+      ['a8df82e6e6774f817b500ee10202a968'],
  18
+      '/etc/chatscripts/provider' => ['/usr/share/ppp/provider.chatscript'],
  19
+      '/etc/default/console-setup' =>
  20
+      ['0fb6cec686d0410993bdf17192bee7d6',
  21
+       'b684fd43b74ac60c6bdafafda8236ed3',
  22
+       '/usr/share/console-setup/console-setup'],
  23
+      '/etc/default/grub' => ['ee9df6805efb2a7d1ba3f8016754a119',
  24
+                              'ad9283019e54cedfc1f58bcc5e615dce'],
  25
+      '/etc/default/irqbalance' => ['7e10d364b9f72b11d7bf7bd1cfaeb0ff'],
  26
+      '/etc/default/keyboard' => ['06d66484edaa2fbf89aa0c1ec4989857'],
  27
+      '/etc/default/locale' => ['164aba1ef1298affaa58761647f2ceba',
  28
+                                '7c32189e775ac93487aa4a01dffbbf76'],
  29
+      '/etc/default/rcS' => ['/usr/share/initscripts/default.rcS'],
  30
+      '/etc/environment' => ['44ad415fac749e0c39d6302a751db3f2'],
  31
+      '/etc/hosts.allow' => ['8c44735847c4f69fb9e1f0d7a32e94c1'],
  32
+      '/etc/hosts.deny' => ['92a0a19db9dc99488f00ac9e7b28eb3d'],
  33
+      '/etc/initramfs-tools/modules' =>
  34
+      ['/usr/share/initramfs-tools/modules'],
  35
+      '/etc/inputrc' => ['/usr/share/readline/inputrc'],
  36
+      '/etc/iscsi/iscsid.conf' => ['6c6fd718faae84a4ab1b276e78fea471'],
  37
+      '/etc/kernel-img.conf' => ['f1ed9c3e91816337aa7351bdf558a442'],
  38
+      '/etc/ld.so.conf' => ['4317c6de8564b68d628c21efa96b37e4'],
  39
+      '/etc/networks' => ['/usr/share/base-files/networks'],
  40
+      '/etc/nsswitch.conf' => ['/usr/share/base-files/nsswitch.conf'],
  41
+      '/etc/pam.d/common-account' => ['9d50c7dda6ba8b6a8422fd4453722324'],
  42
+      '/etc/pam.d/common-auth' => ['a326c972f4f3d20e5f9e1b06eef4d620'],
  43
+      '/etc/pam.d/common-password' => ['9f2fbf01b1a36a017b16ea62c7ff4c22'],
  44
+      '/etc/pam.d/common-session' => ['e2b72dd3efb2d6b29698f944d8723ab1'],
  45
+      '/etc/pam.d/common-session-noninteractive' =>
  46
+      ['508d44b6daafbc3d6bd587e357a6ff5b'],
  47
+      '/etc/ppp/chap-secrets' => ['faac59e116399eadbb37644de6494cc4'],
  48
+      '/etc/ppp/pap-secrets' => ['698c4d412deedc43dde8641f84e8b2fd'],
  49
+      '/etc/ppp/peers/provider' => ['/usr/share/ppp/provider.peer'],
  50
+      '/etc/profile' => ['/usr/share/base-files/profile'],
  51
+      '/etc/python/debian_config' => ['7f4739eb8858d231601a5ed144099ac8'],
  52
+      '/etc/rc.local' => ['10fd9f051accb6fd1f753f2d48371890'],
  53
+      '/etc/rsyslog.d/50-default.conf' =>
  54
+      ['/usr/share/rsyslog/50-default.conf'],
  55
+      '/etc/security/opasswd' => ['d41d8cd98f00b204e9800998ecf8427e'],
  56
+      '/etc/sgml/xml-core.cat' => ['bcd454c9bf55a3816a134f9766f5928f'],
  57
+      '/etc/shells' => ['0e85c87e09d716ecb03624ccff511760'],
  58
+      '/etc/ssh/sshd_config' => ['e24f749808133a27d94fda84a89bb27b',
  59
+                                 '8caefdd9e251b7cc1baa37874149a870'],
  60
+      '/etc/sudoers' => ['02f74ccbec48997f402a063a172abb48'],
  61
+      '/etc/ufw/after.rules' => ['/usr/share/ufw/after.rules'],
  62
+      '/etc/ufw/after6.rules' => ['/usr/share/ufw/after6.rules'],
  63
+      '/etc/ufw/before.rules' => ['/usr/share/ufw/before.rules'],
  64
+      '/etc/ufw/before6.rules' => ['/usr/share/ufw/before6.rules'],
  65
+      '/etc/ufw/ufw.conf' => ['/usr/share/ufw/ufw.conf']
  66
+    }
  67
+
  68
+    NEW_FILES_WITHIN = %w(cron.d logrotate.d rsyslog.d init)
  69
+    CHECKSUM_FILES = %w(bash.bashrc environment inputrc rc.local ssh/ssh_config ssh/sshd_config)
  70
+    
  71
+    def initialize
  72
+      @telfile = []
  73
+
  74
+      if fails?("uname -a | grep -q Ubuntu")
  75
+        fatal "Sorry, --infer can only run on an Ubuntu machine."
  76
+      end
  77
+      
  78
+      append "#" * 72
  79
+      append "# Telfile inferred from #{`hostname`.strip} at #{Time.now}"
  80
+      append "#" * 72
  81
+      append
  82
+      
  83
+      user
  84
+      ruby
  85
+      apt
  86
+      packages
  87
+      files
  88
+
  89
+      banner "Done!"
  90
+      $stderr.puts
  91
+      @telfile.each { |i| puts i }
  92
+    end
  93
+
  94
+    def append(s = nil)
  95
+      @telfile << (s || "")
  96
+    end
  97
+
  98
+    def user
  99
+      append "user #{`whoami`.strip.inspect}"
  100
+    end
  101
+
  102
+    def ruby
  103
+      version = `ruby --version`
  104
+      ruby = nil
  105
+      case version
  106
+      when /Ruby Enterprise Edition/ then ruby = "REE"
  107
+      when /1\.8\.7/ then ruby = "1.8.7"
  108
+      when /1\.9\.2/ then ruby = "1.9.2"
  109
+      end
  110
+      append "ruby #{ruby.inspect}" if ruby
  111
+    end
  112
+
  113
+    def apt
  114
+      banner "Calculating apt sources and keys..."
  115
+      list = run_capture_lines("cat /etc/apt/sources.list /etc/apt/sources.list.d/*.list")
  116
+      list = list.grep(/^deb /).sort
  117
+      list.each do |line|
  118
+        if line =~ /^deb http:\/\/(\S+)\s+(\S+)/
  119
+          source, dist = $1, $2
  120
+          file = source.chomp("/").gsub(/[^a-z0-9.-]/, "_")
  121
+          file = "/var/lib/apt/lists/#{file}_dists_#{dist}_Release"
  122
+          next if !File.exists?(file)
  123
+
  124
+          verify = run_capture("gpgv --keyring /etc/apt/trusted.gpg #{file}.gpg #{file} 2>&1")
  125
+          key = verify[/key ID ([A-Z0-9]{8})$/, 1]
  126
+          next if key == "437D05B5" # canonical key
  127
+          append "apt #{line.inspect}, :key => #{key.inspect}"
  128
+        end
  129
+      end
  130
+    end
  131
+
  132
+    def packages
  133
+      banner "Looking for interesting packages..."      
  134
+      @packages = Apt.new.added
  135
+      if !@packages.empty?
  136
+        append
  137
+        append "# Note: You should read this package list very carefully and remove"
  138
+        append "# packages that you don't want on your server."
  139
+        append
  140
+        append "packages %w(#{@packages.join(" ")})"
  141
+      end
  142
+    end
  143
+
  144
+    def files
  145
+      banner "Looking for interesting files..."            
  146
+      files = []
  147
+
  148
+      # read checksums from dpkg status
  149
+      conf = { }
  150
+      File.readlines("/var/lib/dpkg/status").each do |line|
  151
+        if line =~ /^ (\S+) ([0-9a-f]{32})/
  152
+          conf[$1] = $2
  153
+        end
  154
+      end
  155
+
  156
+      # look for changed conf files
  157
+      $stderr.puts "  scanning conf files from interesting packages..."
  158
+      @packages.each do |pkg|
  159
+        list = run_capture_lines("dpkg -L #{pkg}")
  160
+        list = list.select { |i| i =~ /^\/etc/ }.sort
  161
+        list = list.select { |i| File.file?(i) }
  162
+        list = list.select { |i| conf[i] && conf[i] != md5sum(i) }
  163
+        files += list
  164
+      end
  165
+
  166
+      # look for new files in NEW_FILES_WITHIN
  167
+      dirs = NEW_FILES_WITHIN.map { |i| "/etc/#{i}" }
  168
+      dirs.sort.each do |dir|
  169
+        $stderr.puts "  scanning #{dir} for new files..."
  170
+        list = Dir["#{dir}/*"].sort
  171
+        list = list.select { |i| !MD5SUMS[i] }
  172
+        list = list.select { |i| fails?("dpkg -S #{i}") }
  173
+        files += list
  174
+      end
  175
+      
  176
+      # now look for changed files from CHECKSUM_FILES
  177
+      scan = CHECKSUM_FILES.map { |i| "/etc/#{i}" }
  178
+      scan = scan.select { |i| File.file?(i) }
  179
+      scan.each do |i|
  180
+        new_sum = md5sum(i)
  181
+        if old_sum = MD5SUMS[i]
  182
+          match = old_sum.any? do |sum|
  183
+            sum = md5sum(sum) if sum =~ /^\//
  184
+            new_sum == sum
  185
+          end
  186
+          files << i if !match
  187
+        elsif old_sum = conf[i]
  188
+          files << i if new_sum != old_sum
  189
+        end
  190
+      end
  191
+
  192
+      if !files.empty?
  193
+        append
  194
+        append "#" * 72
  195
+        append "# Also, I think these should be included in files/"
  196
+        append "#" * 72
  197
+        append
  198
+        files.sort.each do |i|
  199
+          append "# #{i}"
  200
+        end
  201
+        append
  202
+        append "# You can do that with this magical command:"
  203
+        append "#"
  204
+        append "# mkdir files && cd files && tar cf - #{files.join(" ")} | tar xf -"
  205
+      end
  206
+    end
  207
+
  208
+    class Apt
  209
+      include Util
  210
+
  211
+      BLACKLIST = /^linux-(generic|headers|image)/
  212
+      
  213
+      Package = Struct.new(:name, :status, :deps, :base, :parents)
  214
+
  215
+      def initialize
  216
+        @packages = nil
  217
+        @map = nil
  218
+      end
  219
+
  220
+      def packages
  221
+        if !@packages
  222
+          # run dpkg
  223
+          lines = run_capture_lines("dpkg-query '-f=${Package}\t${Status}\t${Pre-Depends},${Depends},${Recommends}\t${Essential}\t${Priority}\n' -W")
  224
+          @packages = lines.map do |line|
  225
+            name, status, deps, essential, priority = line.split("\t")
  226
+            deps = deps.gsub(/\([^)]+\)/, "")
  227
+            deps = deps.split(/[,|]/)
  228
+            deps = deps.map(&:strip).select { |i| !i.empty? }.sort
  229
+            base = false
  230
+            base = true if essential == "yes"
  231
+            base = true if priority =~ /^(important|required|standard)$/
  232
+            Package.new(name, status, deps, base, [])
  233
+          end
  234
+
  235
+          # calculate ancestors
  236
+          @packages.each do |pkg|
  237
+            pkg.deps.each do |i|
  238
+              if d = self[i]
  239
+                d.parents << pkg.name
  240
+              end
  241
+            end
  242
+          end
  243
+          @packages.each do |pkg|
  244
+            pkg.parents = pkg.parents.sort.uniq
  245
+          end
  246
+        end
  247
+        
  248
+        @packages
  249
+      end
  250
+
  251
+      def [](name)
  252
+        if !@map
  253
+          @map = { }
  254
+          packages.each { |i| @map[i.name] = i }
  255
+        end
  256
+        @map[name]
  257
+      end
  258
+
  259
+      def base_packages
  260
+        packages.select { |i| i.base }.map(&:name)    
  261
+      end
  262
+
  263
+      def ignored_packages
  264
+        list = packages.select { |i| i.base }.map(&:name)
  265
+        list += %w(grub-pc installation-report language-pack-en language-pack-gnome-en linux-generic-pae linux-server os-prober ubuntu-desktop ubuntu-minimal ubuntu-standard wireless-crda)
  266
+        dependencies(list)
  267
+      end
  268
+
  269
+      def dependencies(list)
  270
+        check = list
  271
+        while !check.empty?
  272
+          check = check.map do |i|
  273
+            if pkg = self[i]
  274
+              pkg.deps
  275
+            end
  276
+          end
  277
+          check = check.compact.flatten.uniq.sort
  278
+          check -= list
  279
+          list += check
  280
+        end
  281
+        list.sort
  282
+      end
  283
+
  284
+      def added
  285
+        # calculate raw list
  286
+        ignored = Set.new(ignored_packages)
  287
+        list = packages.select do |i|
  288
+          i.status == "install ok installed" && !ignored.include?(i.name)
  289
+        end
  290
+        list = list.map(&:name)
  291
+
  292
+        # now calculate parents
  293
+        roots = []
  294
+        check = list
  295
+        while !check.empty?
  296
+          check = check.map do |i|
  297
+            if pkg = self[i]
  298
+              if !pkg.parents.empty?
  299
+                pkg.parents
  300
+              else
  301
+                roots << pkg.name
  302
+                nil
  303
+              end
  304
+            end
  305
+          end
  306
+          check = check.compact.flatten.uniq.sort
  307
+          check -= list
  308
+          list += check
  309
+        end
  310
+
  311
+        # blacklist
  312
+        roots = roots.reject { |i| i =~ BLACKLIST }
  313
+        
  314
+        roots.sort
  315
+      end
  316
+    end
  317
+  end
  318
+end
25  lib/teleport/main.rb
@@ -9,14 +9,17 @@ class Main
9 9
     TAR = "#{DIR}.tgz"
10 10
     
11 11
     def initialize(cmd = :teleport)
12  
-      $stderr = $stdout
13 12
       cli(cmd)
14 13
       
15 14
       case @options[:cmd]
16 15
       when :teleport
  16
+        $stderr = $stdout
17 17
         teleport
18 18
       when :install
  19
+        $stderr = $stdout
19 20
         install
  21
+      when :infer
  22
+        infer
20 23
       end
21 24
     end
22 25
     
@@ -31,6 +34,9 @@ def cli(cmd)
31 34
         o.on("-f", "--file FILE", "use this file instead of Telfile") do |f|
32 35
           @options[:file] = f
33 36
         end
  37
+        o.on("-i", "--infer", "infer a new Telfile from YOUR machine") do |f|
  38
+          @options[:cmd] = :infer
  39
+        end
34 40
         o.on_tail("-h", "--help", "print this help text") do
35 41
           puts opt
36 42
           exit(0)
@@ -47,23 +53,19 @@ def cli(cmd)
47 53
       if @options[:cmd] == :teleport
48 54
         # print this error message early, to give the user a hint
49 55
         # instead of complaining about command line arguments
50  
-        look_for_config!
51  
-        if !(@options[:host] = ARGV.shift)
  56
+        if ARGV.length != 1
52 57
           puts opt
53 58
           exit(1)
54 59
         end
  60
+        @options[:host] = ARGV.shift
55 61
       end
56 62
     end
57 63
 
58  
-    def look_for_config!
  64
+    # Read Telfile
  65
+    def read_config
59 66
       if !File.exists?(@options[:file])
60 67
         fatal("Sadly, I can't find #{@options[:file]} here. Please create one.")
61 68
       end
62  
-    end
63  
-
64  
-    # Read Telfile
65  
-    def read_config
66  
-      look_for_config!
67 69
       @config = Config.new(@options[:file])
68 70
     end
69 71
 
@@ -142,5 +144,10 @@ def install
142 144
       end
143 145
       Install.new(@config)
144 146
     end
  147
+
  148
+    # try to infer a new Telfile based on the current machine
  149
+    def infer
  150
+      Infer.new
  151
+    end
145 152
   end
146 153
 end
8  lib/teleport/mirror.rb
@@ -31,8 +31,12 @@ def install_file(path)
31 31
         copy_metadata(path, tmp)
32 32
         path = tmp
33 33
       end
34  
-      
35  
-      cp_if_necessary(path, dst, user_for_file(dst), mode_for_file(dst))
  34
+
  35
+      if !File.symlink?(path)
  36
+        cp_if_necessary(path, dst, user_for_file(dst), mode_for_file(dst))
  37
+      else
  38
+        ln_if_necessary(File.readlink(path), dst)
  39
+      end
36 40
     end
37 41
 
38 42
     # Install directory from the teleport data directory into the
23  lib/teleport/util.rb
... ...
@@ -1,4 +1,5 @@
1 1
 require "cgi"
  2
+require "digest/md5"
2 3
 require "etc"
3 4
 require "fileutils"
4 5
 
@@ -56,7 +57,7 @@ def run(command, args = nil)
56 57
     end
57 58
 
58 59
     # Run a command, raise an error upon failure. The output is
59  
-    # capture as a string and returned.
  60
+    # captured as a string and returned.
60 61
     def run_capture(command, *args)
61 62
       if !args.empty?
62 63
         args = args.flatten.map { |i| shell_escape(i) }.join(" ")
@@ -72,6 +73,13 @@ def run_capture(command, *args)
72 73
       result
73 74
     end
74 75
 
  76
+    # Run a command and split the result into lines, raise an error
  77
+    # upon failure. The output is captured as an array of strings and
  78
+    # returned.
  79
+    def run_capture_lines(command, *args)
  80
+      run_capture(command, args).split("\n")
  81
+    end
  82
+
75 83
     # Run a command but don't send any output to $stdout/$stderr.
76 84
     def run_quietly(command, *args)
77 85
       if !args.empty?
@@ -304,7 +312,18 @@ def process_by_pid?(pidfile)
304 312
       end
305 313
       false
306 314
     end
307  
-    
  315
+
  316
+    # Calculate the md5 checksum for a file
  317
+    def md5sum(path)
  318
+      digest, buf = Digest::MD5.new, ""
  319
+      File.open(path) do |f|
  320
+        while f.read(4096, buf)
  321
+          digest.update(buf)
  322
+        end
  323
+      end
  324
+      digest.hexdigest
  325
+    end
  326
+
308 327
     private
309 328
 
310 329
     # Returns true if verbosity is turned on.

0 notes on commit 199d0cc

Please sign in to comment.
Something went wrong with that request. Please try again.