From 5789d743301e501f0871221a8d66ff4daca627fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ladislav=20Slez=C3=A1k?= Date: Tue, 11 Feb 2020 15:43:20 +0100 Subject: [PATCH 1/4] Integrated the yupdate script (bsc#1162826) --- Rakefile | 1 + bin/yupdate | 777 +++++++++++++++++++++++++++++ doc/yupdate.md | 258 ++++++++++ package/yast2-installation.spec | 2 + test/helpers.rb | 17 + test/test_helper.rb | 14 +- test/yupdate/gem_installer_test.rb | 34 ++ test/yupdate/inst_sys_test.rb | 48 ++ 8 files changed, 1150 insertions(+), 1 deletion(-) create mode 100755 bin/yupdate create mode 100644 doc/yupdate.md create mode 100644 test/yupdate/gem_installer_test.rb create mode 100644 test/yupdate/inst_sys_test.rb diff --git a/Rakefile b/Rakefile index 10322cc82..f9b2ec008 100644 --- a/Rakefile +++ b/Rakefile @@ -9,6 +9,7 @@ Yast::Tasks.configuration do |conf| # TODO: move to src/client and verify if needed conf.install_locations["control/*.rb"] = Packaging::Configuration::YAST_DIR + "/clients" conf.install_locations["startup"] = Packaging::Configuration::YAST_LIB_DIR + conf.install_locations["bin/*"] = File.join(Packaging::Configuration::DESTDIR, "/usr/bin/") end # safety check - make sure the RNG file is up to date diff --git a/bin/yupdate b/bin/yupdate new file mode 100755 index 000000000..ef8175eca --- /dev/null +++ b/bin/yupdate @@ -0,0 +1,777 @@ +#!/usr/bin/env ruby + +# This script updates the YaST files in the inst-sys with the files from +# a GitHub repository or a running tarball web server. +# +# See the help text below for the details or see the documentation in +# the doc/yupdate.md file. +# +# Note: The reason why all classes are in a single file is that +# it allows to easy update the script to the latest version +# or add it to an older (not supported) installer with: +# +# curl https://raw.githubusercontent.com/yast/yast-installation/master/bin/yupdate > /usr/bin/yupdate +# chmod +x /usr/bin/yupdate +# yupdate ... +# + +require "fileutils" +require "find" +require "json" +require "net/http" +require "pathname" +require "shellwords" +require "singleton" +require "socket" +require "tmpdir" +require "uri" + +# for logging to y2log +require "yast" + +module YUpdate + # version of the script + class Version + MAJOR = 0 + MINOR = 1 + PATCH = 0 + + STRING = "#{MAJOR}.#{MINOR}.#{PATCH}".freeze + end + + # handle the "help" command line option + class HelpCommand + def run + name = File.basename(__FILE__) + puts <<-HELP +This is a helper script for updating the YaST installer in the installation system. + +Usage: #{name} + +Commands: + + patch Patch the installer with the sources from GitHub, + is a repository name (including the organization + or the user name, if missing "yast" is used by default), + is the Git branch to use + + patch Patch the installer with the sources provided by + a generic HTTP server + + patch Patch the installer with the sources provided by + a "rake server" task (for details see + https://github.com/yast/yast-rake/#server), + the default port is 8000 + + servers List the rake servers running on the remote machine + + overlay create [] Create a new writable overlay for directory , + if no directory is specified it creates overlays + for the default YaST directories + + overlay list Print the created writable overlays + + overlay reset Remove all overlays, restore the system to + the original state + + overlay files Print the changed files (removed files are not listed) + + overlay diff Print the diff of the changed overlay files + + version Print the script version + + help Print this help + +WARNING: This script is intended only for testing and development purposes, + using this tool makes the installation not supported! + + See more details in + https://github.com/yast/yast-installation/blob/master/doc/yupdate.md +HELP + end + end + + # a helper module for printing and logging + module Logger + include Yast::Logger + + # print the message on STDOUT and also write it to the y2log + # @param message [String] the message + def msg(message) + puts message + log.info(message) + end + end + + # a simple /etc/install.inf parser/writer, + # we need to disable the YaST self-update feature to avoid conflicts + class InstallInf + attr_reader :path + + # read the file + def initialize(path = "/etc/install.inf") + @path = path + @values = File.read(path).lines.map(&:chomp) + end + + # get value for the key + def [](key) + line = find_line(key) + return nil unless line + + line.match(/^#{Regexp.escape(key)}:\s*(.*)/)[1] + end + + # set value for the key + def []=(key, val) + line = find_line(key) + + if line + # update the existing key + line.replace("#{key}: #{val}") + else + # add a new key + values << "#{key}: #{val}" + end + end + + # write the file back + def write + File.write(path, values.join("\n")) + end + + private + + attr_reader :values + + def find_line(key) + values.find { |l| l.start_with?(key + ":") } + end + end + + # Class for managing the OverlayFS mounts + # + # Each OverlayFS mount these directories: + # - upper and lower directories - the files in the upper directory + # shadow the files in the lower directory, the result can mounted + # at another 3rd place + # - working directory - for storing additional metadata + # (e.g. removed files) + # + # See more details in + # https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt + # + # The yupdate script uses /var/lib/YaST2/overlayfs for managing the OverlayFS, + # specifically: + # + # - /var/lib/YaST2/overlayfs/original - contains the original state + # - /var/lib/YaST2/overlayfs/upper - contains the new/changed files + # - /var/lib/YaST2/overlayfs/workdir - temporary metadata + # + # To avoid conflicts the original path names are converted to use the + # underscore (_) instead of slash (/) in the name, moreover the underscores + # are double-escaped to support underscores in the original names. E.g. + # /usr/lib/YaST2 (for which the real path is /mounts/mp_0001/usr/lib/YaST2 + # is mounted to /var/lib/YaST2/overlayfs/original/_mounts_mp__0001_usr_lib_YaST2. + class OverlayFS + include YUpdate::Logger + + OVERLAY_PREFIX = "/var/lib/YaST2/overlayfs".freeze + + YAST_OVERLAYS = [ + "/usr/lib/YaST2", + "/usr/lib64/YaST2", + "/usr/share/autoinstall", + "/usr/share/applications/YaST2" + ].freeze + + attr_reader :dir, :orig_dir + + # manage the OverlayFS for this directory + def initialize(directory) + @orig_dir = directory + # expand symlinks + @dir = File.realpath(directory) + raise "Path is not a directory: #{dir}" unless File.directory?(dir) + end + + # create an OverlayFS overlay for this directory if it is not writable + def create + return if File.writable?(dir) + msg "Adding overlay for #{orig_dir}..." + + FileUtils.mkdir_p(upperdir) + FileUtils.mkdir_p(workdir) + FileUtils.mkdir_p(origdir) + + # make the original content available in a separate directory + system("mount --bind #{dir.shellescape} #{origdir.shellescape}") + # mark the mount as a private otherwise the overlay would propagate + # through the bind mount and we would see the changed content here + system("mount --make-private #{origdir.shellescape}") + + system("mount -t overlay overlay -o lowerdir=#{dir.shellescape}," \ + "upperdir=#{upperdir.shellescape},workdir=#{workdir.shellescape} #{dir.shellescape}") + end + + # delete the OverlayFS for this directory, all changes will be reverted back + def delete + system "umount #{dir.shellescape}" + system "umount #{origdir.shellescape}" + FileUtils.rm_rf([upperdir, workdir, origdir]) + end + + # print the modified files in this directory + def print_files + modified_files { |f, _modif, _orig| puts f } + end + + # print the diff for the changed files in this directory + def print_diff + modified_files do |f, _modif, orig| + next unless File.exist?(f) && File.exist?(orig) + system("diff -u #{orig.shellescape} #{f.shellescape}") + end + end + + # find all OverlayFS mounts in the system + def self.find_all + mounts = `mount` + mounts.lines.each_with_object([]) do |line, arr| + arr << new(Regexp.last_match[1]) if line =~ /^overlay on (.*) type overlay / + end + end + + # return the default set of YaST overlays + def self.default_overlays + yast_overlays.map { |o| new(o) } + end + + private + + def dir_name + # escape (double) underscores for correct reverse conversion + dir.gsub("_", "__").tr("/", "_") + end + + def upperdir + File.join(OVERLAY_PREFIX, "upper", dir_name) + end + + def workdir + File.join(OVERLAY_PREFIX, "workdir", dir_name) + end + + def origdir + File.join(OVERLAY_PREFIX, "original", dir_name) + end + + # find the modified files in this directory + def modified_files(&block) + return unless block_given? + + Find.find(upperdir) do |f| + next unless File.file?(f) + upperdir_path = Pathname.new(upperdir) + relative_path = Pathname.new(f).relative_path_from(upperdir_path) + original_path = File.join(origdir, relative_path) + # unescape _ + pth = upperdir_path.basename.to_s.gsub(/([^_])_([^_])/, "\\1/\\2") + .sub(/\A_/, "/").gsub("__", "_").gsub("//", "/") + block.call(File.join(pth, relative_path), f, original_path) + end + end + + def self.yast_overlays + # /usr/share/YaST2/ needs to be handled specially, it is writable + # but contains symlinks to read-only subdirectories, so let's make + # the subdirectories writable + YAST_OVERLAYS + Dir["/usr/share/YaST2/*"].each_with_object([]) do |f, arr| + arr << f if File.directory?(f) && !File.writable?(f) + end + end + + private_class_method :yast_overlays + end + + # a generic HTTP downloader + class Downloader + include YUpdate::Logger + attr_reader :url + + # Create the downloader for the specified URL (only HTTP/HTTPS is supported) + # @param url [String] the URL for downloading + def initialize(url) + @url = url + end + + # download the file, returns the response body or if a block is + # given it passes the response to it, handled HTTP redirection automatically + def download(redirect = 10, &block) + msg "Downloading #{url}" + uri = URI(url) + + while redirect > 0 + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http| + request = Net::HTTP::Get.new uri + http.request(request) do |response| + if response.is_a?(Net::HTTPRedirection) + msg "Redirected to #{response["location"]}" + uri = URI(response["location"]) + redirect -= 1 + elsif response.is_a?(Net::HTTPSuccess) + return block.call(response) if block_given? + # read the rest of the response (the body) + return response.read_body + else + raise "Download failed, error code: #{response.code}" + end + end + end + end + end + end + + # specialized tarball downloader, it can extract the downloaded + # tarball on the fly (without saving the actual tarball) to the target directory + class TarballDownloader < Downloader + def initialize(url) + super + end + + # start the downloading, extract the tarball to the specified directory + def extract_to(dir) + download do |response| + if response["content-type"] !~ /application\/(x-|)gzip/ + raise "Unknown MIME type: #{response["content-type"]}" + end + + # pipe the response body directly to the tar process + IO.popen(["tar", "-C", dir, "--warning=no-timestamp", "-xz"], "wb") do |io| + response.read_body do |chunk| + io.write chunk + end + end + end + end + end + + # specialized tarball downloader which can download the sources + # from GitHub (the "git" tool is missing in the inst-sys), + # instead of "git clone" we download the archive tarball + class GithubDownloader < TarballDownloader + attr_reader :repo, :branch + + def initialize(repo, branch) + super("https://github.com/#{repo}/archive/#{branch}.tar.gz") + @repo = repo + @branch = branch + end + end + + # installing Ruby gems using the "gem" tool + class GemInstaller + # we need this gem for running the "rake install" command + NEEDED_GEM = "yast-rake".freeze + + # install the YaST required gems + def install_required_gems + install_gems(required_gems) + end + + private + + # is the gem installed? + def gem_installed?(gem_name) + gem(gem_name) + true + rescue Gem::LoadError + false + end + + # find the needed gems for running "rake install" + def required_gems + gems = [NEEDED_GEM] + # handle the rake gem specifically, it is present in the system, but + # the /usr/bin/rake file is missing + gems << "rake" if !File.exist?("/usr/bin/rake") + gems + end + + # install the specified gems + def install_gems(gem_names) + gem_names.each do |g| + next if gem_installed?(g) + + add_gem_overlay + system("gem install --no-document --no-format-exec #{gem_names.map(&:shellescape).join(" ")}") + end + end + + # make sure that the gem directory is writable + def add_gem_overlay + overlay = OverlayFS.new(Gem.dir) + overlay.create + end + end + + # install the YaST sources using the "rake install" call + class Installer + include YUpdate::Logger + + attr_reader :src_dir + + # @param src_dir [String] the source directory with unpacked sources + def initialize(src_dir) + @src_dir = src_dir + end + + # install the sources to the inst-sys + def install + Dir.mktmpdir do |tmp| + # first install the files into a temporary location + # using "rake install DESTDIR=..." + install_sources(tmp) + # then find the changed files and update them in the inst-sys + copy_to_system(tmp) + end + end + + private + + # globs for ignored some files + SKIP_FILES = [ + # vim temporary files + "*/.*swp", + # backup files + "*/*.bak", + # skip documentation + "/usr/share/doc/*", + # skip manual pages + "/usr/share/man/*", + # skip sysconfig templates + "/usr/share/fillup-templates/*" + ].freeze + + # install the sources to the specified (temporary) directory + def install_sources(target) + msg "Preparing files..." + + # check for Makefile.cvs, we cannot install packages using autotools + makefile_cvs = Dir["#{src_dir}/**/Makefile.cvs"].first + if makefile_cvs + raise "Found Makefile.cvs, autotools based packages cannot be installed!" + end + + rakefile = Dir["#{src_dir}/**/Rakefile"].first + raise "Rakefile not found, cannot install the package" unless rakefile + + src_dir = File.dirname(rakefile) + Dir.chdir(src_dir) do + `rake install DESTDIR=#{target.shellescape} 2> /dev/null` + end + end + + # should be the file skipped? + def skip_file?(file) + SKIP_FILES.any? { |glob| File.fnmatch?(glob, file) } + end + + # copy the changed files to the ins-sys + def copy_to_system(src) + msg "Copying to system..." + src_path = Pathname.new(src) + cnt = 0 + Find.find(src) do |path| + # TODO: what about symlinks or empty directories? + next unless File.file?(path) + + relative_path = Pathname.new(path).relative_path_from(src_path).to_s + system_file = File.absolute_path(relative_path, "/") + system_dir = File.dirname(system_file) + + next if skip_file?(system_file) + + if File.exist?(system_file) + next if FileUtils.identical?(system_file, path) + add_overlay(system_dir) + # replace a symlink (likely pointing to a read-only location) + FileUtils.rm_f(system_file) if File.symlink?(system_file) + FileUtils.cp(path, system_file) + msg "Updated: #{system_file}" + else + # ensure the directory is writable + if File.exist?(system_dir) + add_overlay(system_dir) + else + # FIXME: maybe an overlay is needed for the upper directory... + FileUtils.mkdir_p(system_dir) + end + + FileUtils.cp(path, system_file) + msg "Added: #{system_file}" + end + cnt += 1 + end + + msg "Number of modified files: #{cnt}" + end + + # ensure that the target directory is writable + def add_overlay(dir) + o = OverlayFS.new(dir) + o.create + end + end + + # handler for the "overlay" command option + class OverlayCommand + def initialize(argv) + @argv = argv + end + + def run + command = @argv.shift + + case command + when "list" + puts OverlayFS.find_all.map(&:dir) + when "create" + dir = @argv.shift + + if dir + ovfs = OverlayFS.new(dir) + ovfs.create + else + OverlayFS.default_overlays.map(&:create) + end + when "reset" + OverlayFS.find_all.map(&:delete) + when "files" + OverlayFS.find_all.map(&:print_files) + when "diff" + OverlayFS.find_all.map(&:print_diff) + else + InvalidCommand.new(command) + end + end + end + + # inst-sys test + class InstSys + # check if the script is running in the inst-sys, + # the script might not work as expected in an installed system + # and using OverlayFS is potentially dangerous + def self.check! + mounts = `mount` + # the inst-sys uses tmpfs (RAM disk) for the root + return if mounts =~ /^tmpfs on \/ type tmpfs/ + + # exit immediately if running in an installed system + $stderr.puts "ERROR: This script can only work in the installation system (inst-sys)!" + exit 1 + end + end + + # parse the command line options + class Options + def self.parse(argv) + command = argv.shift + + case command + when "version" + VersionCommand.new + when "overlay" + InstSys.check! + OverlayCommand.new(argv) + when "patch" + InstSys.check! + PatchCommand.new(argv) + when "servers" + ServersCommand.new(argv) + when "help", "--help", nil + HelpCommand.new + else + InvalidCommand.new(command) + end + end + end + + # handle the "version" command line option + class VersionCommand + def run + puts "#{File.basename(__FILE__)} #{Version::STRING}" + end + end + + # handle the "servers" command line option + class ServersCommand + def initialize(argv) + @argv = argv + end + + def run + host = @argv.shift + + return 1 unless host + + servers = RemoteServer.find(host) + servers.each do |s| + puts "URL: #{s.url}, directory: #{s.dir}" + end + end + end + + # handle the "patch" command line option + class PatchCommand + include YUpdate::Logger + + def initialize(argv) + @argv = argv + end + + def run + arg1 = @argv.shift + arg2 = @argv.shift + + return 1 unless arg1 + + prepare_system + + if arg1 && arg2 + # update from github + install_from_github(arg1, arg2) + elsif arg1.start_with?("http") && arg1.end_with?(".tar.gz") + # upgrade from URL + install_from_tar(arg1) + elsif !arg2 + # otherwise treat it as a hostname providing a tarball server + install_from_servers(arg1) + else + raise "Invalid arguments" + end + + # TODO: only when something has been updated? + disable_self_update + end + + private + + def install_from_github(repo, branch) + # add the default "yast" GitHub organization if missing + repo = "yast/#{repo}" unless repo.include?("/") + downloader = GithubDownloader.new(repo, branch) + install_tar(downloader) + end + + def install_from_tar(url) + downloader = TarballDownloader.new(url) + install_tar(downloader) + end + + def install_from_servers(hostname) + servers = RemoteServer.find(hostname) + + servers.each do |s| + msg "Installing from #{s.url}..." + url = "#{s.url}/archive/current.tar.gz" + install_from_tar(url) + end + end + + def install_sources(src_dir) + i = Installer.new(src_dir) + i.install + end + + # prepare the inst-sys for installation: + # - make the YaST directories writable + # - install the needed Ruby gems (yast-rake) + def prepare_system + OverlayFS.default_overlays.map(&:create) + g = GemInstaller.new + g.install_required_gems + end + + def install_tar(downloader) + Dir.mktmpdir do |download_dir| + downloader.extract_to(download_dir) + install_sources(download_dir) + end + end + + # /etc/install.inf key + SELF_UPDATE_KEY = "SelfUpdate".freeze + + # disable the self update in the install.inf file + def disable_self_update + inf = InstallInf.new + return if inf[SELF_UPDATE_KEY] == "0" + + msg "Disabling the YaST SelfUpdate feature in install.inf!" + inf[SELF_UPDATE_KEY] = "0" + inf.write + end + end + + # handle invalid command line options + class InvalidCommand + def initialize(cmd) + @cmd = cmd + end + + def run + raise "Invalid command: #{cmd}" + end + + private + + attr_reader :cmd + end + + # Query the remote server for the running servers + class RemoteServer + attr_reader :url, :dir + + def initialize(url, dir) + @url = url + @dir = dir + end + + def self.find(host) + host += ":8000" unless host.include?(":") + + url = "http://#{host}/servers/index.json" + u = URI(url) + u.path = "" + + downloader = Downloader.new(url) + JSON.parse(downloader.download).map do |server| + u.port = server["port"] + new(u.to_s, server["dir"]) + end + end + end + + # the main script application + class Application + include YUpdate::Logger + + def run(argv = ARGV) + cmd = Options.parse(argv) + cmd.run + rescue StandardError => e + # the global exception handler + msg("ERROR: #{e.message}") + exit 1 + end + end +end + +# do not execute the script when the file is loaded by some other script +# e.g. by a test, allow testing parts of the code without executing it as a whole +if __FILE__ == $PROGRAM_NAME + # main + app = YUpdate::Application.new + app.run +end diff --git a/doc/yupdate.md b/doc/yupdate.md new file mode 100644 index 000000000..797e767e9 --- /dev/null +++ b/doc/yupdate.md @@ -0,0 +1,258 @@ +# The yupdate Script + +This is a documentation for the `yupdate` helper script, +which is included in the YaST installer in SLE15-SP2/openSUSE +Leap 15.2 (or newer) and in the openSUSE Tumbleweed since +build 2020xxxx. + +## The Introduction + +**Problem**: You are developing a feature for the installer and you need to +test your changes frequently. For extra fun, the change is spread across +multiple repositories. + +The YaST installation system is quite different to an +usual Linux installed system. The root filesystem +is stored in a RAM disk and most files are read-only. +That makes it quite difficult to modify the YaST installer +if you need to debug a problem or test a fix. + +There are some possibilities for updating the YaST installer +(see [Alternative](#Alternative)) +but they are usually not trivial and need special preparations. +For this reason we created a special `yupdate` script which makes +the process easier. + +However, in some cases this easier way cannot be used, see the +[limitations](#limitations) section below. + + +## Self-update + +After patching the installer the `yupdate` script disables +the YaST self-update feature because it could conflict with it +and overwrite the changes. + +If you need some changes from the self-update then use the `startshell=1` +boot option, start the installer and allow the self-update step to finish, +then abort the installation and use the `ypdate` script to apply the +changes on top of the self-update. + +## Warning + +:warning: **Patching the installer with the `yupdate` script makes +the installation unsupported!** :warning: + +The script is intended for developers to test new features or bug fixes. + +It can be used by customers for testing as well, but it should not be used +on production systems! + +## Installation + +The `yupdate` script should run in the inst-sys. Since SLE15-SP2/openSUSE +Leap 15.2, openSUSE Tumbleweed 2020xxxx, it ~~is~~ will be preinstalled. + +For older releases, run: + +```shell +curl https://raw.githubusercontent.com/yast/yast-installation/master/bin/yupdate > /usr/bin/yupdate +chmod +x /usr/bin/yupdate +``` + +You can also use this command to update the included script +to the latest version. + +## Basic Use Cases + +This script is intended to help in the following scenarios. + +### Make the inst-sys Writable + +As already mentioned, the files in the installation system are read only. To be +able to patch the installer the script must be able to make the files writable. +It does that automatically for the updated files, but maybe you would like to +use this feature also for some other non-YaST files. + +To make a directory writable in the inst-sys run command + +```shell +yupdate overlay create +``` + +This will create a writable overlay above the specified directory. If you do not +specify any directory it will create writable overlays for the default YaST +directories. + +Then you can easily edit the files using the included `vim` editor +or by other tools like `sed` or overwrite by external files. + +### Patch YaST from GitHub Sources + +To update or install an YaST package directly from the GitHub source code +repository use command + +```shell +yupdate patch +``` + +where `github_slug` is a `user`/`repository` name, if the `user` value is +missing the default "yast" is used. The `branch` in the source branch to +install, for example `master` or `SLE-15-SP2`. + + +#### Examples + +```shell +# install the latest version of yast2-installation from upstream +yupdate patch yast-installation master +# install from a fork +yupdate patch my_fork/yast-installation my_branch +``` + +#### Notes + +- Make sure that you use a branch compatible with the running inst-sys, + installing the latest version in an older release might not work + as expect, the installer might crash or behave unexpectedly. +- There is no dependency resolution, if the new installed package + requires newer dependant packages then they must be installed manually. + +### Patch YaST from Locally Modified Sources + +Installing from GitHub sources is easy, but sometimes you do not want to +push every single change to GitHub, you would like to just use the current +files from you local Git checkout. + +In that case run + +```shell +rake server +``` + +in your YaST module Git checkout. This will run a web server providing source +tarball similar to the GitHub archive used in the previous case. + +*Note: You need "yast-rake" Ruby gem version 0.2.37 or newer.* + +Then run + +```shell +yupdate patch +``` + +where `` is the machine host name or the IP address where you run +the `rake server` task. To make it easier the rake task prints these values at +the start. + +By default this will use port 8000, if the server uses another port just add +`:` followed by the port number. + +*Note: Make sure the server port is open in the firewall configuration, +see the [documentation](https://github.com/yast/yast-rake/#server) for +more details.* + +#### Patching Multiple Packages + +The `yast patch` command installs the sources from all running `rake server` +servers. If you need to update sources from several packages you can just +run `rake server` in all of them and install them with a single `yupdate` +call. + +### Patch YaST from a Generic Tarball Archive + +This is similar to the previous cases, but the source tarball is not generated +dynamically by a server, but it is a statically hosted file. + +Example: + +```shell +yupdate patch http://myserver.example.com/test/yast2.tar.gz +``` + +## Other Commands + +### Listing OverlayFS Mounts + +To see the list of mounted OverlayFS run + +```shell +yupdate overlay list +``` + +### Listing Updated Files + +To see the list of changed files + +```shell +yupdate overlay files +``` + +### Displaying Changes in the System + +To see the applied changes to the system run + +```shell +yupdate overlay diff +``` + +This will display a diff for all changed files, it does not report +deleted or new files. + +### Restoring the System + +To revert all changes run + +```shell +yupdate overlay reset +``` + +This will remove *all* OverlayFS mounts and restore the system to the original +state. + +## Limitations + +- The script only works with Ruby source files, it cannot compile and + install C/C++ or other sources (the compiler and development libraries + are missing in the inst-sys) +- Works only with the packages which use `Rakefile` for installation, + it does not work with autotools based packages (again, autoconf/automake + are also missing in the inst-sys) + +## Alternative + +1. For all repos, run `rake osc:build` +2. Collect the resulting RPMs +3. Run a server, eg. with `ruby -run -e httpd -- -p 8888 .` +4. Type a loooong boot line to pass them all as DUD=http://....rpm + (or write that into a file and use the [info]( + https://en.opensuse.org/SDB:Linuxrc#p_info) option) + +## Implementation Details + +### OverlayFS + +To make the inst-sys directories writable we use the Linux OverlayFS +which can merge already existing file systems ("union filesystem"). + +See more details in the [Linux Kernel Documentation]( +https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt). + +### Installing the Files + +For installing the sources the script uses the `rake install DESTDIR=...` +command and install the files into a temporary directory. Then it compares +the new files with the original files and if there is a change the old +file is rewritten by the new file. + +This also skips some not needed files like documentation, manual pages, +editor backup files, etc... + +This saves some memory as we do not need to shadow the not modified files +with the same content. + +### Logging + +The messages printed on the console are also saved in the `y2log` file. +That means it should be easy to find out that someone patched the installer +when analyzing logs from a bug report. diff --git a/package/yast2-installation.spec b/package/yast2-installation.spec index d205960ed..f437d9f05 100644 --- a/package/yast2-installation.spec +++ b/package/yast2-installation.spec @@ -193,6 +193,8 @@ systemctl enable YaST2-Firstboot.service # systemd service files %{_unitdir} +# yupdate script +%{_bindir}/ %{yast_clientdir} %{yast_moduledir} %{yast_desktopdir} diff --git a/test/helpers.rb b/test/helpers.rb index e4dc52e6f..0556d10f1 100644 --- a/test/helpers.rb +++ b/test/helpers.rb @@ -20,4 +20,21 @@ def fixtures_dir(*dirs) def load_fixture(*path) File.read(fixtures_dir(*path)) end + + # Execute the passed block and capture both $stdout and $stderr streams. + # @return [Array] A [STDOUT, STDERR] pair + def capture_stdio + stdout_orig = $stdout + stderr_orig = $stderr + + begin + $stdout = StringIO.new + $stderr = StringIO.new + yield + [$stdout.string, $stderr.string] + ensure + $stdout = stdout_orig + $stderr = stderr_orig + end + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 61211631d..c75da3454 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -45,8 +45,9 @@ def stub_module(name) add_filter "/test/" end + bindir = File.expand_path("../../bin", __FILE__) # For coverage we need to load all ruby files - SimpleCov.track_files("#{srcdir}/**/*.rb") + SimpleCov.track_files("{#{srcdir}/**/*.rb,#{bindir}/*}") # use coveralls for on-line code coverage reporting at Travis CI if ENV["TRAVIS"] @@ -63,3 +64,14 @@ def stub_module(name) config.include Yast::I18n # available in it/let/before/... config.include Helpers # custom helpers end + +# require the "bin/yupdate" script for testing it, unfortunately we cannot use +# a simple require/require_relative for it, let's share the workaround in a single place +def require_yupdate + # - "require"/"require_relative" do not work for files without the ".rb" extension + # - adding the "yupdate.rb" -> "yupdate" symlink works but then code coverage + # somehow does not find the executed code and reports zero coverage there + # - "load" works fine but we need to ensure calling it only once + # to avoid the "already initialized constant" Ruby warnings + load File.expand_path("../bin/yupdate", __dir__) unless defined?(YUpdate) +end diff --git a/test/yupdate/gem_installer_test.rb b/test/yupdate/gem_installer_test.rb new file mode 100644 index 000000000..0570f2b07 --- /dev/null +++ b/test/yupdate/gem_installer_test.rb @@ -0,0 +1,34 @@ +#! /usr/bin/env rspec + +require_relative "../test_helper" +require_yupdate + +describe YUpdate::GemInstaller do + describe "#install_required_gems" do + before do + allow(File).to receive(:exist?).with("/usr/bin/rake").and_return(true) + allow_any_instance_of(YUpdate::OverlayFS).to receive :create + allow(subject).to receive(:gem).and_raise(Gem::LoadError) + allow(subject).to receive(:system) + end + + it "installs the required gems" do + allow(subject).to receive(:system).with("gem install --no-document --no-format-exec yast-rake") + subject.install_required_gems + end + + it "skips already installed gems" do + expect(subject).to receive(:gem).and_return(true) + expect(subject).to_not receive(:system) + subject.install_required_gems + end + + it "it makes the gem directory writable" do + ovfs = double + expect(YUpdate::OverlayFS).to receive(:new).with(Gem.dir).and_return(ovfs) + expect(ovfs).to receive(:create) + + subject.install_required_gems + end + end +end diff --git a/test/yupdate/inst_sys_test.rb b/test/yupdate/inst_sys_test.rb new file mode 100644 index 000000000..588722245 --- /dev/null +++ b/test/yupdate/inst_sys_test.rb @@ -0,0 +1,48 @@ +#! /usr/bin/env rspec + +require_relative "../test_helper" +require_yupdate + +describe YUpdate::InstSys do + describe ".check!" do + context "when running in an inst-sys" do + before do + expect(described_class).to receive(:`).with("mount").and_return(<<-MOUNT +tmpfs on / type tmpfs (rw,relatime,size=1508624k,nr_inodes=0) +tmpfs on / type tmpfs (rw,relatime,size=1508624k,nr_inodes=0) +proc on /proc type proc (rw,relatime) +sysfs on /sys type sysfs (rw,relatime) +MOUNT + ) + end + + it "does not exit" do + expect(described_class).to_not receive(:exit) + described_class.check! + end + end + + context "when running in a normal system" do + before do + expect(described_class).to receive(:`).with("mount").and_return(<<-MOUNT +proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) +/dev/sda1 on / type btrfs (rw,relatime,ssd,space_cache,subvolid=267,subvol=/@/.snapshots/1/snapshot) +/dev/sda2 on /home type ext4 (rw,relatime,stripe=32596,data=ordered) +MOUNT + ) + allow(described_class).to receive(:exit).with(1) + end + + it "exits with status 1" do + expect(described_class).to receive(:exit).with(1) + # capture the std streams just to not break the rspec output + capture_stdio { described_class.check! } + end + + it "prints an error on STDERR" do + _stdout, stderr = capture_stdio { described_class.check! } + expect(stderr).to match(/ERROR: .*inst-sys/) + end + end + end +end From 9d1ca22c3edd99a4c1df98599e2545db888eb678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ladislav=20Slez=C3=A1k?= Date: Mon, 17 Feb 2020 17:35:51 +0100 Subject: [PATCH 2/4] Code review fixes --- bin/yupdate | 57 ++++++++++++++++++++-------------- doc/yupdate.md | 6 ++-- test/yupdate/inst_sys_test.rb | 17 +++------- test/yupdate/overlayfs_test.rb | 35 +++++++++++++++++++++ 4 files changed, 76 insertions(+), 39 deletions(-) create mode 100644 test/yupdate/overlayfs_test.rb diff --git a/bin/yupdate b/bin/yupdate index ef8175eca..a127c7c1f 100755 --- a/bin/yupdate +++ b/bin/yupdate @@ -63,7 +63,9 @@ Commands: https://github.com/yast/yast-rake/#server), the default port is 8000 - servers List the rake servers running on the remote machine + servers List the rake servers running on the remote machine, + ask the specified server to report all `rake server` + instances overlay create [] Create a new writable overlay for directory , if no directory is specified it creates overlays @@ -188,6 +190,7 @@ HELP attr_reader :dir, :orig_dir # manage the OverlayFS for this directory + # @param directory [String] the directory def initialize(directory) @orig_dir = directory # expand symlinks @@ -223,12 +226,12 @@ HELP # print the modified files in this directory def print_files - modified_files { |f, _modif, _orig| puts f } + iterate_files { |f, _modif, _orig| puts f } end # print the diff for the changed files in this directory def print_diff - modified_files do |f, _modif, orig| + iterate_files do |f, _modif, orig| next unless File.exist?(f) && File.exist?(orig) system("diff -u #{orig.shellescape} #{f.shellescape}") end @@ -247,27 +250,36 @@ HELP yast_overlays.map { |o| new(o) } end - private - - def dir_name - # escape (double) underscores for correct reverse conversion - dir.gsub("_", "__").tr("/", "_") - end - def upperdir - File.join(OVERLAY_PREFIX, "upper", dir_name) + OverlayFS.escape_path("upper", dir) end def workdir - File.join(OVERLAY_PREFIX, "workdir", dir_name) + OverlayFS.escape_path("workdir", dir) end def origdir - File.join(OVERLAY_PREFIX, "original", dir_name) + OverlayFS.escape_path("original", dir) + end + + def self.escape_path(subdir, path) + # escape (double) underscores for correct reverse conversion + File.join(OVERLAY_PREFIX, subdir, path.gsub("_", "__").tr("/", "_")) + end + + def self.unescape_path(path) + Pathname.new(path).basename.to_s.gsub(/([^_])_([^_])/, "\\1/\\2") + .sub(/\A_/, "/").gsub("__", "_").gsub("//", "/") end - # find the modified files in this directory - def modified_files(&block) + private + + # find the files in this directory + # the block is called with three parameters: + # - the path to the new file in the overlay directory + # - the original path in / + # - the path to the original file in the overlay directory + def iterate_files(&block) return unless block_given? Find.find(upperdir) do |f| @@ -275,9 +287,8 @@ HELP upperdir_path = Pathname.new(upperdir) relative_path = Pathname.new(f).relative_path_from(upperdir_path) original_path = File.join(origdir, relative_path) - # unescape _ - pth = upperdir_path.basename.to_s.gsub(/([^_])_([^_])/, "\\1/\\2") - .sub(/\A_/, "/").gsub("__", "_").gsub("//", "/") + # unescape the path + pth = unescape_path(upperdir_path) block.call(File.join(pth, relative_path), f, original_path) end end @@ -349,7 +360,7 @@ HELP # pipe the response body directly to the tar process IO.popen(["tar", "-C", dir, "--warning=no-timestamp", "-xz"], "wb") do |io| response.read_body do |chunk| - io.write chunk + io.write(chunk) end end end @@ -563,9 +574,8 @@ HELP # the script might not work as expected in an installed system # and using OverlayFS is potentially dangerous def self.check! - mounts = `mount` - # the inst-sys uses tmpfs (RAM disk) for the root - return if mounts =~ /^tmpfs on \/ type tmpfs/ + # the inst-sys contains the /.packages.initrd file with a list of packages + return if File.exist?("/.packages.initrd") # exit immediately if running in an installed system $stderr.puts "ERROR: This script can only work in the installation system (inst-sys)!" @@ -612,8 +622,7 @@ HELP def run host = @argv.shift - - return 1 unless host + raise "Missing server name argument" unless host servers = RemoteServer.find(host) servers.each do |s| diff --git a/doc/yupdate.md b/doc/yupdate.md index 797e767e9..a2e279a0b 100644 --- a/doc/yupdate.md +++ b/doc/yupdate.md @@ -154,7 +154,7 @@ more details.* #### Patching Multiple Packages -The `yast patch` command installs the sources from all running `rake server` +The `yupdate patch` command installs the sources from all running `rake server` servers. If you need to update sources from several packages you can just run `rake server` in all of them and install them with a single `yupdate` call. @@ -226,7 +226,9 @@ state. 3. Run a server, eg. with `ruby -run -e httpd -- -p 8888 .` 4. Type a loooong boot line to pass them all as DUD=http://....rpm (or write that into a file and use the [info]( - https://en.opensuse.org/SDB:Linuxrc#p_info) option) + https://en.opensuse.org/SDB:Linuxrc#p_info) option + or build a single DUD file from the RPMs with the [`mkdud`]( + https://github.com/wfeldt/mkdud) script) ## Implementation Details diff --git a/test/yupdate/inst_sys_test.rb b/test/yupdate/inst_sys_test.rb index 588722245..1375256bd 100644 --- a/test/yupdate/inst_sys_test.rb +++ b/test/yupdate/inst_sys_test.rb @@ -4,16 +4,12 @@ require_yupdate describe YUpdate::InstSys do + let(:file) { "/.packages.initrd" } + describe ".check!" do context "when running in an inst-sys" do before do - expect(described_class).to receive(:`).with("mount").and_return(<<-MOUNT -tmpfs on / type tmpfs (rw,relatime,size=1508624k,nr_inodes=0) -tmpfs on / type tmpfs (rw,relatime,size=1508624k,nr_inodes=0) -proc on /proc type proc (rw,relatime) -sysfs on /sys type sysfs (rw,relatime) -MOUNT - ) + expect(File).to receive(:exist?).with(file).and_return(true) end it "does not exit" do @@ -24,12 +20,7 @@ context "when running in a normal system" do before do - expect(described_class).to receive(:`).with("mount").and_return(<<-MOUNT -proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) -/dev/sda1 on / type btrfs (rw,relatime,ssd,space_cache,subvolid=267,subvol=/@/.snapshots/1/snapshot) -/dev/sda2 on /home type ext4 (rw,relatime,stripe=32596,data=ordered) -MOUNT - ) + expect(File).to receive(:exist?).with(file).and_return(false) allow(described_class).to receive(:exit).with(1) end diff --git a/test/yupdate/overlayfs_test.rb b/test/yupdate/overlayfs_test.rb new file mode 100644 index 000000000..4a8c27642 --- /dev/null +++ b/test/yupdate/overlayfs_test.rb @@ -0,0 +1,35 @@ +#! /usr/bin/env rspec + +require_relative "../test_helper" +require_yupdate + +describe YUpdate::OverlayFS do + # testing data + let(:orig_path) { "/test/test__test" } + let(:escaped_path) { "/var/lib/YaST2/overlayfs/upper/_test_test____test" } + + before do + # mock the checks for existing directory + allow(File).to receive(:realpath) { |d| d } + allow(File).to receive(:directory?).and_return(true) + end + + describe "#upperdir" do + it "returns the path in the 'upper' subdirectory" do + o = YUpdate::OverlayFS.new(orig_path) + expect(o.upperdir).to match(/\/upper\//) + end + end + + describe ".escape_path" do + it "escapes the path and adds a prefix" do + expect(YUpdate::OverlayFS.escape_path("upper", orig_path)).to eq(escaped_path) + end + end + + describe ".unescape_path" do + it "unescapes the path and removes the prefix" do + expect(YUpdate::OverlayFS.unescape_path(escaped_path)).to eq(orig_path) + end + end +end From 4da3e8d4f434cdfc5d777a2080b0edee3bc3f6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ladislav=20Slez=C3=A1k?= Date: Tue, 18 Feb 2020 16:51:57 +0100 Subject: [PATCH 3/4] Changes --- package/yast2-installation.changes | 7 +++++++ package/yast2-installation.spec | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/package/yast2-installation.changes b/package/yast2-installation.changes index 78acd036d..457d7a46e 100644 --- a/package/yast2-installation.changes +++ b/package/yast2-installation.changes @@ -1,3 +1,10 @@ +------------------------------------------------------------------- +Fri Feb 21 09:45:10 UTC 2020 - Ladislav Slezák + +- Added "yupdate" script to simplify patching the installer + (bsc#1163691) +- 4.2.35 + ------------------------------------------------------------------- Thu Feb 20 15:12:39 UTC 2020 - Imobach Gonzalez Sosa diff --git a/package/yast2-installation.spec b/package/yast2-installation.spec index f437d9f05..f5cb1fccc 100644 --- a/package/yast2-installation.spec +++ b/package/yast2-installation.spec @@ -16,7 +16,7 @@ # Name: yast2-installation -Version: 4.2.34 +Version: 4.2.35 Release: 0 Group: System/YaST License: GPL-2.0-only From 541d0f7734eaf590d1699834b60afb30ab63670a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ladislav=20Slez=C3=A1k?= Date: Thu, 20 Feb 2020 14:57:04 +0100 Subject: [PATCH 4/4] Use the Array form of the system() call and "open-uri" --- bin/yupdate | 65 ++++++++++++------------------ test/yupdate/gem_installer_test.rb | 8 +++- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/bin/yupdate b/bin/yupdate index a127c7c1f..758bc9885 100755 --- a/bin/yupdate +++ b/bin/yupdate @@ -19,6 +19,7 @@ require "fileutils" require "find" require "json" require "net/http" +require "open-uri" require "pathname" require "shellwords" require "singleton" @@ -208,19 +209,19 @@ HELP FileUtils.mkdir_p(origdir) # make the original content available in a separate directory - system("mount --bind #{dir.shellescape} #{origdir.shellescape}") + system("mount", "--bind", dir, origdir) # mark the mount as a private otherwise the overlay would propagate # through the bind mount and we would see the changed content here - system("mount --make-private #{origdir.shellescape}") + system("mount", "--make-private", origdir) - system("mount -t overlay overlay -o lowerdir=#{dir.shellescape}," \ - "upperdir=#{upperdir.shellescape},workdir=#{workdir.shellescape} #{dir.shellescape}") + system("mount", "-t", "overlay", "overlay", "-o", "lowerdir=#{dir},"\ + "upperdir=#{upperdir},workdir=#{workdir}", dir) end # delete the OverlayFS for this directory, all changes will be reverted back def delete - system "umount #{dir.shellescape}" - system "umount #{origdir.shellescape}" + system("umount", dir) + system("umount", origdir) FileUtils.rm_rf([upperdir, workdir, origdir]) end @@ -233,7 +234,7 @@ HELP def print_diff iterate_files do |f, _modif, orig| next unless File.exist?(f) && File.exist?(orig) - system("diff -u #{orig.shellescape} #{f.shellescape}") + system("diff", "-u", orig, f) end end @@ -317,28 +318,14 @@ HELP end # download the file, returns the response body or if a block is - # given it passes the response to it, handled HTTP redirection automatically - def download(redirect = 10, &block) + # given it passes the response to it, handles HTTP redirection automatically + def download(&block) msg "Downloading #{url}" - uri = URI(url) - - while redirect > 0 - Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http| - request = Net::HTTP::Get.new uri - http.request(request) do |response| - if response.is_a?(Net::HTTPRedirection) - msg "Redirected to #{response["location"]}" - uri = URI(response["location"]) - redirect -= 1 - elsif response.is_a?(Net::HTTPSuccess) - return block.call(response) if block_given? - # read the rest of the response (the body) - return response.read_body - else - raise "Download failed, error code: #{response.code}" - end - end - end + + if block_given? + URI.open(url) { |f| block.call(f) } + else + URI.open(url, &:read) end end end @@ -352,15 +339,15 @@ HELP # start the downloading, extract the tarball to the specified directory def extract_to(dir) - download do |response| - if response["content-type"] !~ /application\/(x-|)gzip/ - raise "Unknown MIME type: #{response["content-type"]}" + download do |input| + if input.content_type !~ /application\/(x-|)gzip/ + raise "Unknown MIME type: #{input.content_type}" end # pipe the response body directly to the tar process IO.popen(["tar", "-C", dir, "--warning=no-timestamp", "-xz"], "wb") do |io| - response.read_body do |chunk| - io.write(chunk) + while (buffer = input.read(4096)) + io.write(buffer) end end end @@ -402,7 +389,8 @@ HELP # find the needed gems for running "rake install" def required_gems - gems = [NEEDED_GEM] + gems = [] + gems << NEEDED_GEM if !gem_installed?(NEEDED_GEM) # handle the rake gem specifically, it is present in the system, but # the /usr/bin/rake file is missing gems << "rake" if !File.exist?("/usr/bin/rake") @@ -411,12 +399,9 @@ HELP # install the specified gems def install_gems(gem_names) - gem_names.each do |g| - next if gem_installed?(g) - - add_gem_overlay - system("gem install --no-document --no-format-exec #{gem_names.map(&:shellescape).join(" ")}") - end + return if gem_names.empty? + add_gem_overlay + system("gem", "install", "--no-document", "--no-format-exec", *gem_names) end # make sure that the gem directory is writable diff --git a/test/yupdate/gem_installer_test.rb b/test/yupdate/gem_installer_test.rb index 0570f2b07..ab4dd3d4e 100644 --- a/test/yupdate/gem_installer_test.rb +++ b/test/yupdate/gem_installer_test.rb @@ -13,7 +13,13 @@ end it "installs the required gems" do - allow(subject).to receive(:system).with("gem install --no-document --no-format-exec yast-rake") + expect(subject).to receive(:system).with( + "gem", + "install", + "--no-document", + "--no-format-exec", + "yast-rake" + ) subject.install_required_gems end