diff --git a/addons/host-upgrades/addon.rb b/addons/host-upgrades/addon.rb index c8f757751..6ece5647b 100644 --- a/addons/host-upgrades/addon.rb +++ b/addons/host-upgrades/addon.rb @@ -1,20 +1,46 @@ # frozen_string_literal: true Pharos.addon 'host-upgrades' do - version '0.1.0' + version '0.2.0' license 'Apache License 2.0' config { - attribute :interval, Pharos::Types::String + attribute :schedule, Pharos::Types::String + attribute :schedule_window, Pharos::Types::String + attribute :reboot, Pharos::Types::Bool.default(false) + attribute :drain, Pharos::Types::Bool.default(true) } config_schema { - required(:interval).filled(:str?, :duration?) + required(:schedule).filled(:str?, :cron?) + optional(:schedule_window).filled(:str?, :duration?) + optional(:reboot).filled(:bool?) + optional(:drain).filled(:bool?) } + # @return [String] + def schedule + cron = Fugit::Cron.parse(config.schedule) + + cron.to_cron_s + end + + # @return [String] + def schedule_window + return '0' if !config.schedule_window + + s = Fugit::Duration.parse(config.schedule_window).to_sec + + "#{s}s" + end + install { apply_resources( - interval: duration.parse(config.interval).to_sec + schedule: schedule, + schedule_window: schedule_window, # only supports h, m, s; not D, M, Y + reboot: config.reboot, + drain: config.reboot && config.drain, + journal: false, # disabled due to https://github.com/kontena/pharos-host-upgrades/issues/15 ) } end diff --git a/addons/host-upgrades/resources/10-clusterrole.yml b/addons/host-upgrades/resources/10-clusterrole.yml new file mode 100644 index 000000000..e9b300b47 --- /dev/null +++ b/addons/host-upgrades/resources/10-clusterrole.yml @@ -0,0 +1,62 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: host-upgrades +rules: +# locking +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - watch +- apiGroups: + - apps + resources: + - daemonsets + resourceNames: + - host-upgrades + verbs: + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - update + +# drain +- apiGroups: + - "" + resources: + - nodes + verbs: + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - delete +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create diff --git a/addons/host-upgrades/resources/20-serviceaccount.yml b/addons/host-upgrades/resources/20-serviceaccount.yml new file mode 100644 index 000000000..d40fa1b22 --- /dev/null +++ b/addons/host-upgrades/resources/20-serviceaccount.yml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: host-upgrades diff --git a/addons/host-upgrades/resources/30-clusterrolebinding.yml b/addons/host-upgrades/resources/30-clusterrolebinding.yml new file mode 100644 index 000000000..29cbc2ea0 --- /dev/null +++ b/addons/host-upgrades/resources/30-clusterrolebinding.yml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: host-upgrades +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: host-upgrades +subjects: +- kind: ServiceAccount + namespace: kube-system + name: host-upgrades diff --git a/addons/host-upgrades/resources/40-configmap.yml b/addons/host-upgrades/resources/40-configmap.yml new file mode 100644 index 000000000..28cb2da90 --- /dev/null +++ b/addons/host-upgrades/resources/40-configmap.yml @@ -0,0 +1,159 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + namespace: kube-system + name: host-upgrades + labels: + app: host-upgrades +data: + yum-cron.conf: | + [commands] + # What kind of update to use: + # default = yum upgrade + # security = yum --security upgrade + # security-severity:Critical = yum --sec-severity=Critical upgrade + # minimal = yum --bugfix update-minimal + # minimal-security = yum --security update-minimal + # minimal-security-severity:Critical = --sec-severity=Critical update-minimal + update_cmd = default + + # Whether a message should be emitted when updates are available, + # were downloaded, or applied. + update_messages = yes + + # Whether updates should be downloaded when they are available. + download_updates = yes + + # Whether updates should be applied when they are available. Note + # that download_updates must also be yes for the update to be applied. + apply_updates = yes + + # Maximum amout of time to randomly sleep, in minutes. The program + # will sleep for a random amount of time between 0 and random_sleep + # minutes before running. This is useful for e.g. staggering the + # times that multiple systems will access update servers. If + # random_sleep is 0 or negative, the program will run immediately. + # 6*60 = 360 + random_sleep = 0 + + [emitters] + # Name to use for this system in messages that are emitted. If + # system_name is None, the hostname will be used. + system_name = None + + # How to send messages. Valid options are stdio and email. If + # emit_via includes stdio, messages will be sent to stdout; this is useful + # to have cron send the messages. If emit_via includes email, this + # program will send email itself according to the configured options. + # If emit_via is None or left blank, no messages will be sent. + emit_via = stdio + + # The width, in characters, that messages that are emitted should be + # formatted to. + output_width = 80 + + + [email] + # The address to send email messages from. + # NOTE: 'localhost' will be replaced with the value of system_name. + email_from = root@localhost + + # List of addresses to send messages to. + email_to = root + + # Name of the host to connect to to send email messages. + email_host = localhost + + + [groups] + # NOTE: This only works when group_command != objects, which is now the default + # List of groups to update + group_list = None + + # The types of group packages to install + group_package_types = mandatory, default + + [base] + # This section overrides yum.conf + + # Use this to filter Yum core messages + # -4: critical + # -3: critical+errors + # -2: critical+errors+warnings (default) + debuglevel = 1 + + # skip_broken = True + mdpolicy = group:main + + # Uncomment to auto-import new gpg keys (dangerous) + # assumeyes = True + + unattended-upgrades.conf: | + // Override system /etc/apt/apt.conf.d/50unattended-upgrades + #clear "Unattended-Upgrade::Allowed-Origins"; + + // Automatically upgrade packages from these (origin:archive) pairs + Unattended-Upgrade::Allowed-Origins { + "${distro_id}:${distro_codename}"; + "${distro_id}:${distro_codename}-security"; + // Extended Security Maintenance; doesn't necessarily exist for + // every release and this system may not have it installed, but if + // available, the policy for updates is such that unattended-upgrades + // should also install from here by default. + "${distro_id}ESM:${distro_codename}"; + // "${distro_id}:${distro_codename}-updates"; + // "${distro_id}:${distro_codename}-proposed"; + // "${distro_id}:${distro_codename}-backports"; + }; + + // List of packages to not update (regexp are supported) + Unattended-Upgrade::Package-Blacklist { + // "vim"; + // "libc6"; + // "libc6-dev"; + // "libc6-i686"; + }; + + // This option allows you to control if on a unclean dpkg exit + // unattended-upgrades will automatically run + // dpkg --force-confold --configure -a + // The default is true, to ensure updates keep getting installed + //Unattended-Upgrade::AutoFixInterruptedDpkg "false"; + + // Split the upgrade into the smallest possible chunks so that + // they can be interrupted with SIGUSR1. This makes the upgrade + // a bit slower but it has the benefit that shutdown while a upgrade + // is running is possible (with a small delay) + //Unattended-Upgrade::MinimalSteps "true"; + + // Install all unattended-upgrades when the machine is shuting down + // instead of doing it in the background while the machine is running + // This will (obviously) make shutdown slower + //Unattended-Upgrade::InstallOnShutdown "true"; + + // Send email to this address for problems or packages upgrades + // If empty or unset then no email is sent, make sure that you + // have a working mail setup on your system. A package that provides + // 'mailx' must be installed. E.g. "user@example.com" + //Unattended-Upgrade::Mail "root"; + + // Set this value to "true" to get emails only on errors. Default + // is to always send a mail if Unattended-Upgrade::Mail is set + //Unattended-Upgrade::MailOnlyOnError "true"; + + // Do automatic removal of new unused dependencies after the upgrade + // (equivalent to apt-get autoremove) + Unattended-Upgrade::Remove-Unused-Dependencies "true"; + + // Automatically reboot *WITHOUT CONFIRMATION* + // if the file /var/run/reboot-required is found after the upgrade + //Unattended-Upgrade::Automatic-Reboot "false"; + + // If automatic reboot is enabled and needed, reboot at the specific + // time instead of immediately + // Default: "now" + //Unattended-Upgrade::Automatic-Reboot-Time "02:00"; + + // Use apt bandwidth limit feature, this example limits the download + // speed to 70kb/sec + //Acquire::http::Dl-Limit "70"; diff --git a/addons/host-upgrades/resources/50-daemonset.yml.erb b/addons/host-upgrades/resources/50-daemonset.yml.erb new file mode 100644 index 000000000..f8e3ff76d --- /dev/null +++ b/addons/host-upgrades/resources/50-daemonset.yml.erb @@ -0,0 +1,79 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: host-upgrades + namespace: kube-system + labels: + app: host-upgrades +spec: + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: host-upgrades + spec: + serviceAccountName: host-upgrades + containers: + - name: host-upgrades + image: "quay.io/kontena/pharos-host-upgrades-<%= arch.name %>:<%= version %>" + imagePullPolicy: IfNotPresent + command: + - pharos-host-upgrades + args: + - "--schedule=<%= schedule %>" + <% if schedule_window %> + - "--schedule-window=<%= schedule_window %>" + <% end %> + <% if reboot %> + - --reboot + <% end %> + <% if drain %> + - --drain + <% end %> + env: + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_DAEMONSET + value: host-upgrades + - name: KUBE_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - name: config + mountPath: /etc/host-upgrades + - name: host + mountPath: /run/host-upgrades + - name: dbus + mountPath: /var/run/dbus + <% if journal %> + - name: journal + mountPath: /run/log/journal + <% end %> + volumes: + - name: config + configMap: + name: host-upgrades + - name: host + hostPath: + path: /run/host-upgrades + type: DirectoryOrCreate + - name: dbus + hostPath: + path: /var/run/dbus + type: Directory + <% if journal %> + - name: journal + hostPath: + path: /var/log/journal + type: Directory + <% end %> + restartPolicy: Always + tolerations: + - effect: NoSchedule + operator: Exists diff --git a/addons/host-upgrades/resources/daemonset.yml.erb b/addons/host-upgrades/resources/daemonset.yml.erb deleted file mode 100644 index 4cb97d2e2..000000000 --- a/addons/host-upgrades/resources/daemonset.yml.erb +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: host-upgrades - namespace: kube-system -spec: - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - name: host-upgrades - spec: - hostPID: true - containers: - - name: host-upgrades - image: "docker.io/<%= arch.name == 'amd64' ? 'debian' : 'arm64v8/debian' %>:9" - imagePullPolicy: IfNotPresent - command: - - nsenter - args: - - --target=1 - - --mount - - --uts - - --net - - --pid - - -- - - sh - - -c - - "while true; sleep <%= interval %>; do /usr/bin/unattended-upgrade -d; done" - securityContext: - privileged: true - restartPolicy: Always - tolerations: - - effect: NoSchedule - operator: Exists \ No newline at end of file diff --git a/lib/pharos/addon.rb b/lib/pharos/addon.rb index aafb72a53..2536ad475 100644 --- a/lib/pharos/addon.rb +++ b/lib/pharos/addon.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true require 'dry-validation' +require 'fugit' + require_relative 'addons/struct' require_relative 'logging' @@ -28,9 +30,21 @@ def duration?(value) !Fugit::Duration.parse(value).nil? end + def cron?(value) + cron = Fugit::Cron.parse(value) + + return false if !cron + return false if cron.seconds != [0] + + true + end + def self.messages super.merge( - en: { errors: { duration?: 'is not valid duration' } } + en: { errors: { + duration?: 'is not valid duration', + cron?: 'is not a valid crontab' + } } ) end end diff --git a/pharos-cluster.gemspec b/pharos-cluster.gemspec index 78011d0ec..5bf039cf5 100644 --- a/pharos-cluster.gemspec +++ b/pharos-cluster.gemspec @@ -31,7 +31,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "dry-validation", "0.11.1" spec.add_runtime_dependency "dry-struct", "0.4.0" spec.add_runtime_dependency "deep_merge" - spec.add_runtime_dependency "fugit" + spec.add_runtime_dependency "fugit", "~> 1.1.2" spec.add_runtime_dependency "rouge", "~> 3.1" spec.add_runtime_dependency "tty-prompt", "~> 0.16" diff --git a/spec/pharos/addons/host_upgrades_spec.rb b/spec/pharos/addons/host_upgrades_spec.rb new file mode 100644 index 000000000..7dfcad970 --- /dev/null +++ b/spec/pharos/addons/host_upgrades_spec.rb @@ -0,0 +1,122 @@ +require 'pharos/addon' +require "./addons/host-upgrades/addon" + +describe Pharos::Addons::HostUpgrades do + let(:cluster_config) { Pharos::Config.load( + hosts: [ {role: 'master', address: '192.0.2.1'} ], + ) } + let(:config) { { } } + let(:cpu_arch) { double(:cpu_arch ) } + let(:master) { double(:host, address: '192.0.2.1') } + + subject do + described_class.new({enabled: true}.merge(config), enabled: true, master: master, cpu_arch: cpu_arch, cluster_config: cluster_config) + end + + describe "#validate" do + it "fails with missing schedule" do + result = described_class.validate({enabled: true}) + + expect(result).to_not be_success + expect(result.errors.dig(:schedule)).to eq ["is missing"] + end + + describe 'schedule' do + it "rejects empty schedule" do + result = described_class.validate({enabled: true, schedule: ''}) + + expect(result).to_not be_success + expect(result.errors.dig(:schedule)).to eq ["must be filled"] + end + + it "rejects invalid schedule" do + result = described_class.validate({enabled: true, schedule: '3'}) + + expect(result).to_not be_success + expect(result.errors.dig(:schedule)).to match [/is not a valid crontab/] + end + + it "rejects short cron schedule" do + result = described_class.validate({enabled: true, schedule: '0 * * *'}) + + expect(result).to_not be_success + expect(result.errors.dig(:schedule)).to match [/is not a valid crontab/] + end + + it "rejects long cron schedule" do + result = described_class.validate({enabled: true, schedule: '30 0 0 * * *'}) + + expect(result).to_not be_success + expect(result.errors.dig(:schedule)).to match [/is not a valid crontab/] + end + end + end + + describe '#schedule' do + let(:config) { { schedule: schedule } } + + context "with a @ schedule" do + let(:schedule) { '@daily' } + + it "normalizes it" do + expect(subject.schedule).to eq '0 0 * * *' + end + end + + context "with a seconds schedule" do + let(:schedule) { '0 30 3 * * *' } + + it "normalizes it" do + expect(subject.schedule).to eq '30 3 * * *' + end + end + + context "with a simple schedule" do + let(:schedule) { '0 0 * * *' } + + it "passes it through" do + expect(subject.schedule).to eq schedule + end + end + + context "with a range/interval schedule" do + let(:schedule) { '0 3-8/2 * * *' } + + it "normalizes it" do + expect(subject.schedule).to eq '0 3,5,7 * * *' + end + end + + context "with a weekday schedule" do + let(:schedule) { '0 3 * * 1,3,5' } + + it "normalizes it" do + expect(subject.schedule).to eq schedule + end + end + end + + describe '#schedule_window' do + context "by default" do + it "normalizes to zero" do + expect(subject.schedule_window).to eq '0' + end + end + + context "with a simple duration" do + let(:config) { { schedule_window: "1 day" } } + + it "normalizes to seconds" do + expect(subject.schedule_window).to eq '86400s' + end + end + + context "with a simple duration" do + let(:config) { { schedule_window: "1h30m" } } + + it "normalizes to seconds" do + expect(subject.schedule_window).to eq '5400s' + end + end + end +end