From b666703d738da894dd74f5aebbf384611ca2f9c4 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Thu, 14 Nov 2013 12:29:55 +0200 Subject: [PATCH] profile_id detection for non-TurnKey systems --- cmd_backup.py | 4 +- cmd_init.py | 21 ++--- cmd_internals/cmd_detect_profile.py | 106 ------------------------- cmd_internals/cmd_detect_profile_id.py | 41 ++++++++++ cmd_list.py | 5 +- cmd_restore.py | 22 +++-- debian/control | 2 +- dummyhub.py | 16 ++-- hub.py | 6 +- registry.py | 36 +++++---- version.py | 77 ++++++++++++++++-- 11 files changed, 180 insertions(+), 156 deletions(-) delete mode 100755 cmd_internals/cmd_detect_profile.py create mode 100755 cmd_internals/cmd_detect_profile_id.py diff --git a/cmd_backup.py b/cmd_backup.py index 9621d02..4a3ef7b 100755 --- a/cmd_backup.py +++ b/cmd_backup.py @@ -144,7 +144,7 @@ from registry import registry, update_profile, hub_backups from conf import Conf -from version import Version +from version import detect_profile_id from stdtrap import UnitedStdTrap from utils import is_writeable, fmt_title, fmt_timestamp @@ -373,7 +373,7 @@ def main(): if not registry.hbr: registry.hbr = hb.new_backup_record(registry.key, - str(Version.from_system()), + detect_profile_id(), get_server_id()) conf.address = registry.hbr.address diff --git a/cmd_init.py b/cmd_init.py index 6403ad7..cac7de9 100755 --- a/cmd_init.py +++ b/cmd_init.py @@ -12,11 +12,13 @@ """ Initialization (start here) -By default, this links TKLBAM to your Hub account and downloads a backup profile, -which is used to calculate the list of system changes we need to backup. -The profile usually describes the installation state of a TurnKey appliance and -contains a list of packages, filesystem paths to scan for changes and an index of -the contents of those paths which records timestamps, ownership and permissions. +By default, this links TKLBAM to your Hub account and downloads an appropriate +backup profile, which is used to calculate the list of system changes we need +to backup. On a TurnKey system the profile describes the installation state of +the appliance and contains a list of packages, filesystem paths to scan for +changes and an index of the contents of those paths which records timestamps, +ownership and permissions. On a non-TurnKey system the default backup profile +will not describe installation state, only a list of directories to backup. Arguments: @@ -29,11 +31,12 @@ --force-profile=PROFILE_ID Force a specific backup profile (e.g., "core", "turnkey-core-13.0-wheezy-amd64") - Default value: String in /etc/turnkey_version + Without --force-profile the profile_id is + automatically detected. - Special value: "empty" creates an empty - backup profile. Backup configurations will - only be taken from /etc/tklbam. + --force-profile=empty "empty" is a special profile_id value that creates an empty + backup profile. Backup configurations will only be taken + from /etc/tklbam. --force-profile=PATH Path to a custom backup profile Details: tklbam-internal create-profile --help diff --git a/cmd_internals/cmd_detect_profile.py b/cmd_internals/cmd_detect_profile.py deleted file mode 100755 index 210f1e8..0000000 --- a/cmd_internals/cmd_detect_profile.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) 2013 Liraz Siri -# -# This file is part of TKLBAM (TurnKey Linux BAckup and Migration). -# -# TKLBAM is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 3 of -# the License, or (at your option) any later version. -# -import re -import sys -import getopt -from os.path import * - -def _get_turnkey_version(root): - path = join(root, 'etc/turnkey_version') - if not exists(path): - return - - return file(path).read().strip() - -def _parse_keyvals(path): - if not exists(path): - return - d = {} - for line in file(path).readlines(): - line = line.strip() - if not line: - continue - - m = re.match(r'(.*?)="?(.*?)"?$', line) - key, val = m.groups() - d[key] = val - return d - -def _get_os_release(root): - path = join(root, "etc/os-release") - return _parse_keyvals(path) - -def _get_lsb_release(root): - path = join(root, "etc/lsb-release") - return _parse_keyvals(path) - -def _get_debian_version(root): - path = join(root, "etc/debian_version") - if not exists(path): - return - - s = file(path).read().strip() - m = re.match(r'^(\d+)\.', s) - if m: - return m.group(1) - - if '/' in s: - return s.replace('/', '_') - -def detect_profile(root): - val = _get_turnkey_version(root) - if val: - return val - - os_release = _get_os_release(root) - if os_release: - try: - return "%s-%s" % (os_release['ID'], os_release['VERSION_ID']) - except KeyError: - pass - - lsb_release = _get_lsb_release(root) - if lsb_release: - try: - return "%s-%s" % (lsb_release['DISTRIB_ID'].lower(), - lsb_release['DISTRIB_RELEASE']) - except KeyError: - pass - - debian_version = _get_debian_version(root) - if debian_version: - return "debian-" + debian_version - - return "generic" - -def usage(e=None): - if e: - print >> sys.stderr, "error: " + str(e) - print >> sys.stderr, "Syntax: %s [ path/to/root ]" % sys.argv[0] - sys.exit(1) - -def main(): - try: - opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['help']) - except getopt.GetoptError, e: - usage(e) - - for opt, val in opts: - if opt in ('-h', '--help'): - usage() - - root = args[0] if args else '/' - - print detect_profile(root) - -if __name__ == "__main__": - main() diff --git a/cmd_internals/cmd_detect_profile_id.py b/cmd_internals/cmd_detect_profile_id.py new file mode 100755 index 0000000..97635cd --- /dev/null +++ b/cmd_internals/cmd_detect_profile_id.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +# +# Copyright (c) 2013 Liraz Siri +# +# This file is part of TKLBAM (TurnKey Linux BAckup and Migration). +# +# TKLBAM is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 3 of +# the License, or (at your option) any later version. +# +""" +Determine your system's default backup profile_id +""" +import sys +import getopt + +from version import detect_profile_id + +def usage(e=None): + if e: + print >> sys.stderr, "error: " + str(e) + print >> sys.stderr, "Syntax: %s [ path/to/root ]" % sys.argv[0] + sys.exit(1) + +def main(): + try: + opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['help']) + except getopt.GetoptError, e: + usage(e) + + for opt, val in opts: + if opt in ('-h', '--help'): + usage() + + root = args[0] if args else '/' + + print detect_profile_id(root) + +if __name__ == "__main__": + main() diff --git a/cmd_list.py b/cmd_list.py index 27d3550..c5db1fc 100755 --- a/cmd_list.py +++ b/cmd_list.py @@ -18,7 +18,7 @@ %id Backup id %label Descriptive label - %turnkey_version Appliance version code + %profile_id Appliance version code %server_id Associated server id (- if empty) %created Date the backup record was created %updated Date the backup record was last updated @@ -63,6 +63,9 @@ def __init__(self, format): self.tpl = string.Template(tpl) def __call__(self, hbr): + # backwards compat. hack + hbr = hbr.copy() + hbr['turnkey_version'] = hbr['profile_id'] return self.tpl.substitute(**hbr) def key_has_passphrase(key): diff --git a/cmd_restore.py b/cmd_restore.py index 1841fb2..5709d84 100755 --- a/cmd_restore.py +++ b/cmd_restore.py @@ -151,7 +151,7 @@ from registry import registry, update_profile, hub_backups -from version import Version +from version import TurnKeyVersion from utils import is_writeable, fmt_timestamp, fmt_title from conf import Conf @@ -169,10 +169,18 @@ class ExitCode: INCOMPATIBLE = 10 BADPASSPHRASE = 11 -def do_compatibility_check(backup_turnkey_version, interactive=True): +def do_compatibility_check(backup_profile_id, interactive=True): + # unless both backup and restore are TurnKey skip compatibility check + try: + backup_codename = TurnKeyVersion.from_string(backup_profile_id).codename + except TurnKeyVersion.Error: + return - backup_codename = Version.from_string(backup_turnkey_version).codename - local_codename = Version.from_system().codename + turnkey_version = TurnKeyVersion.from_system() + if not turnkey_version: + return + + local_codename = turnkey_version.codename if local_codename == backup_codename: return @@ -186,8 +194,8 @@ def fmt(s): print "WARNING: INCOMPATIBLE APPLIANCE BACKUP" print "======================================" print - print "Restoring a %s backup to a %s appliance is not recommended." % (backup_codename, local_codename) - print "For best results, restore to a fresh %s installation instead." % backup_codename + print "Restoring a %s backup to a %s appliance may create complications." % (backup_codename, local_codename) + print "For best results try restoring instead to a fresh %s installation." % backup_codename if not interactive: sys.exit(ExitCode.INCOMPATIBLE) @@ -434,7 +442,7 @@ def main(): if hbr: if not opt_force and not raw_download_path: - do_compatibility_check(hbr.turnkey_version, interactive) + do_compatibility_check(hbr.profile_id, interactive) if opt_key and \ keypacket.fingerprint(hbr.key) != keypacket.fingerprint(opt_key): diff --git a/debian/control b/debian/control index 7f7dd29..968d920 100644 --- a/debian/control +++ b/debian/control @@ -7,6 +7,6 @@ Standards-Version: 3.6.1 Package: tklbam Architecture: any -Depends: python (>= 2.4), tklbam-squid (>= 2.7.STABLE9-2.1turnkey+25), tklbam-duplicity (>= 0.6.18), tklbam-python-boto (>= 2.3.0), turnkey-pylib (>= 0.5), turnkey-version, python-crypto, pycurl-wrapper (>= 1.2), python-simplejson, ca-certificates, ntpdate +Depends: python (>= 2.4), tklbam-squid (>= 2.7.STABLE9-2.1turnkey+25), tklbam-duplicity (>= 0.6.18), tklbam-python-boto (>= 2.3.0), turnkey-pylib (>= 0.5), python-crypto, pycurl-wrapper (>= 1.2), python-simplejson, ca-certificates, ntpdate Section: misc Description: TurnKey Linux Backup and Migration agent diff --git a/dummyhub.py b/dummyhub.py index 56e1360..f6537e5 100644 --- a/dummyhub.py +++ b/dummyhub.py @@ -78,12 +78,12 @@ def subscribe(self): def unsubscribe(self): self.credentials = None - def new_backup(self, address, key, turnkey_version, server_id=None): + def new_backup(self, address, key, profile_id, server_id=None): self.backups_max += 1 id = str(self.backups_max) backup_record = DummyBackupRecord(id, address, key, \ - turnkey_version, server_id) + profile_id, server_id) self.backups[id] = backup_record @@ -142,11 +142,11 @@ def _parse_duplicity_sessions(path): class DummyBackupRecord(AttrDict): # backup_id, address - def __init__(self, backup_id, address, key, turnkey_version, server_id): + def __init__(self, backup_id, address, key, profile_id, server_id): self.backup_id = backup_id self.address = address self.key = key - self.turnkey_version = turnkey_version + self.profile_id = profile_id self.server_id = server_id self.created = datetime.now() @@ -218,8 +218,8 @@ def add_user(self): return user - def get_profile(self, turnkey_version): - matches = glob.glob("%s/%s.tar.*" % (self.path.profiles, turnkey_version)) + def get_profile(self, profile_id): + matches = glob.glob("%s/%s.tar.*" % (self.path.profiles, profile_id)) if not matches: return None @@ -308,7 +308,7 @@ def get_new_profile(self, profile_id, profile_timestamp): return DummyProfileArchive(profile_id, archive, archive_timestamp) - def new_backup_record(self, key, turnkey_version, server_id=None): + def new_backup_record(self, key, profile_id, server_id=None): # in the real implementation the hub would create a bucket not a dir... # the real implementation would have to make sure this is unique path = "/var/tmp/duplicity/" + base64.b32encode(os.urandom(10)) @@ -316,7 +316,7 @@ def new_backup_record(self, key, turnkey_version, server_id=None): address = "file://" + path backup_record = self.user.new_backup(address, key, - turnkey_version, server_id) + profile_id, server_id) dummydb.save() diff --git a/hub.py b/hub.py index 15b2942..1848c77 100644 --- a/hub.py +++ b/hub.py @@ -139,7 +139,7 @@ def __init__(self, response): self.address = response['address'] self.backup_id = response['backup_id'] self.server_id = response['server_id'] - self.turnkey_version = response['turnkey_version'] + self.profile_id = response['turnkey_version'] self.created = self._parse_datetime(response['date_created']) self.updated = self._parse_datetime(response['date_updated']) @@ -215,8 +215,8 @@ def get_new_profile(self, profile_id, profile_timestamp): return ProfileArchive(profile_id, archive_path, archive_timestamp) - def new_backup_record(self, key, turnkey_version, server_id=None): - attrs = {'key': key, 'turnkey_version': turnkey_version} + def new_backup_record(self, key, profile_id, server_id=None): + attrs = {'key': key, 'turnkey_version': profile_id} if server_id: attrs['server_id'] = server_id diff --git a/registry.py b/registry.py index bd9e0f4..76f8dfb 100644 --- a/registry.py +++ b/registry.py @@ -23,7 +23,7 @@ from utils import AttrDict import hub -from version import Version +from version import TurnKeyVersion, detect_profile_id import conf @@ -42,14 +42,19 @@ class ProfileNotFound(Exception): - Restore existing backups - Backup raw directories with the --raw-upload option -Also, you can use the --force-profile option to work around this in several ways: +You can use the --force-profile option to fix or workaround a missing profile +in several ways: -- Download a profile for the another system (e.g., --force-profile=core). - Use an empty profile with --force-profile=empty - Use a custom profile with --force-profile=path/to/custom/profile/ tklbam-internal create-profile --help +Also, if TKLBAM is linked to the Hub you can: + +- Download a profile for another TurnKey system with --force-profile=codename (e.g., "core") +- Download the all-purpose generic profile with --force-profile=generic + Run "tklbam-init --help" for further details. """ @@ -167,7 +172,7 @@ def profile(self, val=UNDEFINED): timestamp = int(os.stat(self.path.profile.stamp).st_mtime) profile_id = self._file_str(self.path.profile.profile_id) if profile_id is None: - profile_id = str(Version.from_system()) + profile_id = detect_profile_id() return Profile(self.path.profile, profile_id, timestamp) else: profile_archive = val @@ -221,7 +226,7 @@ def _update_profile(self, profile_id=None): if self.profile: profile_id = self.profile.profile_id else: - profile_id = str(Version.from_system()) + profile_id = detect_profile_id() if profile_id == self.EMPTY_PROFILE or isdir(profile_id): self.profile = profile_id @@ -255,7 +260,7 @@ def _update_profile(self, profile_id=None): def update_profile(self, profile_id=None): if profile_id is None: - # don't empty empty or custom profiles + # don't attempt to update empty or custom profiles if self.profile and \ (self.profile.profile_id == registry.EMPTY_PROFILE or self.profile.profile_id.startswith(registry.CUSTOM_PROFILE + ":")): @@ -269,19 +274,22 @@ def update_profile(self, profile_id=None): if not re.match(r'^turnkey-', profile_id): profile_id = "turnkey-" + profile_id - if not Version.from_string(profile_id).is_complete(): + if not TurnKeyVersion.from_string(profile_id).is_complete(): completed_profile_id = _complete_profile_id(profile_id) - try: - self._update_profile(completed_profile_id) - return - except: - pass + if completed_profile_id: + try: + self._update_profile(completed_profile_id) + return + except: + pass raise first_exception def _complete_profile_id(partial): - partial = Version.from_string(partial) - system = Version.from_system() + partial = TurnKeyVersion.from_string(partial) + system = TurnKeyVersion.from_system() + if not system: + return if partial.arch is None: partial.arch = system.arch diff --git a/version.py b/version.py index 513798e..01c8953 100644 --- a/version.py +++ b/version.py @@ -10,13 +10,15 @@ # import re import executil +from os.path import * from utils import AttrDict class Error(Exception): pass -class Version(AttrDict): +class TurnKeyVersion(AttrDict): + Error = Error def __init__(self, codename, release=None, arch=None): AttrDict.__init__(self) self.codename = codename @@ -37,10 +39,7 @@ def from_system(cls): try: system_version = file("/etc/turnkey_version").readline().strip() except: - try: - system_version = executil.getoutput("turnkey-version") - except executil.ExecError: - return None + return return cls.from_string(system_version) @@ -65,3 +64,71 @@ def from_string(cls, version): if m: name = m.group(1) return cls(name) + +def _get_turnkey_version(root): + path = join(root, 'etc/turnkey_version') + if not exists(path): + return + + return file(path).read().strip() + +def _parse_keyvals(path): + if not exists(path): + return + d = {} + for line in file(path).readlines(): + line = line.strip() + if not line: + continue + + m = re.match(r'(.*?)="?(.*?)"?$', line) + key, val = m.groups() + d[key] = val + return d + +def _get_os_release(root): + path = join(root, "etc/os-release") + return _parse_keyvals(path) + +def _get_lsb_release(root): + path = join(root, "etc/lsb-release") + return _parse_keyvals(path) + +def _get_debian_version(root): + path = join(root, "etc/debian_version") + if not exists(path): + return + + s = file(path).read().strip() + m = re.match(r'^(\d+)\.', s) + if m: + return m.group(1) + + if '/' in s: + return s.replace('/', '_') + +def detect_profile_id(root='/'): + val = _get_turnkey_version(root) + if val: + return val + + os_release = _get_os_release(root) + if os_release: + try: + return "%s-%s" % (os_release['ID'], os_release['VERSION_ID']) + except KeyError: + pass + + lsb_release = _get_lsb_release(root) + if lsb_release: + try: + return "%s-%s" % (lsb_release['DISTRIB_ID'].lower(), + lsb_release['DISTRIB_RELEASE']) + except KeyError: + pass + + debian_version = _get_debian_version(root) + if debian_version: + return "debian-" + debian_version + + return "generic"