Starting prototyping of pyvsb-ng

main.py

@@ -0,0 +1,56 @@
+import sys
+import logging
+
+from pyvsb.backup import Backuper
+
+class OutputHandler(logging.Handler):
+    """
+    Log handler that logs debug and info messages to stdout and all other
+    messages to stderr.
+    """
+
+    def __init__(self, *args, **kwargs):
+        logging.Handler.__init__(self, *args, **kwargs)
+
+
+    def emit(self, record):
+        self.acquire()
+
+        try:
+            stream = sys.stdout if record.levelno <= logging.INFO else sys.stderr
+            print >> stream, self.format(record).encode("utf-8")
+        except:
+            self.handleError(record)
+        finally:
+            self.release()
+
+
+def setup(debug_mode = False, filter = None, max_log_name_length = 16, level = None):
+    """Sets up the logging."""
+
+    logging.addLevelName(logging.DEBUG,   "D")
+    logging.addLevelName(logging.INFO,    "I")
+    logging.addLevelName(logging.WARNING, "W")
+    logging.addLevelName(logging.ERROR,   "E")
+
+    log = logging.getLogger()
+
+    log.setLevel(logging.DEBUG if debug_mode else logging.INFO)
+    if level is not None:
+        log.setLevel(level)
+
+    format = ""
+    if debug_mode:
+        format += "%(asctime)s.%(msecs)03d (%(filename)12.12s:%(lineno)04d) [%(name){0}.{0}s]: ".format(max_log_name_length)
+    format += "%(levelname)s: %(message)s"
+
+    handler = OutputHandler()
+    handler.setFormatter(logging.Formatter(format, "%Y.%m.%d %H:%M:%S"))
+    if filter is not None:
+        handler.addFilter(filter)
+
+    log.addHandler(handler)
+
+if __name__ == "__main__":
+    setup(True)
+    Backuper().backup()

pyvsb/backup.py

@@ -0,0 +1,107 @@
+import errno
+import logging
+import os
+import stat
+
+import psys
+from psys import eintr_retry
+
+from .core import Error
+from .storage import Storage
+
+LOG = logging.getLogger(__name__)
+
+# TODO
+class FileTypeChangedError(Exception):
+    pass
+
+# TODO
+class Backuper:
+    def __init__(self):
+        self.__items = [ "tests/test_root/etc", "tests/test_root/home" ]
+        self.__open_flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NOATIME
+        self.__storage = Storage()
+
+    def backup(self):
+        for item in self.__items:
+            self.__backup(item, toplevel = True)
+
+
+    def __backup(self, path, toplevel = False):
+        """Backups the specified path."""
+
+        LOG.debug("Backing up '%s'...", path)
+
+        try:
+            stat_info = os.lstat(path)
+
+            if stat.S_ISREG(stat_info.st_mode):
+                self.__backup_file(path)
+            else:
+                if stat.S_ISLNK(stat_info.st_mode):
+                    try:
+                        link_target = os.readlink(path)
+                    except EnvironmentError as e:
+                        if e.errno == errno.EINVAL:
+                            raise FileTypeChangedError()
+                        else:
+                            raise
+                else:
+                    link_target = ""
+
+                self.__storage.add(path, stat_info, link_target = link_target)
+
+            if stat.S_ISDIR(stat_info.st_mode):
+                for filename in os.listdir(path):
+                    self.__backup(os.path.join(path, filename))
+        except FileTypeChangedError as e:
+            LOG.error("Failed to backup %s: it has suddenly changed its type during the backup.", path)
+        except Exception as e:
+            if (
+                isinstance(e, EnvironmentError) and
+                e.errno in ( errno.ENOENT, errno.ENOTDIR ) and not toplevel
+            ):
+                LOG.warning("Failed to backup %s: it has suddenly vanished.", path)
+            else:
+                LOG.error("Failed to backup %s: %s.", path, psys.e(e))
+
+
+    def __backup_file(self, path):
+        """Backups the specified file."""
+
+        try:
+            try:
+                fd = eintr_retry(os.open)(path, self.__open_flags)
+            except EnvironmentError as e:
+                # If O_NOATIME flag was specified, but the effective user ID
+                # of the caller did not match the owner of the file and the
+                # caller was not privileged (CAP_FOWNER), the EPERM will be
+                # returned.
+                if e.errno == errno.EPERM and self.__open_flags & os.O_NOATIME:
+                    # Just disable this flag on a first EPERM error
+                    LOG.error("Got EPERM error. Disabling O_NOATIME for file opening operations...") # TODO: debug
+                    self.__open_flags &= ~os.O_NOATIME
+                    fd = eintr_retry(os.open)(path, self.__open_flags)
+                else:
+                    raise
+        except EnvironmentError as e:
+            # When O_NOFOLLOW is specified, indicates that this is a
+            # symbolic link.
+            if e.errno == errno.ELOOP:
+                raise FileTypeChangedError()
+            else:
+                raise
+
+        try:
+            file_obj = os.fdopen(fd, "rb")
+        except:
+            try:
+                eintr_retry(os.close)(fd)
+            except Exception as e:
+                LOG.error("Unable to close a file: %s.", psys.e(e))
+
+            raise
+
+        with file_obj:
+            stat_info = os.fstat(file_obj.fileno())
+            self.__storage.add(path, stat_info, file_obj = file_obj)

pyvsb/core.py

@@ -0,0 +1,8 @@
+# TODO
+
+class Error(Exception):
+    """A base class for all exceptions the module throws."""
+
+    def __init__(self, error, *args, **kwargs):
+        super(Error, self).__init__(
+            error.format(*args, **kwargs) if args or kwargs else error)

pyvsb/storage.py

@@ -0,0 +1,71 @@
+import grp
+import logging
+import os
+import pwd
+import stat
+import tarfile
+
+LOG = logging.getLogger(__name__)
+
+class Storage:
+    def __init__(self):
+        self.__storage = tarfile.open("storage", "w")
+        self.__inodes = {}
+
+    def add(self, path, stat_info, link_target = "", file_obj = None):
+        """Adds a file to the storage."""
+
+        LOG.debug("Storing file '%s'...", path)
+
+        tar_info = _get_tar_info(path, stat_info, link_target)
+        self.__storage.addfile(tar_info, fileobj = file_obj)
+
+
+def _get_tar_info(path, stat_info, link_target):
+    """Returns a TarInfo object for the specified file."""
+
+    tar_info = tarfile.TarInfo()
+    stat_mode = stat_info.st_mode
+
+    if stat.S_ISREG(stat_mode):
+        tar_info.type = tarfile.REGTYPE
+    elif stat.S_ISDIR(stat_mode):
+        tar_info.type = tarfile.DIRTYPE
+    elif stat.S_ISLNK(stat_mode):
+        tar_info.type = tarfile.SYMTYPE
+    elif stat.S_ISFIFO(stat_mode):
+        tar_info.type = tarfile.FIFOTYPE
+    elif stat.S_ISCHR(stat_mode):
+        tar_info.type = tarfile.CHRTYPE
+    elif stat.S_ISBLK(stat_mode):
+        tar_info.type = tarfile.BLKTYPE
+    else:
+        raise Exception("File type is not supported")
+
+    tar_info.name = path.lstrip("/")
+    tar_info.mode = stat_mode
+    tar_info.uid = stat_info.st_uid
+    tar_info.gid = stat_info.st_gid
+    tar_info.mtime = stat_info.st_mtime
+    tar_info.linkname = link_target
+
+    if tar_info.type == tarfile.REGTYPE:
+        tar_info.size = stat_info.st_size
+    else:
+        tar_info.size = 0
+
+    try:
+        tar_info.uname = pwd.getpwuid(stat_info.st_uid)[0]
+    except KeyError:
+        pass
+
+    try:
+        tar_info.gname = grp.getgrgid(stat_info.st_gid)[0]
+    except KeyError:
+        pass
+
+    if tar_info.type in ( tarfile.CHRTYPE, tarfile.BLKTYPE ):
+        tar_info.devmajor = os.major(stat_info.st_rdev)
+        tar_info.devminor = os.minor(stat_info.st_rdev)
+
+    return tar_info