From fff5ff59896293c4aa0da1c98b8bc5d8350ea117 Mon Sep 17 00:00:00 2001 From: Matteo Ceccarello Date: Wed, 8 Apr 2015 22:53:01 +0200 Subject: [PATCH 1/3] test: Fix caching issues (#111) This commit deals with the problems of caching LilyPond versions between Travis-CI builds reported in #111. Specifically, it turned out that there is not a separate cache per build-matrix entry (since LilyPond is not a supported language), but only a dedicated cache per branch. The issue was then that only one version of LilyPond was actually used to run the tests, since the installation path was identical for all versions. The solution introduced in this commit is to have an explicit cache management mechanism: - Each LilyPond version is installed in its own folder under the $HOME/.lilypond cache directory, for instance $HOME/.lilypond/linux-64/2.18.2-1 - Every time a cached LilyPond version is used, the timestamp is written to a file in the cache, for later retrieval - After all the tests have been run, the cache is cleaned: LilyPond versions that have not been used for more than one day (this is configurable) are removed from the cache. Note that since this cleanup phase is run after the tests have been run, all the LilyPond versions that have actually been used in the last test run are kept in the cache --- test/automated_tests.py | 71 ++++++++------- test/common_functions.py | 1 - test/install_lilypond.py | 47 +--------- test/lilycmd.py | 187 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 225 insertions(+), 81 deletions(-) mode change 100644 => 100755 test/common_functions.py mode change 100644 => 100755 test/install_lilypond.py create mode 100644 test/lilycmd.py diff --git a/test/automated_tests.py b/test/automated_tests.py index 1897778b..6a7366f2 100755 --- a/test/automated_tests.py +++ b/test/automated_tests.py @@ -1,16 +1,16 @@ #!/usr/bin/env python -import subprocess as sp import os import os.path as osp import shutil import sys import re +from lilycmd import LilyCmd -from common_functions import print_separator, home_dir, install_root +from common_functions import print_separator -class SimpleTests: +class SimpleTests(object): """Run simple intergration tests. Specifically, this script will look for all the files in `usage-examples` directories. All these files will be compiled with LilyPond. If the compilation results in a @@ -56,24 +56,25 @@ def __init__(self, cmd=None): # root directory self.openlilylib_dir = self.__openlilylib_dir() - # LilyPond command - if self.is_ci_run(): - try: - self.lily_command = osp.join(install_root, - "bin", - "lilypond") - self.lilypond_version = self.__lilypond_version() - except KeyError: - sys.exit('Environment variable {} not set. Aborting'.format(self.lily_version_var)) + if 'CI' in os.environ and bool(os.environ["CI"]): + # TODO check definition + lily_platform = os.environ["LILY_PLATFORM"] + lily_version = os.environ["LILY_VERSION"] + + self.lily_command = LilyCmd.with_version(lily_platform, + lily_version) + if not self.lily_command.installed: + raise Exception('The required lilypond version is not installed') + self.lilypond_version = self.lily_command.version else: - self.lily_command = cmd if cmd else "lilypond" - self.lilypond_version = self.__lilypond_version() + self.lily_command = LilyCmd.system(cmd if cmd else "lilypond") + self.lilypond_version = self.lily_command.version # Add include path and other options to generated LilyPond command - self.lily_command_with_includes = [self.lily_command, - "-dno-point-and-click", - "-I", self.openlilylib_dir, - "-I", os.path.join(self.openlilylib_dir, "ly")] + self.lily_command_with_includes_args = [ + "-dno-point-and-click", + "-I", self.openlilylib_dir, + "-I", os.path.join(self.openlilylib_dir, "ly")] # initialize some lists self.test_files = [] self.included_tests = [] @@ -102,14 +103,6 @@ def __collect_all_in_dir(self, dirname): if os.path.isfile(test_fname) and self.is_lilypond_file(test_fname): self.test_files.append(test_fname) - - def __lilypond_version(self): - """Determine the LilyPond version actually run by the command self.lily_command""" - lily = sp.Popen([self.lily_command, "-v"], stdout=sp.PIPE, stderr=sp.PIPE) - version_line = lily.communicate()[0].splitlines()[0] - return re.search(r"\d+\.\d+\.\d+", version_line).group(0) - - def __openlilylib_dir(self): """Return the root directory of openLilyLib. It's the parent directory of the script.""" @@ -228,7 +221,9 @@ def print_introduction(self): print "OpenLilyLib directory: {}".format(self.openlilylib_dir) print "LilyPond command to be used:" - print " ".join(self.lily_command_with_includes + ["-o "]) + print " ".join([self.lily_command.command] + + self.lily_command_with_includes_args + + ["-o "]) def report(self): @@ -254,7 +249,8 @@ def report(self): print self.failed_tests[test] print "" print_separator() - sys.exit(1) + return 1 + return 0 def run(self): @@ -271,13 +267,11 @@ def run(self): os.path.dirname(self.__relative_path(test))) if not os.path.exists(test_result_dir): os.makedirs(test_result_dir) - lily = sp.Popen(self.lily_command_with_includes + ['-o', - test_result_dir, - test], - stdout=sp.PIPE, stderr=sp.PIPE) - (out, err) = lily.communicate() - - if lily.returncode == 0: + returncode, out, err = self.lily_command.execute( + self.lily_command_with_includes_args + ['-o', + test_result_dir, + test]) + if returncode == 0: print "------- OK! --------" else: # if test failed, add it to the list of failed tests to be reported later @@ -300,4 +294,9 @@ def run(self): tests.clean_results_dir() tests.collect_tests() tests.run() - tests.report() + retcode = tests.report() + + # cleanup old version of lilypond + LilyCmd.clean_cache() + + sys.exit(retcode) diff --git a/test/common_functions.py b/test/common_functions.py old mode 100644 new mode 100755 index 59de505c..8befc8da --- a/test/common_functions.py +++ b/test/common_functions.py @@ -32,4 +32,3 @@ def load_lily_versions(): def print_separator(): print "" print "="*79, "\n" - diff --git a/test/install_lilypond.py b/test/install_lilypond.py old mode 100644 new mode 100755 index 7658168e..bcdc2311 --- a/test/install_lilypond.py +++ b/test/install_lilypond.py @@ -1,14 +1,10 @@ #!/usr/bin/env python -import subprocess as sp import os -import os.path as osp -import shutil import sys -import collections -import common_functions -from common_functions import print_separator, home_dir, install_root +from lilycmd import LilyCmd +from common_functions import print_separator ############################################################# # Load environment variables @@ -21,43 +17,6 @@ except: sys.exit('\nScript can only be run in CI mode. Aborting\n') -######################### -# Configuration constants - -# Download site for LilyPond distributions -binary_site = "http://download.linuxaudio.org/lilypond/binaries/" -# String template for generating the LilyPond installation command -lily_install_script = "lilypond-install.sh" - - -################################# -# Functions doing the actual work - -def download_url(): - """Format a string representing the URL to download the requested LilyPond distribution""" - return "{}{}/lilypond-{}.{}.sh".format( - binary_site, lily_platform, lily_version, lily_platform) - -def install_distribution(): - """Download and install LilyPond version if not cached""" - lilypond_cmd = os.path.join(install_root, - "bin/lilypond") - print "\nChecking LilyPond presence with {}\n".format(lilypond_cmd) - try: - sp.check_call([lilypond_cmd, '--version']) - print "LilyPond {} is already installed in cache, continuing with test script.".format(lily_version) - except: - print "LilyPond {} is not installed yet.".format(lily_version) - print "Downloading and installing now" - sp.check_call( - ["wget", "-O", - lily_install_script, - download_url()]) - sp.check_call(["sh", lily_install_script, - "--prefix", - install_root, - "--batch"]) - ######################### # Actual script execution @@ -70,4 +29,4 @@ def install_distribution(): print "check LilyPond installation." print "Requested LilyPond version: {}".format(lily_version) - install_distribution() + LilyCmd.install(lily_platform, lily_version) diff --git a/test/lilycmd.py b/test/lilycmd.py new file mode 100644 index 00000000..8951c428 --- /dev/null +++ b/test/lilycmd.py @@ -0,0 +1,187 @@ +import subprocess as sp +import os +import os.path as osp +import shutil +import sys +import re +import datetime +import dateutil.parser +from common_functions import print_separator + +class LilyCmd(object): + """This class represents a lilypond command and provides some + facilities to + + - run LilyPond + - install LilyPond versions from the internet into a local cache + - manage the cache, cleaning old versions + + """ + + # Download site for LilyPond distributions + binary_site = "http://download.linuxaudio.org/lilypond/binaries/" + # String template for generating the LilyPond installation command + lily_install_script = "lilypond-install.sh" + + # After this amount of time a LilyPond version will be removed by + # a call to LilyCmd.clean_cache() + cache_cleanup_interval = datetime.timedelta(days=1) + + # Root directory for the cache + cache_root = osp.join(os.getenv('HOME'), '.lilypond') + + def __init__(self, command_path, cached): + self.command = command_path + self.cached = cached + try: + self.version = self._lilypond_version() + self.installed = True + except: + self.installed = False + + #################################################################### + # Public methods + + @staticmethod + def system(cmd_path='lilypond'): + """Get a new instance of Lilypond provided by the + system. (eg. /usr/bin/lilypond)""" + return LilyCmd(cmd_path, cached=False) + + @classmethod + def with_version(cls, platform, version): + """Get a new version of Lilypond, given the platform and the + version. + + **Note**: This command does not install LilyPond in the local + cache. To check if the instance returned by this method + corresponds to an actually installed LilyPond version, check + the `installed` attribute. The reason for not installing + LilyPond with this command is that we may be interested in all + kind of ancillary information (like the cache directory) + without actually installing LilyPond. + + """ + lily_cmd_path = osp.join( + LilyCmd._cache_directory(platform, version), + 'bin', 'lilypond') + lily_cmd = LilyCmd(lily_cmd_path, cached=True) + return lily_cmd + + def execute(self, args): + """Executes the LilyPond command with the given arguments""" + self._mark_cache() + lily = sp.Popen([self.command] + args, + stdout=sp.PIPE, stderr=sp.PIPE) + (out, err) = lily.communicate() + return lily.returncode, out, err + + @classmethod + def install(cls, platform, version): + """Download and install LilyPond version if not cached""" + lilypond_cmd = LilyCmd.with_version(platform, version) + print "\nChecking LilyPond presence" + if lilypond_cmd.installed: + print ("LilyPond {} is already installed in cache," \ + +" continuing with test script.").format( + lilypond_cmd.version) + else: + print "LilyPond {} is not installed yet.".format(version) + print "Downloading and installing now" + sp.check_call( + ["wget", "-O", + cls.lily_install_script, + cls._download_url(platform, version)]) + sp.check_call(["sh", cls.lily_install_script, + "--prefix", + LilyCmd._cache_directory(platform, version), + "--batch"]) + + @classmethod + def clean_cache(cls): + """Clean the cache from versions of Lilypond older than + `cache_cleanup_interval`""" + print "Clean cache\n" + cached = cls._get_cached_versions() + now = datetime.datetime.now() + for lily in cached: + if lily['last_used'] is None: + print 'Removing cached LilyPond', lily['version'],\ + lily['platform'], '(never used)' + shutil.rmtree(lily['directory']) + elif now - lily['last_used'] > cls.cache_cleanup_interval: + print 'Removing cached LilyPond', lily['version'],\ + lily['platform'], '(last used', \ + lily['last_used'].isoformat(), ')' + shutil.rmtree(lily['directory']) + else: + print 'Keeping cached LilyPond', lily['version'],\ + lily['platform'], '(last used', \ + lily['last_used'].isoformat(), ')' + + #################################################################### + # Private members + + @classmethod + def _cache_directory(cls, platform, version): + """Get the cache directory name for the given platform and version""" + return osp.join(cls.cache_root, platform, version) + + def _lilypond_version(self): + """Determine the LilyPond version actually run + by the command self.lily_command""" + lily = sp.Popen([self.command, "--version"], + stdout=sp.PIPE, stderr=sp.PIPE) + version_line = lily.communicate()[0].splitlines()[0] + return re.search(r"\d+\.\d+\.\d+", version_line).group(0) + + def _mark_cache_file(self): + """Get the name of the file that will store the timestamp of the last + time this command has been used.""" + if self.cached: + return osp.join(osp.dirname(self.command), '.oll-last-used') + else: + return None + + def _mark_cache(self): + """Write the timestamp of now in the cache file""" + if self.cached: + with open(self._mark_cache_file(), 'w') as mark_file: + mark_file.write(datetime.datetime.now().isoformat()) + + def _last_used(self): + """Returns the last time this command has been used, or None if it was + never used.""" + if self.cached: + if not osp.isfile(self._mark_cache_file()): + return None + with open(self._mark_cache_file(), 'r') as mark_file: + fcontent = mark_file.readline() + return dateutil.parser.parse(fcontent) + + + @classmethod + def _download_url(cls, lily_platform, lily_version): + """Format a string representing the URL to + download the requested LilyPond distribution""" + return "{}{}/lilypond-{}.{}.sh".format( + cls.binary_site, lily_platform, lily_version, lily_platform) + + + @classmethod + def _get_cached_versions(cls): + """Return a list of dictionaries with the attributes of cached + versions""" + versions = [] + for platform in os.listdir(cls.cache_root): + dname = osp.join(cls.cache_root, platform) + if osp.isdir(dname): + for version in os.listdir(dname): + if osp.isdir(osp.join(dname, version)): + cmd = LilyCmd.with_version(platform, version) + versions.append( + {'platform': platform, + 'version': version, + 'last_used': cmd._last_used(), + 'directory': osp.abspath(osp.join(dname, version))}) + return versions From 1be162340d0b09f25b4ff6b2fe28fd5cbdd3fb20 Mon Sep 17 00:00:00 2001 From: Matteo Ceccarello Date: Sat, 2 May 2015 17:59:37 +0200 Subject: [PATCH 2/3] test: Add dateutil as a Travis-CI dependency The package python-dateutil is used for the management of timestamps in the cache of LilyPond versions --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5b1633f6..cd34a81e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ cache: # Download and install LilyPond (stable and devel) # when not present already (cached between builds) install: + - pip install python-dateutil --user - python ./test/install_lilypond.py # This is the script that will actually run the tests From bf3c8475d3fc8bfd58a705cd00403574a0c26252 Mon Sep 17 00:00:00 2001 From: Matteo Ceccarello Date: Sat, 2 May 2015 18:02:22 +0200 Subject: [PATCH 3/3] test: Do not perform a --user install on Travis-CI When using pip, --user installs are not enabled on Travis-CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd34a81e..27a8af39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ cache: # Download and install LilyPond (stable and devel) # when not present already (cached between builds) install: - - pip install python-dateutil --user + - pip install python-dateutil - python ./test/install_lilypond.py # This is the script that will actually run the tests