diff --git a/requirements-dev.txt b/requirements-dev.txt index d252e76..869441f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ pytest-cov pep8 pyflakes WebTest==2.0.17 +pytest-xdist diff --git a/bootstrap.yaml b/springboard.yaml similarity index 90% rename from bootstrap.yaml rename to springboard.yaml index 6bf7b89..1bd18f6 100644 --- a/bootstrap.yaml +++ b/springboard.yaml @@ -1,6 +1,6 @@ # List of content repositories to be working with repositories: - - 'https://github.com/universalcore/unicore-cms-content-gem-tanzania' + unicore-cms-content-gem-tanzania: 'https://github.com/universalcore/unicore-cms-content-gem-tanzania' # List of models to be loaded and their mappings, if any. models: diff --git a/springboard/tests/base.py b/springboard/tests/base.py index f156850..d600304 100644 --- a/springboard/tests/base.py +++ b/springboard/tests/base.py @@ -24,7 +24,7 @@ class SpringboardTestCase(TestCase): destroy = 'KEEP_REPO' not in os.environ bootstrap_file = pkg_resources.resource_filename( - 'springboard', 'tests/test_bootstrap.yaml') + 'springboard', 'tests/test_springboard.yaml') working_dir = '.test_repos/' def mk_workspace(self, working_dir=None, diff --git a/springboard/tests/test_bootstrap.yaml b/springboard/tests/test_springboard.yaml similarity index 100% rename from springboard/tests/test_bootstrap.yaml rename to springboard/tests/test_springboard.yaml diff --git a/springboard/tests/test_tools.py b/springboard/tests/test_tools.py index dcf72bc..a23e29c 100644 --- a/springboard/tests/test_tools.py +++ b/springboard/tests/test_tools.py @@ -1,3 +1,4 @@ +import os import shutil from StringIO import StringIO @@ -7,7 +8,21 @@ BootstrapTool) -class TestCloneRepoTool(SpringboardTestCase): +class SpringboardToolTestCase(SpringboardTestCase): + + def mk_workspace_name(self, workspace): + return os.path.basename(workspace.working_dir) + + def mk_workspace_config(self, workspace): + repo_name = self.mk_workspace_name(self.workspace) + return { + 'repositories': { + repo_name: self.workspace.working_dir, + } + } + + +class TestCloneRepoTool(SpringboardToolTestCase): def setUp(self): self.workspace = self.mk_workspace() @@ -17,32 +32,35 @@ def test_clone_repo(self): tool = CloneRepoTool() tool.stdout = StringIO() tool.run( + config=self.mk_workspace_config(self.workspace), verbose=True, clobber=False, - repo_url=self.workspace.working_dir, - repo_dir='%s/test_clone_repo' % (self.working_dir,)) + repo_dir='%s/test_clone_repo' % (self.working_dir,), + repo_name=self.mk_workspace_name(self.workspace)) output = tool.stdout.getvalue() self.assertTrue(output.startswith('Cloning')) self.assertTrue(output.endswith('test_clone_repo.\n')) tool.run( + config=self.mk_workspace_config(self.workspace), verbose=True, clobber=False, - repo_url=self.workspace.working_dir, - repo_dir='%s/test_clone_repo' % (self.working_dir,)) + repo_dir='%s/test_clone_repo' % (self.working_dir,), + repo_name=self.mk_workspace_name(self.workspace)) output = tool.stdout.getvalue() self.assertTrue(output.endswith('already exists, skipping.\n')) tool.run( + config=self.mk_workspace_config(self.workspace), verbose=True, clobber=True, - repo_url=self.workspace.working_dir, - repo_dir='%s/test_clone_repo' % (self.working_dir,)) + repo_dir='%s/test_clone_repo' % (self.working_dir,), + repo_name=self.mk_workspace_name(self.workspace)) output = tool.stdout.getvalue() self.assertTrue(output.endswith('Clobbering existing repository.\n')) -class TestCreateIndex(SpringboardTestCase): +class TestCreateIndex(SpringboardToolTestCase): def setUp(self): self.workspace = self.mk_workspace() @@ -52,22 +70,26 @@ def test_create_index(self): tool = CreateIndexTool() tool.stdout = StringIO() tool.run( + config=self.mk_workspace_config(self.workspace), verbose=True, clobber=False, - repo_dir=self.workspace.working_dir) + repo_dir=self.workspace.working_dir, + repo_name=self.mk_workspace_name(self.workspace)) output = tool.stdout.getvalue() self.assertTrue(output.endswith('Index already exists, skipping.\n')) tool.run( + config=self.mk_workspace_config(self.workspace), verbose=True, clobber=True, - repo_dir=self.workspace.working_dir) + repo_dir=self.workspace.working_dir, + repo_name=self.mk_workspace_name(self.workspace)) output = tool.stdout.getvalue() self.assertTrue( output.endswith('Clobbering existing index.\nIndex created.\n')) -class TestCreateMapping(SpringboardTestCase): +class TestCreateMapping(SpringboardToolTestCase): def setUp(self): self.workspace = self.mk_workspace() @@ -76,28 +98,31 @@ def setUp(self): def test_create_mapping(self): tool = CreateMappingTool() tool.stdout = StringIO() - tool.run(verbose=True, + + config = self.mk_workspace_config(self.workspace) + config['models'] = { + 'elasticgit.tests.base.TestPerson': { + 'properties': { + 'name': { + 'index': 'not_analyzed', + 'type': 'string', + } + } + } + } + + tool.run(config=config, + verbose=True, clobber=False, - config={ - 'models': { - 'elasticgit.tests.base.TestPerson': { - 'properties': { - 'name': { - 'index': 'not_analyzed', - 'type': 'string', - } - } - } - } - }, - repo_dir=self.workspace.working_dir) + repo_dir=self.workspace.working_dir, + repo_name=self.mk_workspace_name(self.workspace)) self.assertEqual( tool.stdout.getvalue(), 'Creating mapping for elasticgit.tests.base.TestPerson.\n' 'Mapping created.\n') -class TestSyncData(SpringboardTestCase): +class TestSyncData(SpringboardToolTestCase): def setUp(self): self.workspace = self.mk_workspace() @@ -106,28 +131,31 @@ def setUp(self): def test_sync_data(self): tool = SyncDataTool() tool.stdout = StringIO() - tool.run(verbose=True, + + config = self.mk_workspace_config(self.workspace) + config['models'] = { + 'elasticgit.tests.base.TestPerson': { + 'properties': { + 'name': { + 'index': 'not_analyzed', + 'type': 'string', + } + } + } + } + + tool.run(config=config, + verbose=True, clobber=False, - config={ - 'models': { - 'elasticgit.tests.base.TestPerson': { - 'properties': { - 'name': { - 'index': 'not_analyzed', - 'type': 'string', - } - } - } - } - }, - repo_dir=self.workspace.working_dir) + repo_dir=self.workspace.working_dir, + repo_name=self.mk_workspace_name(self.workspace)) self.assertEqual( tool.stdout.getvalue(), 'Syncing data for elasticgit.tests.base.TestPerson.\n' 'Data synced.\n') -class TestBootstrapTool(SpringboardTestCase): +class TestBootstrapTool(SpringboardToolTestCase): def setUp(self): self.workspace = self.mk_workspace() @@ -135,25 +163,29 @@ def setUp(self): def test_bootstrap(self): tool = BootstrapTool() tool.stdout = StringIO() - tool.run(verbose=True, + + config = self.mk_workspace_config(self.workspace) + config['models'] = { + 'elasticgit.tests.base.TestPerson': { + 'properties': { + 'name': { + 'index': 'not_analyzed', + 'type': 'string', + } + } + } + } + + tool.run(config=config, + verbose=True, clobber=False, - config={ - 'repositories': [self.workspace.working_dir], - 'models': { - 'elasticgit.tests.base.TestPerson': { - 'properties': { - 'name': { - 'index': 'not_analyzed', - 'type': 'string', - } - } - } - } - }, repo_dir=self.working_dir) + repo_dir=self.working_dir) - output = tool.stdout.getvalue() - self.assertTrue(output.startswith('Cloning')) - self.assertTrue(output.endswith( - 'Destination already exists, skipping.\n' - 'Creating index for master.\n' - 'Index already exists, skipping.\n')) + lines = tool.stdout.getvalue().split('\n') + self.assertTrue(lines[0].startswith('Cloning')) + self.assertTrue( + lines[0].endswith( + '%s.' % (self.mk_workspace_name(self.workspace)))) + self.assertEqual(lines[1], 'Destination already exists, skipping.') + self.assertEqual(lines[2], 'Creating index for master.') + self.assertEqual(lines[3], 'Index already exists, skipping.') diff --git a/springboard/tools.py b/springboard/tools.py index 42ba0df..bfc5d3e 100644 --- a/springboard/tools.py +++ b/springboard/tools.py @@ -11,8 +11,6 @@ from slugify import slugify -from springboard.utils import parse_repo_name - class YAMLFile(object): @@ -24,6 +22,13 @@ def __call__(self, file_name): class SpringboardToolCommand(ToolCommand): command_arguments = ( + CommandArgument( + '-c', '--config', + dest='config', + help='The configuration file to load', + default='springboard.yaml', + type=YAMLFile(), + ), CommandArgument( '-d', '--clobber', dest='clobber', @@ -36,6 +41,11 @@ class SpringboardToolCommand(ToolCommand): help='Verbose output.', default=False, action='store_true'), + CommandArgument( + '-r', '--repo-dir', + dest='repo_dir', + help='Directory to put repositories in.', + default='repos'), ) stdout = sys.stdout @@ -52,26 +62,23 @@ class CloneRepoTool(SpringboardToolCommand): command_help_text = 'Tools for cloning repositories.' command_arguments = SpringboardToolCommand.command_arguments + ( CommandArgument( - 'repo_url', - metavar='repo_url', - help='The URL of the repository to clone.'), - CommandArgument( - '-r', '--repo-dir', - dest='repo_dir', - help='Directory to put repositories in.', - default='repos'), + 'repo_name', + metavar='repo_name', + help='The name of the repository to clone.'), ) - def run(self, verbose, clobber, repo_url, repo_dir): - return self.clone_repo(repo_url, + def run(self, config, verbose, clobber, repo_dir, repo_name): + repo_url = config['repositories'][repo_name] + return self.clone_repo(repo_name, + repo_url, repo_dir=repo_dir, clobber=clobber, verbose=verbose) - def clone_repo(self, repo_url, repo_dir='repos', - clobber=False, verbose=False): + def clone_repo(self, repo_name, repo_url, + repo_dir='repos', clobber=False, verbose=False): self.verbose = verbose - workdir = os.path.join(repo_dir, parse_repo_name(repo_url)) + workdir = os.path.join(repo_dir, repo_name) self.emit('Cloning %s to %s.' % (repo_url, workdir)) if os.path.isdir(workdir) and not clobber: self.emit('Destination already exists, skipping.') @@ -90,13 +97,14 @@ class CreateIndexTool(SpringboardToolCommand): command_help_text = 'Create an Elasticsearch for a repository' command_arguments = SpringboardToolCommand.command_arguments + ( CommandArgument( - 'repo_dir', - metavar='repo_dir', - help='The repository directory'), + 'repo_name', + metavar='repo_name', + help='The repository name'), ) - def run(self, verbose, clobber, repo_dir): - return self.create_index(repo_dir, verbose=verbose, clobber=clobber) + def run(self, config, verbose, clobber, repo_dir, repo_name): + return self.create_index(os.path.join(repo_dir, repo_name), + verbose=verbose, clobber=clobber) def create_index(self, workdir, verbose=False, clobber=False): self.verbose = verbose @@ -125,23 +133,16 @@ class CreateMappingTool(SpringboardToolCommand): command_help_text = 'Upload a mapping for models stored in elastic-git' command_arguments = SpringboardToolCommand.command_arguments + ( CommandArgument( - '-c', '--config', - dest='config', - help='The config file to use for cloning.', - default='bootstrap.yaml', - type=YAMLFile()), - CommandArgument( - '-r', '--repo-dir', - dest='repo_dir', - help='Directory to put repositories in.', - default='repos'), + 'repo_name', + metavar='repo_name', + help='The repository name'), ) - def run(self, verbose, clobber, config, repo_dir): + def run(self, config, verbose, clobber, repo_dir, repo_name): for model_name, mapping in config['models'].items(): model_class = load_class(model_name) - self.create_mapping(repo_dir, model_class, mapping, - verbose=verbose) + self.create_mapping(os.path.join(repo_dir, repo_name), + model_class, mapping, verbose=verbose) def create_mapping(self, repo_dir, model_class, mapping, verbose=False): @@ -159,21 +160,15 @@ class SyncDataTool(SpringboardToolCommand): command_help_text = 'Sync data from a repo with elastic-git' command_arguments = SpringboardToolCommand.command_arguments + ( CommandArgument( - '-c', '--config', - dest='config', - help='The config file to use for cloning.', - default='bootstrap.yaml', - type=YAMLFile()), - CommandArgument( - 'repo_dir', - metavar='repo_dir', - help='The git repository direction to read data from.'), + 'repo_name', + metavar='repo_name', + help='The repository name'), ) - def run(self, verbose, clobber, config, repo_dir): + def run(self, config, verbose, clobber, repo_dir, repo_name): for model_name, mapping in config['models'].items(): model_class = load_class(model_name) - self.sync_data(repo_dir, model_class, + self.sync_data(os.path.join(repo_dir, repo_name), model_class, verbose=verbose, clobber=clobber) @@ -193,26 +188,14 @@ class BootstrapTool(CloneRepoTool, command_name = 'bootstrap' command_help_text = 'Tools for bootstrapping a new content repository.' - command_arguments = SpringboardToolCommand.command_arguments + ( - CommandArgument( - '-c', '--config', - dest='config', - help='The config file to use for cloning.', - default='bootstrap.yaml', - type=YAMLFile()), - CommandArgument( - '-r', '--repo-dir', - dest='repo_dir', - help='Directory to put repositories in.', - default='repos'), - ) - def run(self, verbose, clobber, config, repo_dir): - repos = [self.clone_repo(repo_url, + def run(self, config, verbose, clobber, repo_dir): + repos = [self.clone_repo(repo_name=repo_name, + repo_url=repo_url, repo_dir=repo_dir, clobber=clobber, verbose=verbose) - for repo_url in config['repositories']] + for repo_name, repo_url in config['repositories'].items()] for workdir, _ in repos: index_created = self.create_index(workdir, clobber=clobber,