diff --git a/docs/springboard.gif b/docs/springboard.gif index 8cf8e5b..8855a02 100644 Binary files a/docs/springboard.gif and b/docs/springboard.gif differ diff --git a/springboard/tests/test_tools.py b/springboard/tests/test_tools.py index 28fda45..120f78e 100644 --- a/springboard/tests/test_tools.py +++ b/springboard/tests/test_tools.py @@ -1,11 +1,16 @@ import os import shutil +from ConfigParser import ConfigParser from StringIO import StringIO +import yaml + from springboard.tests import SpringboardTestCase +from springboard.utils import parse_repo_name from springboard.tools.commands import ( CloneRepoTool, CreateIndexTool, CreateMappingTool, SyncDataTool, - BootstrapTool) + BootstrapTool, ImportContentTool) +from springboard.tools.commands.base import YAMLFile class SpringboardToolTestCase(SpringboardTestCase): @@ -22,6 +27,19 @@ def mk_workspace_config(self, workspace): } +class TestYAMLHelper(SpringboardToolTestCase): + + def test_yaml_file(self): + fp, file_name = self.mk_tempfile() + fp.write('foo: bar\n') + fp.close() + + argparse_type = YAMLFile() + self.assertEqual( + argparse_type(file_name), + (file_name, {'foo': 'bar'})) + + class TestCloneRepoTool(SpringboardToolTestCase): def setUp(self): @@ -32,7 +50,8 @@ def test_clone_repo(self): tool = CloneRepoTool() tool.stdout = StringIO() tool.run( - config=self.mk_workspace_config(self.workspace), + config=('springboard.yaml', + self.mk_workspace_config(self.workspace)), verbose=True, clobber=False, repo_dir='%s/test_clone_repo' % (self.working_dir,), @@ -42,7 +61,8 @@ def test_clone_repo(self): self.assertTrue(output.endswith('test_clone_repo.\n')) tool.run( - config=self.mk_workspace_config(self.workspace), + config=('springboard.yaml', + self.mk_workspace_config(self.workspace)), verbose=True, clobber=False, repo_dir='%s/test_clone_repo' % (self.working_dir,), @@ -51,7 +71,8 @@ def test_clone_repo(self): self.assertTrue(output.endswith('already exists, skipping.\n')) tool.run( - config=self.mk_workspace_config(self.workspace), + config=('springboard.yaml', + self.mk_workspace_config(self.workspace)), verbose=True, clobber=True, repo_dir='%s/test_clone_repo' % (self.working_dir,), @@ -70,7 +91,8 @@ def test_create_index(self): tool = CreateIndexTool() tool.stdout = StringIO() tool.run( - config=self.mk_workspace_config(self.workspace), + config=('springboard.yaml', + self.mk_workspace_config(self.workspace)), verbose=True, clobber=False, repo_dir=self.workspace.working_dir, @@ -79,7 +101,8 @@ def test_create_index(self): self.assertTrue(output.endswith('Index already exists, skipping.\n')) tool.run( - config=self.mk_workspace_config(self.workspace), + config=('springboard.yaml', + self.mk_workspace_config(self.workspace)), verbose=True, clobber=True, repo_dir=self.workspace.working_dir, @@ -111,7 +134,7 @@ def test_create_mapping(self): } } - tool.run(config=config, + tool.run(config=('springboard.yaml', config), verbose=True, clobber=False, repo_dir=self.workspace.working_dir, @@ -144,7 +167,7 @@ def test_sync_data(self): } } - tool.run(config=config, + tool.run(config=('springboard.yaml', config), verbose=True, clobber=False, repo_dir=self.workspace.working_dir, @@ -176,7 +199,7 @@ def test_bootstrap(self): } } - tool.run(config=config, + tool.run(config=('springboard.yaml', config), verbose=True, clobber=False, repo_dir=self.working_dir) @@ -189,3 +212,55 @@ def test_bootstrap(self): self.assertEqual(lines[1], 'Destination already exists, skipping.') self.assertEqual(lines[2], 'Creating index for master.') self.assertEqual(lines[3], 'Index already exists, skipping.') + + +class TestImportContentTool(SpringboardToolTestCase): + + def setUp(self): + self.workspace = self.mk_workspace() + + def test_import(self): + tool = ImportContentTool() + tool.stdout = StringIO() + config = self.mk_workspace_config(self.workspace) + config['repositories'] = {} + config['models'] = { + 'elasticgit.tests.base.TestPerson': { + 'properties': { + 'name': { + 'index': 'not_analyzed', + 'type': 'string', + } + } + } + } + + ini_config = self.mk_configfile({ + 'app:main': { + 'unicore.content_repo_url': '', + } + }) + + _, yaml_config = self.mk_tempfile() + tool.run(config=(yaml_config, config), + verbose=True, + clobber=False, + repo_dir=self.working_dir, + repo_url=self.workspace.working_dir, + ini_config=ini_config, + ini_section='app:main', + update_config=True, + repo_name=None) + + cp = ConfigParser() + cp.read(ini_config) + self.assertEqual( + cp.get('app:main', 'unicore.content_repo_url'), + self.workspace.working_dir) + + with open(yaml_config, 'r') as fp: + data = yaml.safe_load(fp) + repo_name = parse_repo_name(self.workspace.working_dir) + self.assertEqual(data['repositories'], { + repo_name: self.workspace.working_dir + }) diff --git a/springboard/tools/commands/__init__.py b/springboard/tools/commands/__init__.py index 73c9674..fad6c74 100644 --- a/springboard/tools/commands/__init__.py +++ b/springboard/tools/commands/__init__.py @@ -4,6 +4,7 @@ from springboard.tools.commands.sync import SyncDataTool from springboard.tools.commands.bootstrap import BootstrapTool from springboard.tools.commands.startapp import StartAppTool +from springboard.tools.commands.import_content import ImportContentTool __all__ = [ 'CloneRepoTool', @@ -12,4 +13,5 @@ 'SyncDataTool', 'BootstrapTool', 'StartAppTool', + 'ImportContentTool', ] diff --git a/springboard/tools/commands/base.py b/springboard/tools/commands/base.py index 0ea2b3f..dbde8f3 100644 --- a/springboard/tools/commands/base.py +++ b/springboard/tools/commands/base.py @@ -8,7 +8,7 @@ class YAMLFile(object): def __call__(self, file_name): with open(file_name, 'r') as fp: - return yaml.safe_load(fp) + return file_name, yaml.safe_load(fp) class SpringboardToolCommand(ToolCommand): diff --git a/springboard/tools/commands/bootstrap.py b/springboard/tools/commands/bootstrap.py index bfc5c84..bb0f565 100644 --- a/springboard/tools/commands/bootstrap.py +++ b/springboard/tools/commands/bootstrap.py @@ -17,20 +17,27 @@ class BootstrapTool(CloneRepoTool, command_arguments = SpringboardToolCommand.command_arguments def run(self, config, verbose, clobber, repo_dir): + config_file, config_data = config repos = [self.clone_repo(repo_name=repo_name, repo_url=repo_url, repo_dir=repo_dir, clobber=clobber, verbose=verbose) - for repo_name, repo_url in config['repositories'].items()] + for repo_name, repo_url + in config_data['repositories'].items()] for workdir, _ in repos: - index_created = self.create_index(workdir, - clobber=clobber, - verbose=verbose) - for model_name, mapping in config.get('models', {}).items(): - model_class = load_class(model_name) - if index_created: - self.create_mapping(workdir, model_class, mapping, - verbose=verbose) - self.sync_data(workdir, model_class, - verbose=verbose, clobber=clobber) + self.bootstrap(workdir, + models=config_data.get('models', {}).items(), + clobber=clobber, verbose=verbose) + + def bootstrap(self, workdir, models=(), clobber=False, verbose=False): + index_created = self.create_index(workdir, + clobber=clobber, + verbose=verbose) + for model_name, mapping in models: + model_class = load_class(model_name) + if index_created: + self.create_mapping(workdir, model_class, mapping, + verbose=verbose) + self.sync_data(workdir, model_class, + verbose=verbose, clobber=clobber) diff --git a/springboard/tools/commands/clone.py b/springboard/tools/commands/clone.py index da20501..d6b1832 100644 --- a/springboard/tools/commands/clone.py +++ b/springboard/tools/commands/clone.py @@ -19,7 +19,8 @@ class CloneRepoTool(SpringboardToolCommand): ) def run(self, config, verbose, clobber, repo_dir, repo_name): - repo_url = config['repositories'][repo_name] + config_file, config_data = config + repo_url = config_data['repositories'][repo_name] return self.clone_repo(repo_name, repo_url, repo_dir=repo_dir, diff --git a/springboard/tools/commands/cookiecutter/startapp/hooks/post_gen_project.sh b/springboard/tools/commands/cookiecutter/startapp/hooks/post_gen_project.sh index 889fc8b..f8acc74 100755 --- a/springboard/tools/commands/cookiecutter/startapp/hooks/post_gen_project.sh +++ b/springboard/tools/commands/cookiecutter/startapp/hooks/post_gen_project.sh @@ -1,3 +1,2 @@ #!/bin/bash pip install -e . -springboard bootstrap -v diff --git a/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/development.ini b/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/development.ini index aff8e2b..8867082 100644 --- a/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/development.ini +++ b/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/development.ini @@ -5,7 +5,7 @@ [app:main] use = egg:{{cookiecutter.app_name}} -unicore.content_repo_url = {{cookiecutter.unicore_content_repo_url}} +; unicore.content_repo_url = ; thumbor.security_key = '{{cookiecutter.thumbor_security_key}}' [celery] diff --git a/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/springboard.yaml b/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/springboard.yaml index e71c6b2..9822d09 100644 --- a/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/springboard.yaml +++ b/springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/springboard.yaml @@ -1,6 +1,5 @@ # List of content repositories to be working with -repositories: - {{cookiecutter.unicore_content_repo_name}}: '{{cookiecutter.unicore_content_repo_url}}' +repositories: {} # List of models to be loaded and their mappings, if any. models: diff --git a/springboard/tools/commands/import_content.py b/springboard/tools/commands/import_content.py new file mode 100644 index 0000000..b364416 --- /dev/null +++ b/springboard/tools/commands/import_content.py @@ -0,0 +1,82 @@ +from ConfigParser import ConfigParser + +import yaml + +from springboard.utils import parse_repo_name +from springboard.tools.commands.bootstrap import BootstrapTool +from springboard.tools.commands.base import ( + SpringboardToolCommand, CommandArgument) + + +class ImportContentTool(BootstrapTool): + + command_name = 'import' + command_help_text = 'Clone and import a content repository locally.' + command_arguments = SpringboardToolCommand.command_arguments + ( + CommandArgument( + 'repo_url', + metavar='repo_url', + help='The URL of the Git content repository to clone.'), + CommandArgument( + '-i', '--ini', + dest='ini_config', + default='development.ini', + help='The paste ini file to update.'), + CommandArgument( + '-s', '--ini-section', + dest='ini_section', + default='app:main', + help='The paste ini section to update'), + CommandArgument( + '-u', '--update-config', + dest='update_config', + default=True, + help='Add the repository to the config files?', + action='store_false'), + CommandArgument( + '-n', '--name', + dest='repo_name', + help='Give the repository a custom name on disk.'), + ) + + def run(self, config, verbose, clobber, repo_dir, repo_url, + ini_config, ini_section, update_config, repo_name): + config_file, config_data = config + repo_name = repo_name or parse_repo_name(repo_url) + workdir, _ = self.clone_repo(repo_name=repo_name, + repo_url=repo_url, + repo_dir=repo_dir, + clobber=clobber, + verbose=verbose) + self.bootstrap( + workdir, + config_data.get('models', {}).items(), + clobber=clobber, + verbose=verbose) + + if not update_config: + return + + repositories = config_data.setdefault('repositories', {}) + + if repo_name not in repositories: + repositories[repo_name] = repo_url + + with open(config_file, 'w') as fp: + yaml.safe_dump(config_data, + stream=fp, default_flow_style=False) + self.emit('Added %s to the %s config file.' % ( + repo_name, config_file)) + + config_key = 'unicore.content_repo_url' + + cp = ConfigParser() + cp.read(ini_config) + if not cp.has_section(ini_section): + cp.add_section(ini_section) + + cp.set(ini_section, config_key, repo_url) + with open(ini_config, 'w') as fp: + cp.write(fp) + self.emit( + 'Updated unicore.content_repo_url in %s.' % (ini_config,)) diff --git a/springboard/tools/commands/mapping.py b/springboard/tools/commands/mapping.py index faacedb..bb6b904 100644 --- a/springboard/tools/commands/mapping.py +++ b/springboard/tools/commands/mapping.py @@ -21,7 +21,8 @@ class CreateMappingTool(SpringboardToolCommand): ) def run(self, config, verbose, clobber, repo_dir, repo_name): - for model_name, mapping in config.get('models', {}).items(): + config_file, config_data = config + for model_name, mapping in config_data.get('models', {}).items(): model_class = load_class(model_name) self.create_mapping(os.path.join(repo_dir, repo_name), model_class, mapping, verbose=verbose) diff --git a/springboard/tools/commands/startapp.py b/springboard/tools/commands/startapp.py index 071c60e..a4929b3 100644 --- a/springboard/tools/commands/startapp.py +++ b/springboard/tools/commands/startapp.py @@ -3,7 +3,6 @@ import shutil from cookiecutter.main import cookiecutter -from springboard.utils import parse_repo_name from springboard.tools.commands.base import ( SpringboardToolCommand, CommandArgument) @@ -17,11 +16,6 @@ class StartAppTool(SpringboardToolCommand): 'app_name', metavar='app_name', help='The name of the application to start.'), - CommandArgument( - '-r', '--repo-url', - dest='unicore_content_repo_url', - required=True, - help='The content repository to use.'), CommandArgument( '--security-key', dest='thumbor_security_key', @@ -55,12 +49,6 @@ class StartAppTool(SpringboardToolCommand): ) def run(self, **options): - if options['unicore_content_repo_url']: - options.update({ - 'unicore_content_repo_name': parse_repo_name( - options['unicore_content_repo_url']), - }) - cookiecutter( pkg_resources.resource_filename( 'springboard', 'tools/commands/cookiecutter/startapp'), diff --git a/springboard/tools/commands/sync.py b/springboard/tools/commands/sync.py index bbe9edd..f23f8cf 100644 --- a/springboard/tools/commands/sync.py +++ b/springboard/tools/commands/sync.py @@ -21,7 +21,8 @@ class SyncDataTool(SpringboardToolCommand): ) def run(self, config, verbose, clobber, repo_dir, repo_name): - for model_name, mapping in config.get('models', {}).items(): + config_file, config_data = config + for model_name, mapping in config_data.get('models', {}).items(): model_class = load_class(model_name) self.sync_data(os.path.join(repo_dir, repo_name), model_class, verbose=verbose, diff --git a/springboard/tools/main.py b/springboard/tools/main.py index e86d9e9..ddec651 100644 --- a/springboard/tools/main.py +++ b/springboard/tools/main.py @@ -8,6 +8,7 @@ from springboard.tools.commands import SyncDataTool from springboard.tools.commands import BootstrapTool from springboard.tools.commands import StartAppTool +from springboard.tools.commands import ImportContentTool def get_parser(): # pragma: no cover @@ -21,6 +22,7 @@ def get_parser(): # pragma: no cover add_command(subparsers, CreateMappingTool) add_command(subparsers, SyncDataTool) add_command(subparsers, StartAppTool) + add_command(subparsers, ImportContentTool) return parser