Make an explicit 'load content' step in bootstrapping.

springboard/tests/test_tools.py

@@ -1,11 +1,16 @@
 import os
 import shutil
+from ConfigParser import ConfigParser
 from StringIO import StringIO
 
+import yaml
+
 from springboard.tests import SpringboardTestCase
+from springboard.utils import parse_repo_name
 from springboard.tools.commands import (
     CloneRepoTool, CreateIndexTool, CreateMappingTool, SyncDataTool,
-    BootstrapTool)
+    BootstrapTool, ImportContentTool)
+from springboard.tools.commands.base import YAMLFile
 
 
 class SpringboardToolTestCase(SpringboardTestCase):
@@ -22,6 +27,19 @@ def mk_workspace_config(self, workspace):
         }
 
 
+class TestYAMLHelper(SpringboardToolTestCase):
+
+    def test_yaml_file(self):
+        fp, file_name = self.mk_tempfile()
+        fp.write('foo: bar\n')
+        fp.close()
+
+        argparse_type = YAMLFile()
+        self.assertEqual(
+            argparse_type(file_name),
+            (file_name, {'foo': 'bar'}))
+
+
 class TestCloneRepoTool(SpringboardToolTestCase):
 
     def setUp(self):
@@ -32,7 +50,8 @@ def test_clone_repo(self):
         tool = CloneRepoTool()
         tool.stdout = StringIO()
         tool.run(
-            config=self.mk_workspace_config(self.workspace),
+            config=('springboard.yaml',
+                    self.mk_workspace_config(self.workspace)),
             verbose=True,
             clobber=False,
             repo_dir='%s/test_clone_repo' % (self.working_dir,),
@@ -42,7 +61,8 @@ def test_clone_repo(self):
         self.assertTrue(output.endswith('test_clone_repo.\n'))
 
         tool.run(
-            config=self.mk_workspace_config(self.workspace),
+            config=('springboard.yaml',
+                    self.mk_workspace_config(self.workspace)),
             verbose=True,
             clobber=False,
             repo_dir='%s/test_clone_repo' % (self.working_dir,),
@@ -51,7 +71,8 @@ def test_clone_repo(self):
         self.assertTrue(output.endswith('already exists, skipping.\n'))
 
         tool.run(
-            config=self.mk_workspace_config(self.workspace),
+            config=('springboard.yaml',
+                    self.mk_workspace_config(self.workspace)),
             verbose=True,
             clobber=True,
             repo_dir='%s/test_clone_repo' % (self.working_dir,),
@@ -70,7 +91,8 @@ def test_create_index(self):
         tool = CreateIndexTool()
         tool.stdout = StringIO()
         tool.run(
-            config=self.mk_workspace_config(self.workspace),
+            config=('springboard.yaml',
+                    self.mk_workspace_config(self.workspace)),
             verbose=True,
             clobber=False,
             repo_dir=self.workspace.working_dir,
@@ -79,7 +101,8 @@ def test_create_index(self):
         self.assertTrue(output.endswith('Index already exists, skipping.\n'))
 
         tool.run(
-            config=self.mk_workspace_config(self.workspace),
+            config=('springboard.yaml',
+                    self.mk_workspace_config(self.workspace)),
             verbose=True,
             clobber=True,
             repo_dir=self.workspace.working_dir,
@@ -111,7 +134,7 @@ def test_create_mapping(self):
             }
         }
 
-        tool.run(config=config,
+        tool.run(config=('springboard.yaml', config),
                  verbose=True,
                  clobber=False,
                  repo_dir=self.workspace.working_dir,
@@ -144,7 +167,7 @@ def test_sync_data(self):
             }
         }
 
-        tool.run(config=config,
+        tool.run(config=('springboard.yaml', config),
                  verbose=True,
                  clobber=False,
                  repo_dir=self.workspace.working_dir,
@@ -176,7 +199,7 @@ def test_bootstrap(self):
             }
         }
 
-        tool.run(config=config,
+        tool.run(config=('springboard.yaml', config),
                  verbose=True,
                  clobber=False,
                  repo_dir=self.working_dir)
@@ -189,3 +212,55 @@ def test_bootstrap(self):
         self.assertEqual(lines[1], 'Destination already exists, skipping.')
         self.assertEqual(lines[2], 'Creating index for master.')
         self.assertEqual(lines[3], 'Index already exists, skipping.')
+
+
+class TestImportContentTool(SpringboardToolTestCase):
+
+    def setUp(self):
+        self.workspace = self.mk_workspace()
+
+    def test_import(self):
+        tool = ImportContentTool()
+        tool.stdout = StringIO()
+        config = self.mk_workspace_config(self.workspace)
+        config['repositories'] = {}
+        config['models'] = {
+            'elasticgit.tests.base.TestPerson': {
+                'properties': {
+                    'name': {
+                        'index': 'not_analyzed',
+                        'type': 'string',
+                    }
+                }
+            }
+        }
+
+        ini_config = self.mk_configfile({
+            'app:main': {
+                'unicore.content_repo_url': '',
+            }
+        })
+
+        _, yaml_config = self.mk_tempfile()
+        tool.run(config=(yaml_config, config),
+                 verbose=True,
+                 clobber=False,
+                 repo_dir=self.working_dir,
+                 repo_url=self.workspace.working_dir,
+                 ini_config=ini_config,
+                 ini_section='app:main',
+                 update_config=True,
+                 repo_name=None)
+
+        cp = ConfigParser()
+        cp.read(ini_config)
+        self.assertEqual(
+            cp.get('app:main', 'unicore.content_repo_url'),
+            self.workspace.working_dir)
+
+        with open(yaml_config, 'r') as fp:
+            data = yaml.safe_load(fp)
+            repo_name = parse_repo_name(self.workspace.working_dir)
+            self.assertEqual(data['repositories'], {
+                repo_name: self.workspace.working_dir
+            })

springboard/tools/commands/__init__.py

@@ -4,6 +4,7 @@
 from springboard.tools.commands.sync import SyncDataTool
 from springboard.tools.commands.bootstrap import BootstrapTool
 from springboard.tools.commands.startapp import StartAppTool
+from springboard.tools.commands.import_content import ImportContentTool
 
 __all__ = [
     'CloneRepoTool',
@@ -12,4 +13,5 @@
     'SyncDataTool',
     'BootstrapTool',
     'StartAppTool',
+    'ImportContentTool',
 ]

springboard/tools/commands/base.py

@@ -8,7 +8,7 @@ class YAMLFile(object):
 
     def __call__(self, file_name):
         with open(file_name, 'r') as fp:
-            return yaml.safe_load(fp)
+            return file_name, yaml.safe_load(fp)
 
 
 class SpringboardToolCommand(ToolCommand):

springboard/tools/commands/bootstrap.py

@@ -17,20 +17,27 @@ class BootstrapTool(CloneRepoTool,
     command_arguments = SpringboardToolCommand.command_arguments
 
     def run(self, config, verbose, clobber, repo_dir):
+        config_file, config_data = config
         repos = [self.clone_repo(repo_name=repo_name,
                                  repo_url=repo_url,
                                  repo_dir=repo_dir,
                                  clobber=clobber,
                                  verbose=verbose)
-                 for repo_name, repo_url in config['repositories'].items()]
+                 for repo_name, repo_url
+                 in config_data['repositories'].items()]
         for workdir, _ in repos:
-            index_created = self.create_index(workdir,
-                                              clobber=clobber,
-                                              verbose=verbose)
-            for model_name, mapping in config.get('models', {}).items():
-                model_class = load_class(model_name)
-                if index_created:
-                    self.create_mapping(workdir, model_class, mapping,
-                                        verbose=verbose)
-                self.sync_data(workdir, model_class,
-                               verbose=verbose, clobber=clobber)
+            self.bootstrap(workdir,
+                           models=config_data.get('models', {}).items(),
+                           clobber=clobber, verbose=verbose)
+
+    def bootstrap(self, workdir, models=(), clobber=False, verbose=False):
+        index_created = self.create_index(workdir,
+                                          clobber=clobber,
+                                          verbose=verbose)
+        for model_name, mapping in models:
+            model_class = load_class(model_name)
+            if index_created:
+                self.create_mapping(workdir, model_class, mapping,
+                                    verbose=verbose)
+            self.sync_data(workdir, model_class,
+                           verbose=verbose, clobber=clobber)

springboard/tools/commands/clone.py

@@ -19,7 +19,8 @@ class CloneRepoTool(SpringboardToolCommand):
     )
 
     def run(self, config, verbose, clobber, repo_dir, repo_name):
-        repo_url = config['repositories'][repo_name]
+        config_file, config_data = config
+        repo_url = config_data['repositories'][repo_name]
         return self.clone_repo(repo_name,
                                repo_url,
                                repo_dir=repo_dir,

springboard/tools/commands/cookiecutter/startapp/hooks/post_gen_project.sh

@@ -1,3 +1,2 @@
 #!/bin/bash
 pip install -e .
-springboard bootstrap -v

springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/development.ini

@@ -5,7 +5,7 @@
 
 [app:main]
 use = egg:{{cookiecutter.app_name}}
-unicore.content_repo_url = {{cookiecutter.unicore_content_repo_url}}
+; unicore.content_repo_url =
 ; thumbor.security_key = '{{cookiecutter.thumbor_security_key}}'
 
 [celery]

springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/springboard.yaml

@@ -1,6 +1,5 @@
 # List of content repositories to be working with
-repositories:
-  {{cookiecutter.unicore_content_repo_name}}: '{{cookiecutter.unicore_content_repo_url}}'
+repositories: {}
 
 # List of models to be loaded and their mappings, if any.
 models:

springboard/tools/commands/import_content.py

@@ -0,0 +1,82 @@
+from ConfigParser import ConfigParser
+
+import yaml
+
+from springboard.utils import parse_repo_name
+from springboard.tools.commands.bootstrap import BootstrapTool
+from springboard.tools.commands.base import (
+    SpringboardToolCommand, CommandArgument)
+
+
+class ImportContentTool(BootstrapTool):
+
+    command_name = 'import'
+    command_help_text = 'Clone and import a content repository locally.'
+    command_arguments = SpringboardToolCommand.command_arguments + (
+        CommandArgument(
+            'repo_url',
+            metavar='repo_url',
+            help='The URL of the Git content repository to clone.'),
+        CommandArgument(
+            '-i', '--ini',
+            dest='ini_config',
+            default='development.ini',
+            help='The paste ini file to update.'),
+        CommandArgument(
+            '-s', '--ini-section',
+            dest='ini_section',
+            default='app:main',
+            help='The paste ini section to update'),
+        CommandArgument(
+            '-u', '--update-config',
+            dest='update_config',
+            default=True,
+            help='Add the repository to the config files?',
+            action='store_false'),
+        CommandArgument(
+            '-n', '--name',
+            dest='repo_name',
+            help='Give the repository a custom name on disk.'),
+    )
+
+    def run(self, config, verbose, clobber, repo_dir, repo_url,
+            ini_config, ini_section, update_config, repo_name):
+        config_file, config_data = config
+        repo_name = repo_name or parse_repo_name(repo_url)
+        workdir, _ = self.clone_repo(repo_name=repo_name,
+                                     repo_url=repo_url,
+                                     repo_dir=repo_dir,
+                                     clobber=clobber,
+                                     verbose=verbose)
+        self.bootstrap(
+            workdir,
+            config_data.get('models', {}).items(),
+            clobber=clobber,
+            verbose=verbose)
+
+        if not update_config:
+            return
+
+        repositories = config_data.setdefault('repositories', {})
+
+        if repo_name not in repositories:
+            repositories[repo_name] = repo_url
+
+            with open(config_file, 'w') as fp:
+                yaml.safe_dump(config_data,
+                               stream=fp, default_flow_style=False)
+            self.emit('Added %s to the %s config file.' % (
+                repo_name, config_file))
+
+        config_key = 'unicore.content_repo_url'
+
+        cp = ConfigParser()
+        cp.read(ini_config)
+        if not cp.has_section(ini_section):
+            cp.add_section(ini_section)
+
+        cp.set(ini_section, config_key, repo_url)
+        with open(ini_config, 'w') as fp:
+            cp.write(fp)
+        self.emit(
+            'Updated unicore.content_repo_url in %s.' % (ini_config,))

springboard/tools/commands/mapping.py

@@ -21,7 +21,8 @@ class CreateMappingTool(SpringboardToolCommand):
     )
 
     def run(self, config, verbose, clobber, repo_dir, repo_name):
-        for model_name, mapping in config.get('models', {}).items():
+        config_file, config_data = config
+        for model_name, mapping in config_data.get('models', {}).items():
             model_class = load_class(model_name)
             self.create_mapping(os.path.join(repo_dir, repo_name),
                                 model_class, mapping, verbose=verbose)

springboard/tools/commands/startapp.py

@@ -3,7 +3,6 @@
 import shutil
 from cookiecutter.main import cookiecutter
 
-from springboard.utils import parse_repo_name
 from springboard.tools.commands.base import (
     SpringboardToolCommand, CommandArgument)
 
@@ -17,11 +16,6 @@ class StartAppTool(SpringboardToolCommand):
             'app_name',
             metavar='app_name',
             help='The name of the application to start.'),
-        CommandArgument(
-            '-r', '--repo-url',
-            dest='unicore_content_repo_url',
-            required=True,
-            help='The content repository to use.'),
         CommandArgument(
             '--security-key',
             dest='thumbor_security_key',
@@ -55,12 +49,6 @@ class StartAppTool(SpringboardToolCommand):
     )
 
     def run(self, **options):
-        if options['unicore_content_repo_url']:
-            options.update({
-                'unicore_content_repo_name': parse_repo_name(
-                    options['unicore_content_repo_url']),
-            })
-
         cookiecutter(
             pkg_resources.resource_filename(
                 'springboard', 'tools/commands/cookiecutter/startapp'),

springboard/tools/commands/sync.py

@@ -21,7 +21,8 @@ class SyncDataTool(SpringboardToolCommand):
     )
 
     def run(self, config, verbose, clobber, repo_dir, repo_name):
-        for model_name, mapping in config.get('models', {}).items():
+        config_file, config_data = config
+        for model_name, mapping in config_data.get('models', {}).items():
             model_class = load_class(model_name)
             self.sync_data(os.path.join(repo_dir, repo_name), model_class,
                            verbose=verbose,