Merge branch 'release/3.0.0'

VERSION

@@ -1 +1 @@
-2.0.1
+3.0.0

development.ini

@@ -6,13 +6,14 @@
 [app:main]
 use = egg:springboard
 
-# These are the paths of the repo folders, either relative to unicore.repos_dir
-# or absolute. The index on Elasticsearch corresponds to the path basename. For
-# that reason, the basename must be lowercase, may not start with _ or contain
-# illegal characters.
+# A repo URL can be a local path, either relative to repos_dir, or absolute.
+# Alternatively, a URL can point to a unicore.distribute repo endpoint.
+# The index on Elasticsearch corresponds to the basename (minus .git or .json suffix).
+# For that reason, the basename must be alphanumeric and lowercase. It may also not
+# start with _.
 # https://github.com/elastic/elasticsearch/issues/6736
 
-unicore.content_repos =
+unicore.content_repo_urls =
     gem-tz
     mama-tz
 ; thumbor.security_key = ''

requirements.txt

@@ -1,4 +1,4 @@
-elastic-git>=1.1.0
+elastic-git>=1.4.0
 unicore.content
 unicore.google
 unicore.distribute

springboard.yaml

@@ -4,6 +4,7 @@ repositories:
   mama-tz: 'https://github.com/universalcore/unicore-cms-content-mama-tz-qa'
 
 # List of models to be loaded and their mappings, if any.
+# NOTE: only necessary if manually setting up Elasticsearch index
 models:
   unicore.content.models.Category:
     properties:

springboard/tasks.py

@@ -0,0 +1,17 @@
+from pyramid_celery import celery_app as app
+
+from elasticgit import EG
+from elasticgit.storage import RemoteStorageManager
+
+from springboard.utils import is_remote_repo_url, parse_repo_name
+
+
+@app.task(ignore_result=True)
+def pull(repo_url, index_prefix=None, es=None):
+    if is_remote_repo_url(repo_url):
+        sm = RemoteStorageManager(repo_url)
+        sm.pull()
+    else:
+        index_prefix = index_prefix or parse_repo_name(repo_url)
+        workspace = EG.workspace(repo_url, index_prefix=index_prefix, es=es)
+        workspace.pull()

springboard/tests/base.py

@@ -70,7 +70,7 @@ def mk_app(self, workspace, ini_config={}, settings={}, main=main,
 
         settings_defaults = {
             'unicore.repos_dir': self.working_dir,
-            'unicore.content_repos': workspace.working_dir,
+            'unicore.content_repo_urls': workspace.working_dir,
         }
         settings_defaults.update(settings)
 

springboard/tests/test_language.py

@@ -9,7 +9,7 @@ def setUp(self):
         self.workspace = self.mk_workspace()
         settings = {
             'unicore.repos_dir': self.working_dir,
-            'unicore.content_repos': self.workspace.working_dir,
+            'unicore.content_repo_urls': self.workspace.working_dir,
             'available_languages': '\n'.join([
                 'eng_GB',
                 'swa_KE',

springboard/tests/test_search.py

@@ -11,7 +11,7 @@ def setUp(self):
         self.workspace = self.mk_workspace()
         settings = {
             'unicore.repos_dir': self.working_dir,
-            'unicore.content_repos': self.workspace.working_dir,
+            'unicore.content_repo_urls': self.workspace.working_dir,
             'available_languages': '\n'.join([
                 'eng_GB',
                 'swa_KE',

springboard/tests/test_tasks.py

@@ -0,0 +1,43 @@
+from pyramid import testing
+
+from mock import patch
+
+from springboard.tests.base import SpringboardTestCase
+from springboard.tasks import pull
+from springboard.utils import repo_url
+
+
+class TestTasks(SpringboardTestCase):
+
+    def setUp(self):
+        self.config = testing.setUp()
+        celery_config = self.mk_configfile({
+            'celery': {
+                'celery_always_eager': 'true'}
+            })
+        self.config.include('pyramid_celery')
+        self.config.configure_celery(celery_config)
+
+    def tearDown(self):
+        testing.tearDown()
+
+    @patch('springboard.tasks.RemoteStorageManager')
+    @patch('springboard.tasks.EG.workspace')
+    def test_pull(self, mocked_workspace, mocked_rsm):
+        local_repo_url = repo_url('repos', 'foo')
+        remote_repo_url = repo_url('repos', 'http://localhost/repos/foo.json')
+        es = {'urls': ['http://host:port']}
+
+        pull.delay(local_repo_url, index_prefix='foo', es=es)
+        mocked_workspace.assert_called_once_with(
+            local_repo_url,
+            index_prefix='foo',
+            es=es)
+        mocked_workspace.pull.assert_called_once()
+        mocked_rsm.assert_not_called()
+
+        mocked_workspace.reset_mock()
+        pull.delay(remote_repo_url, index_prefix='foo', es=es)
+        mocked_rsm.assert_called_once_with(remote_repo_url)
+        mocked_rsm.pull.assert_called_once()
+        mocked_workspace.assert_not_called()

springboard/tests/test_tools.py

@@ -1,5 +1,6 @@
 import os
 import shutil
+import responses
 from ConfigParser import ConfigParser
 from StringIO import StringIO
 
@@ -228,13 +229,11 @@ class TestImportContentTool(SpringboardToolTestCase):
 
     def setUp(self):
         self.workspace = self.mk_workspace()
-
-    def test_import(self):
-        tool = ImportContentTool()
-        tool.stdout = StringIO()
-        config = self.mk_workspace_config(self.workspace)
-        config['repositories'] = {}
-        config['models'] = {
+        self.tool = ImportContentTool()
+        self.tool.stdout = StringIO()
+        self.config = self.mk_workspace_config(self.workspace)
+        self.config['repositories'] = {}
+        self.config['models'] = {
             'elasticgit.tests.base.TestPerson': {
                 'properties': {
                     'name': {
@@ -244,33 +243,68 @@ def test_import(self):
                 }
             }
         }
-
-        ini_config = self.mk_configfile({
+        self.ini_config = self.mk_configfile({
             'app:main': {
-                'unicore.content_repos': '',
+                'unicore.content_repo_urls': '',
             }
         })
+        _, self.yaml_config = self.mk_tempfile()
 
-        _, yaml_config = self.mk_tempfile()
-        tool.run(config=(yaml_config, config),
-                 verbose=True,
-                 clobber=False,
-                 repo_dir=self.working_dir,
-                 repo_url=self.workspace.working_dir,
-                 ini_config=ini_config,
-                 ini_section='app:main',
-                 update_config=True,
-                 repo_name=None)
+    def test_import_local(self):
+        self.tool.run(
+            config=(self.yaml_config, self.config),
+            verbose=True,
+            clobber=False,
+            repo_dir=self.working_dir,
+            repo_url=self.workspace.working_dir,
+            ini_config=self.ini_config,
+            ini_section='app:main',
+            update_config=True,
+            repo_name=None,
+            repo_host=None)
 
         cp = ConfigParser()
-        cp.read(ini_config)
+        cp.read(self.ini_config)
         self.assertEqual(
-            cp.get('app:main', 'unicore.content_repos').strip(),
+            cp.get('app:main', 'unicore.content_repo_urls').strip(),
             os.path.basename(self.workspace.working_dir))
 
-        with open(yaml_config, 'r') as fp:
+        with open(self.yaml_config, 'r') as fp:
             data = yaml.safe_load(fp)
             repo_name = parse_repo_name(self.workspace.working_dir)
             self.assertEqual(data['repositories'], {
                 repo_name: self.workspace.working_dir
             })
+
+    @responses.activate
+    def test_import_remote(self):
+        repo_name = parse_repo_name(self.workspace.working_dir)
+        repo_location = 'http://localhost:8080/repos/%s.json' % repo_name
+        responses.add_callback(
+            responses.POST,
+            'http://localhost:8080/repos.json',
+            callback=lambda _: (301, {'Location': repo_location}, ''))
+
+        self.tool.run(
+            config=(self.yaml_config, self.config),
+            verbose=True,
+            clobber=False,
+            repo_dir=self.working_dir,
+            repo_url=self.workspace.working_dir,
+            ini_config=self.ini_config,
+            ini_section='app:main',
+            update_config=True,
+            repo_name=None,
+            repo_host='http://localhost:8080')
+
+        cp = ConfigParser()
+        cp.read(self.ini_config)
+        self.assertEqual(
+            cp.get('app:main', 'unicore.content_repo_urls').strip(),
+            repo_location)
+
+        with open(self.yaml_config, 'r') as fp:
+            data = yaml.safe_load(fp)
+            self.assertEqual(data['repositories'], {
+                repo_name: self.workspace.working_dir
+            })

springboard/tests/test_utils.py

@@ -1,3 +1,4 @@
+import os
 from unittest import TestCase
 
 from mock import patch
@@ -7,7 +8,8 @@
 
 from springboard.tests.base import SpringboardTestCase
 from springboard.utils import (
-    parse_repo_name, config_list, config_dict, Paginator)
+    parse_repo_name, config_list, config_dict, Paginator,
+    is_remote_repo_url, repo_url, CachingRepoHelper)
 
 
 class TestUtils(TestCase):
@@ -44,6 +46,31 @@ def test_config_dict(self):
         }, config_dict('foo=bar\ngum=tree\nelastic=search\n'))
         self.assertEqual({}, config_dict(''))
 
+    def test_is_remote_repo_url(self):
+        self.assertTrue(is_remote_repo_url('http://domain/repo/foo'))
+        self.assertTrue(is_remote_repo_url('https://domain/repo/foo'))
+        self.assertFalse(is_remote_repo_url('/repos/foo'))
+        self.assertFalse(is_remote_repo_url('foo'))
+
+    def test_repo_url(self):
+        self.assertEqual(repo_url('repos', 'http://domain/repo/foo'),
+                         'http://domain/repo/foo')
+        self.assertEqual(repo_url('repos', 'https://domain/repo/foo'),
+                         'https://domain/repo/foo')
+        self.assertEqual(repo_url('repos', '/bar/foo'), '/bar/foo')
+        self.assertEqual(repo_url('repos', 'foo'),
+                         os.path.abspath('repos/foo'))
+
+    @patch('springboard.utils.RepoHelper.active_branch_name')
+    def test_cachingrepohelper(self, mocked_branch_name):
+        mocked_branch_name.return_value = 'branch-foo'
+        repo = CachingRepoHelper('http://domain/repo/foo')
+        self.assertEqual(repo.active_branch_name(), 'branch-foo')
+        mocked_branch_name.assert_called_once()
+        # check that 2nd call is cached
+        self.assertEqual(repo.active_branch_name(), 'branch-foo')
+        mocked_branch_name.assert_called_once()
+
 
 class TestPaginator(TestCase):
 

springboard/tests/test_views.py

@@ -13,7 +13,7 @@ def setUp(self):
         self.workspace = self.mk_workspace()
         self.config = testing.setUp(settings={
             'unicore.repos_dir': self.working_dir,
-            'unicore.content_repos': self.workspace.working_dir,
+            'unicore.content_repo_urls': self.workspace.working_dir,
         })
 
     def tearDown(self):
@@ -68,26 +68,27 @@ def test_404_page(self):
         self.assertIn('<h1>404 not found</h1>', resp.body)
         self.assertEqual(resp.status_int, 404)
 
-    @mock.patch('unicore.distribute.tasks.fastforward.delay')
+    @mock.patch('springboard.tasks.pull.delay')
     def test_api_notify(self, mock_delay):
         request = self.mk_request()
         request.method = 'POST'
 
         views = CoreViews(request)
         response = views.api_notify()
         mock_delay.assert_called_once()
-        (working_dir, index_prefix), _ = mock_delay.call_args_list[0]
+        args, kwargs = mock_delay.call_args
         self.assertEqual(response, {})
-        self.assertEqual(working_dir, self.workspace.working_dir)
-        self.assertEqual(index_prefix, self.workspace.index_prefix)
+        self.assertEqual(kwargs['repo_url'], self.workspace.working_dir)
+        self.assertEqual(kwargs['index_prefix'], self.workspace.index_prefix)
+        self.assertEqual(kwargs['es'], views.es_settings)
 
     def test_multiple_repos(self):
         workspace1 = self.workspace
         workspace2 = self.mk_workspace(name='test_multiple_repos-2')
         testing.setUp(settings={
             'unicore.repos_dir': self.working_dir,
-            'unicore.content_repos': '\n%s\n%s' % (workspace1.working_dir,
-                                                   workspace2.working_dir),
+            'unicore.content_repo_urls': '\n%s\n%s' % (workspace1.working_dir,
+                                                       workspace2.working_dir),
         })
         views = CoreViews(self.mk_request())
         indexes = map(

springboard/tools/commands/base.py

@@ -36,7 +36,7 @@ class SpringboardToolCommand(ToolCommand):
         CommandArgument(
             '-r', '--repo-dir',
             dest='repo_dir',
-            help='Directory to put repositories in.',
+            help='Local directory to put repositories in.',
             default='repos'),
     )
 

springboard/tools/commands/bootstrap.py

@@ -13,7 +13,8 @@ class BootstrapTool(CloneRepoTool,
                     SyncDataTool):
 
     command_name = 'bootstrap'
-    command_help_text = 'Tools for bootstrapping a new content repository.'
+    command_help_text = (
+        'Tools for bootstrapping new content repositories locally.')
     command_arguments = SpringboardToolCommand.command_arguments
 
     def run(self, config, verbose, clobber, repo_dir):

springboard/tools/commands/clone.py

@@ -10,7 +10,7 @@
 class CloneRepoTool(SpringboardToolCommand):
 
     command_name = 'clone'
-    command_help_text = 'Tools for cloning repositories.'
+    command_help_text = 'Tools for cloning repositories locally.'
     command_arguments = SpringboardToolCommand.command_arguments + (
         CommandArgument(
             '-rn', '--repo-name',

springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/development.ini

@@ -5,7 +5,7 @@
 
 [app:main]
 use = egg:{{cookiecutter.app_name}}
-; unicore.content_repos =
+; unicore.content_repo_urls =
 ; thumbor.security_key = '{{cookiecutter.thumbor_security_key}}'
 
 [celery]

springboard/tools/commands/cookiecutter/startapp/{{cookiecutter.app_name}}/springboard.yaml

@@ -2,6 +2,7 @@
 repositories: {}
 
 # List of models to be loaded and their mappings, if any.
+# NOTE: only necessary if manually setting up Elasticsearch index
 models:
   unicore.content.models.Category:
     properties: