allow for internal and external waterbutler urls, docker-compose fix

.docker-compose.env

@@ -8,8 +8,9 @@ API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
-WATERBUTLER_URL=http://192.168.168.167:7777
+WATERBUTLER_URL=http://localhost:7777
+WATERBUTLER_INTERNAL_URL=http://192.168.168.167:7777
 CAS_SERVER_URL=http://192.168.168.167:8080
-MFR_SERVER_URL=http://192.168.168.167:7778
-SHARE_URL=http://192.168.168.167:8000/
+MFR_SERVER_URL=http://localhost:7778
+#SHARE_URL=http://192.168.168.167:8000/
 BROKER_URL=amqp://guest:guest@192.168.168.167:5672/

README-docker-compose.md

@@ -182,6 +182,16 @@ _NOTE: After making changes to `Environment Variables` or `Volume Mounts` (e.g.
     - `Single Instance only`
   - Configure `.docker-compose.env` `<APP>_REMOTE_DEBUG` environment variables to match these settings.
 
+## Application Tests
+- Run All Tests
+  - `$ docker-compose run --rm web invoke test`
+
+- Run OSF Specific Tests
+  - `$ docker-compose run --rm web invoke test_osf`
+
+- Test a Specific Module
+  - `$ docker-compose run --rm web invoke test_module -m tests/test_conferences.py`
+
 ## Managing Container State
 
 Restart a container:

addons/base/models.py

@@ -587,7 +587,7 @@ def _get_fileobj_child_metadata(self, filenode, user, cookie=None, version=None)
             kwargs['cookie'] = cookie
         if version:
             kwargs['version'] = version
-        metadata_url = waterbutler_url_for('metadata', **kwargs)
+        metadata_url = waterbutler_url_for('metadata', _internal=True, **kwargs)
 
         res = requests.get(metadata_url)
         if res.status_code != 200:

tasks/__init__.py

@@ -557,6 +557,7 @@ def test_module(ctx, module=None):
     'tests/test_subjects.py',
     'tests/test_tokens.py',
     'tests/test_webtests.py',
+    'tests/test_utils.py',
 ]
 @task
 def test_osf(ctx):

tests/test_conferences.py

@@ -211,6 +211,7 @@ def test_upload(self, mock_put, mock_get_url):
             'osfstorage',
             '/' + self.attachment.filename,
             self.node,
+            _internal=True,
             user=self.user,
         )
         mock_put.assert_called_with(
@@ -230,6 +231,7 @@ def test_upload_no_file_name(self, mock_put, mock_get_url):
             'osfstorage',
             '/' + settings.MISSING_FILE_NAME,
             self.node,
+            _internal=True,
             user=self.user,
         )
         mock_put.assert_called_with(

tests/test_models.py

@@ -4837,21 +4837,17 @@ def test_enqueue_called(self, enqueue_task):
         assert_equals(task.args[2], False)
         assert_equals(task.args[3], {'title'})
 
+    @mock.patch('website.project.tasks.settings.SHARE_URL', None)
+    @mock.patch('website.project.tasks.settings.SHARE_API_TOKEN', None)
     @mock.patch('website.project.tasks.requests')
     def test_skips_no_settings(self, requests):
-        from website.project.tasks import settings
-        settings.SHARE_URL = None
-        settings.SHARE_API_TOKEN = None
-
         on_node_updated(self.node._id, self.user._id, False, {'is_public'})
         assert_false(requests.post.called)
 
+    @mock.patch('website.project.tasks.settings.SHARE_URL', 'https://share.osf.io')
+    @mock.patch('website.project.tasks.settings.SHARE_API_TOKEN', 'Token')
     @mock.patch('website.project.tasks.requests')
     def test_updates_share(self, requests):
-        from website.project.tasks import settings
-        settings.SHARE_URL = 'https://share.osf.io'
-        settings.SHARE_API_TOKEN = 'Token'
-
         on_node_updated(self.node._id, self.user._id, False, {'is_public'})
 
         kwargs = requests.post.call_args[1]
@@ -4861,12 +4857,10 @@ def test_updates_share(self, requests):
         assert_equals(kwargs['headers']['Authorization'], 'Bearer Token')
         assert_equals(graph[0]['uri'], '{}{}/'.format(settings.DOMAIN, self.node._id))
 
+    @mock.patch('website.project.tasks.settings.SHARE_URL', 'https://share.osf.io')
+    @mock.patch('website.project.tasks.settings.SHARE_API_TOKEN', 'Token')
     @mock.patch('website.project.tasks.requests')
     def test_update_share_correctly(self, requests):
-        from website.project.tasks import settings
-        settings.SHARE_URL = 'https://share.osf.io'
-        settings.SHARE_API_TOKEN = 'Token'
-
         cases = [{
             'is_deleted': False,
             'attrs': {'is_public': True, 'is_deleted': False, 'spam_status': SpamStatus.HAM}

tests/test_registrations/test_archiver.py

@@ -379,6 +379,7 @@ def callback(request, uri, headers):
                 node=self.src,
                 user=self.user,
                 view_only=True,
+                _internal=True,
             )
             httpretty.register_uri(httpretty.GET,
                                    url,

tests/test_utils.py

@@ -11,7 +11,7 @@
 import blinker
 
 from tests.base import OsfTestCase
-from tests.factories import RegistrationFactory
+from osf_tests.factories import RegistrationFactory
 
 from framework.routing import Rule, json_renderer
 from framework.utils import secure_filename
@@ -211,6 +211,18 @@ def test_waterbutler_url_for(self):
         assert_in('path=path', url)
         assert_in('provider=provider', url)
 
+    def test_waterbutler_url_for_internal(self):
+        settings.WATERBUTLER_INTERNAL_URL = 'http://1.2.3.4:7777'
+        with self.app.test_request_context():
+            url = waterbutler_url_for('upload', 'provider', 'path', mock.Mock(_id='_id'), _internal=True)
+
+        assert_not_in(settings.WATERBUTLER_URL, url)
+        assert_in(settings.WATERBUTLER_INTERNAL_URL, url)
+        assert_in('nid=_id', url)
+        assert_in('/file?', url)
+        assert_in('path=path', url)
+        assert_in('provider=provider', url)
+
     def test_waterbutler_url_for_implicit_cookie(self):
         with self.app.test_request_context() as context:
             context.request.cookies = {settings.COOKIE_NAME: 'cookie'}
@@ -388,11 +400,7 @@ def zero_if_not_check(item, check, checkFn=lambda n: n):
 class TestProjectUtils(OsfTestCase):
 
     def set_registered_date(self, reg, date):
-        reg._fields['registered_date'].__set__(
-            reg,
-            date,
-            safe=True
-        )
+        reg.registered_date = date
         reg.save()
 
     def test_get_recent_public_registrations(self):

website/addons/base/__init__.py

@@ -769,6 +769,7 @@ def _get_fileobj_child_metadata(self, filenode, user, cookie=None, version=None)
             kwargs['version'] = version
         metadata_url = waterbutler_url_for(
             'metadata',
+            _internal=True,
             **kwargs
         )
         res = requests.get(metadata_url)

website/addons/base/views.py

@@ -643,11 +643,11 @@ def addon_view_or_download_file(auth, path, provider, **kwargs):
     # TODO clean up these urls and unify what is used as a version identifier
     if request.method == 'HEAD':
         return make_response(('', 200, {
-            'Location': file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier))
+            'Location': file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render'))
         }))
 
     if action == 'download':
-        return redirect(file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier)))
+        return redirect(file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render')))
 
     if action == 'get_guid':
         draft_id = extras.get('draft')

website/archiver/tasks.py

@@ -210,7 +210,7 @@ def archive_addon(addon_short_name, job_pk, stat_result):
     src_provider = src.get_addon(addon_name)
     folder_name = src_provider.archive_folder_name
     cookie = user.get_or_create_cookie()
-    copy_url = settings.WATERBUTLER_URL + '/ops/copy'
+    copy_url = settings.WATERBUTLER_INTERNAL_URL + '/ops/copy'
     if addon_name == 'dataverse':
         # The dataverse API will not differentiate between published and draft files
         # unless expcicitly asked. We need to create seperate folders for published and

website/conferences/utils.py

@@ -79,7 +79,7 @@ def upload_attachment(user, node, attachment):
     attachment.seek(0)
     name = '/' + (attachment.filename or settings.MISSING_FILE_NAME)
     content = attachment.read()
-    upload_url = util.waterbutler_url_for('upload', 'osfstorage', name, node, user=user)
+    upload_url = util.waterbutler_url_for('upload', 'osfstorage', name, node, user=user, _internal=True)
 
     requests.put(
         upload_url,

website/settings/defaults.py

@@ -300,6 +300,7 @@ def parent_dir(path):
 DEFAULT_HMAC_SECRET = 'changeme'
 DEFAULT_HMAC_ALGORITHM = hashlib.sha256
 WATERBUTLER_URL = 'http://localhost:7777'
+WATERBUTLER_INTERNAL_URL = WATERBUTLER_URL
 WATERBUTLER_ADDRS = ['127.0.0.1']
 
 # Test identifier namespaces

website/util/__init__.py

@@ -146,7 +146,7 @@ def is_json_request():
     return content_type and ('application/json' in content_type)
 
 
-def waterbutler_url_for(route, provider, path, node, user=None, **kwargs):
+def waterbutler_url_for(route, provider, path, node, user=None, _internal=False, **kwargs):
     """DEPRECATED Use waterbutler_api_url_for
     Reverse URL lookup for WaterButler routes
     :param str route: The action to preform, upload, download, delete...
@@ -156,7 +156,7 @@ def waterbutler_url_for(route, provider, path, node, user=None, **kwargs):
     :param User user: The user whos cookie will be used or None
     :param dict kwargs: Addition query parameters to be appended
     """
-    url = furl.furl(website_settings.WATERBUTLER_URL)
+    url = furl.furl(website_settings.WATERBUTLER_INTERNAL_URL if _internal else website_settings.WATERBUTLER_URL)
     url.path.segments.append(waterbutler_action_map[route])
 
     url.args.update({
@@ -182,9 +182,9 @@ def waterbutler_url_for(route, provider, path, node, user=None, **kwargs):
     return url.url
 
 
-def waterbutler_api_url_for(node_id, provider, path='/', **kwargs):
+def waterbutler_api_url_for(node_id, provider, path='/', _internal=False, **kwargs):
     assert path.startswith('/'), 'Path must always start with /'
-    url = furl.furl(website_settings.WATERBUTLER_URL)
+    url = furl.furl(website_settings.WATERBUTLER_INTERNAL_URL if _internal else website_settings.WATERBUTLER_URL)
     segments = ['v1', 'resources', node_id, 'providers', provider] + path.split('/')[1:]
     url.path.segments.extend([urllib.quote(x.encode('utf-8')) for x in segments])
     url.args.update(kwargs)