Merge branch 'feature/files-on-anything' of https://github.com/Center…

…ForOpenScience/osf.io into feature/node-preprint

.coveragerc

@@ -1,11 +1,11 @@
 [run]
-include =
-    api/*
-    framework/*
-    scripts/*
-    website/*
 omit =
     *tests/*
+    */migrations/*
+    */settings/*
+    */templates/*
+    */wsgi.py
+
 [report]
 exclude_lines =
     raise NotImplementedError

.gitignore

@@ -196,3 +196,6 @@ docker-compose.override.yml
 .unison*
 .docker-sync/
 ssl/
+
+# pyenv
+.python-version

.travis.yml

@@ -147,6 +147,9 @@ install:
 script:
     - invoke test_travis_$TEST_BUILD -n 4
 
+after_success:
+    - coveralls || true
+
 before_cache:
   - rm -Rf $HOME/.cache/pip/http
   - rm -f $HOME/.cache/pip/log/debug.log

README.md

@@ -2,6 +2,8 @@
 
 The code for [https://osf.io](https://osf.io).
 
+[![Coverage Status](https://coveralls.io/repos/github/CenterForOpenScience/osf.io/badge.svg)](https://coveralls.io/github/CenterForOpenScience/osf.io)
+
 - `master` Build Status: [![Build Status](https://travis-ci.org/CenterForOpenScience/osf.io.svg?branch=master)](https://travis-ci.org/CenterForOpenScience/osf.io)
 - `develop` Build Status: [![Build Status](https://travis-ci.org/CenterForOpenScience/osf.io.svg?branch=develop)](https://travis-ci.org/CenterForOpenScience/osf.io)
 - Public Repo: https://github.com/CenterForOpenScience/osf.io/

addons/base/views.py

@@ -180,17 +180,17 @@ def check_access(node, auth, action, cas_resp):
 
     # Users attempting to register projects with components might not have
     # `write` permissions for all components. This will result in a 403 for
-    # all `copyto` actions as well as `copyfrom` actions if the component
+    # all `upload` actions as well as `copyfrom` actions if the component
     # in question is not public. To get around this, we have to recursively
     # check the node's parent node to determine if they have `write`
     # permissions up the stack.
     # TODO(hrybacki): is there a way to tell if this is for a registration?
-    # All nodes being registered that receive the `copyto` action will have
+    # All nodes being registered that receive the `upload` action will have
     # `node.is_registration` == True. However, we have no way of telling if
     # `copyfrom` actions are originating from a node being registered.
     # TODO This is raise UNAUTHORIZED for registrations that have not been archived yet
     if isinstance(node, AbstractNode):
-        if action == 'copyfrom' or (action == 'copyto' and node.is_registration):
+        if action == 'copyfrom' or (action == 'upload' and node.is_registration):
             parent = node.parent_node
             while parent:
                 if parent.can_edit(auth):

addons/github/models.py

@@ -231,7 +231,7 @@ def to_json(self, user):
                 'node_has_auth': True,
                 'github_user': self.user or '',
                 'github_repo': self.repo or '',
-                'github_repo_full_name': '{0} / {1}'.format(self.user, self.repo) if (self.user and self.repo) else '',
+                'github_repo_full_name': '{0}/{1}'.format(self.user, self.repo) if (self.user and self.repo) else '',
                 'auth_osf_name': owner.fullname,
                 'auth_osf_url': owner.url,
                 'auth_osf_id': owner._id,

addons/github/templates/github_node_settings.mako

@@ -43,13 +43,9 @@
             <div class="col-md-6 m-b-sm">
                 <select id="githubSelectRepo" class="form-control" ${'disabled' if not is_owner or is_registration else ''}>
                     <option>-----</option>
-                        % if is_owner:
-                            % for repo_name in repo_names:
-                                <option value="${repo_name['path']}" ${'selected' if repo_name == github_repo_full_name else ''}>${repo_name['path']}</option>
-                            % endfor
-                        % else:
-                            <option selected>${github_repo_full_name}</option>
-                        % endif
+                    % for repo_name in repo_names:
+                        <option value="${repo_name['path']}" ${'selected' if repo_name['path'] == github_repo_full_name else ''}>${repo_name['path']}</option>
+                    % endfor
                 </select>
             </div>
 

api/base/settings/defaults.py

@@ -28,6 +28,9 @@
         'HOST': os.environ.get('OSF_DB_HOST', '127.0.0.1'),
         'PORT': os.environ.get('OSF_DB_PORT', '5432'),
         'ATOMIC_REQUESTS': True,
+        'TEST': {
+            'SERIALIZE': False,
+        },
     }
 }
 

framework/auth/cas.py

@@ -293,8 +293,6 @@ def make_response_from_ticket(ticket, service_url):
             from website.util import web_url_for
             # orcid attributes can be marked private and not shared, default to orcid otherwise
             fullname = u'{} {}'.format(cas_resp.attributes.get('given-names', ''), cas_resp.attributes.get('family-name', '')).strip()
-            if not fullname:
-                fullname = external_credential['id']
             # TODO [CAS-27]: Remove Access Token From Service Validation
             user = {
                 'external_id_provider': external_credential['provider'],

framework/auth/forms.py

@@ -64,6 +64,15 @@ def __call__(self, form, field):
     widget=BootstrapTextInput(),
 )
 
+name_field_not_required = TextField(
+    'Full Name',
+    [
+        NoHtmlCharacters(),
+    ],
+    filters=[stripped],
+    widget=BootstrapTextInput(),
+)
+
 email_field = TextField('Email Address',
     [
         validators.Required(message=u'Email address is required'),
@@ -159,6 +168,7 @@ class SignInForm(Form):
 
 
 class ResendConfirmationForm(Form):
+    name = name_field_not_required  # If the user's auth already has a fullname this won't appear.
     email = email_field
     accepted_terms_of_service = BooleanField(
         [

framework/auth/views.py

@@ -921,10 +921,12 @@ def external_login_email_get():
         raise HTTPError(http.UNAUTHORIZED)
 
     external_id_provider = session.data['auth_user_external_id_provider']
+    auth_user_fullname = session.data.get('auth_user_fullname')
 
     return {
         'form': form,
-        'external_id_provider': external_id_provider
+        'external_id_provider': external_id_provider,
+        'auth_user_fullname': auth_user_fullname,
     }
 
 
@@ -941,7 +943,7 @@ def external_login_email_post():
 
     external_id_provider = session.data['auth_user_external_id_provider']
     external_id = session.data['auth_user_external_id']
-    fullname = session.data['auth_user_fullname']
+    fullname = session.data.get('auth_user_fullname') or form.name.data
     service_url = session.data['service_url']
 
     # TODO: @cslzchen use user tags instead of destination
@@ -1042,7 +1044,8 @@ def external_login_email_post():
     # Don't go anywhere
     return {
         'form': form,
-        'external_id_provider': external_id_provider
+        'external_id_provider': external_id_provider,
+        'auth_user_fullname': fullname
     }
 
 

osf/migrations/0084_migrate_node_info_for_target.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # Generated by Django 1.11.4 on 2017-11-13 16:14
-from __future__ import unicode_literals
+from __future__ import unicode_literals, division
 import logging
 
 from django.db import migrations, models, connection
@@ -30,11 +30,14 @@ def set_basefilenode_target(apps, schema_editor):
                     target_object_id=models.F('node_id')
                 )
             )
+            end = offset + BATCHSIZE
+            percent = '{:.1f}%'.format(end / max_pk * 100)
             logger.info(
-                'Updated osf_basefilenode {}-{}/{}'.format(
+                'Updated osf_basefilenode {}-{}/{} ({})'.format(
                     offset,
-                    offset + BATCHSIZE,
+                    end,
                     max_pk,
+                    percent,
                 )
             )
 

osf/migrations/0116_merge_20180706_0901.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-07-06 14:01
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osf', '0115_auto_20180628_1253'),
+        ('osf', '0106_merge_20180531_0919'),
+    ]
+
+    operations = [
+    ]

osf/migrations/0117_set_is_root.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.13 on 2018-07-11 18:05
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def set_is_root(state, *args, **kwargs):
+    OsfStorageFolder = state.get_model('osf', 'osfstoragefolder')
+    OsfStorageFolder.objects.filter(nodesettings__isnull=False, is_root__isnull=True).update(is_root=True)
+
+
+def unset_is_root(state, *args, **kwargs):
+    state.get_model('osf', 'osfstoragefolder').objects.filter(is_root=True).update(is_root=None)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('osf', '0116_merge_20180706_0901'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_is_root, unset_is_root)
+    ]

osf_tests/test_elastic_search.py

@@ -1395,6 +1395,7 @@ def test_quickfiles_files_appear_in_search(self):
         quickfiles_root.append_file('GreenLight.mp3')
         find = query_file('GreenLight.mp3')['results']
         assert_equal(len(find), 1)
+        assert find[0]['node_url'] == '/{}/quickfiles/'.format(quickfiles.creator._id)
 
     def test_qatest_quickfiles_files_not_appear_in_search(self):
         quickfiles = QuickFilesNode.objects.get(creator=self.node.creator)

requirements/dev.txt

@@ -6,6 +6,8 @@
 pytest==3.0.3
 pytest-xdist==1.15.0
 pytest-django==3.2.1
+pytest-cov==2.5.1
+python-coveralls==2.9.1
 nose
 factory-boy==2.10.0
 webtest-plus==0.3.3

scripts/analytics/file_summary.py

@@ -1,12 +1,14 @@
 import django
 django.setup()
 
+from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
 import logging
 from dateutil.parser import parse
 from datetime import datetime, timedelta
 from django.utils import timezone
 
+from osf.models import Node, QuickFilesNode
 from website.app import init_app
 from scripts.analytics.base import SummaryAnalytics
 
@@ -27,12 +29,24 @@ def get_events(self, date):
 
         # Convert to a datetime at midnight for queries and the timestamp
         timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=timezone.utc)
-        query_datetime = timestamp_datetime + timedelta(days=1)
 
         file_qs = OsfStorageFile.objects
+        node_content_type = ContentType.objects.get_for_model(Node)
 
-        public_query = Q(node__is_public=True)
-        private_query = Q(node__is_public=False)
+        quickfiles_query = Q(
+            target_object_id__in=QuickFilesNode.objects.values('id'),
+            target_content_type=ContentType.objects.get_for_model(QuickFilesNode)
+        )
+
+        public_query = (quickfiles_query | Q(
+            target_object_id__in=Node.objects.filter(is_public=True).values('id'),
+            target_content_type=node_content_type
+        ))
+
+        private_query = Q(
+            target_object_id__in=Node.objects.filter(is_public=False).values('id'),
+            target_content_type=node_content_type
+        )
 
         daily_query = Q(created__gte=timestamp_datetime)
 

scripts/osfstorage/usage_audit.py

@@ -13,11 +13,12 @@
 import functools
 
 from collections import defaultdict
+from django.contrib.contenttypes.models import ContentType
 
 import progressbar
 
 from framework.celery_tasks import app as celery_app
-from osf.models import TrashedFile
+from osf.models import TrashedFile, Node
 
 from website import mails
 from website.app import init_app
@@ -58,9 +59,10 @@ def add_to_white_list(gtg):
 
 
 def get_usage(node):
-    vids = [each for each in BaseFileNode.active.filter(provider='osfstorage', node=node).values_list('versions', flat=True) if each]
+    node_content_type = ContentType.objects.get_for_model(Node)
 
-    t_vids = [each for eac in TrashedFile.objects.filter(provider='osfstorage', node=node).values_list('versions', flat=True) if each]
+    vids = [each for each in BaseFileNode.active.filter(provider='osfstorage', target_object_id=node.id, target_content_type=node_content_type).values_list('versions', flat=True) if each]
+    t_vids = [each for eac in TrashedFile.objects.filter(provider='osfstorage', target_object_id=node.id, target_content_type=node_content_type).values_list('versions', flat=True) if each]
 
     usage = sum([v.size or 0 for v in FileVersion.objects.filter(id__in=vids)])
     trashed_usage = sum([v.size or 0 for v in FileVersion.objects.filter(id__in=t_vids)])

scripts/remove_after_use/reindex_quickfiles.py

@@ -0,0 +1,31 @@
+import sys
+
+import progressbar
+from django.core.paginator import Paginator
+
+from website.app import setup_django
+setup_django()
+
+from website.search.search import update_file
+from addons.osfstorage.models import OsfStorageFile
+
+PAGE_SIZE = 100
+
+def reindex_quickfiles(dry):
+    qs = OsfStorageFile.objects.filter(node__type='osf.quickfilesnode').order_by('id')
+    count = qs.count()
+    paginator = Paginator(qs, PAGE_SIZE)
+
+    progress_bar = progressbar.ProgressBar(maxval=count).start()
+    n_processed = 0
+    for page_num in paginator.page_range:
+        page = paginator.page(page_num)
+        for quickfile in page.object_list:
+            if not dry:
+                update_file(quickfile)
+        n_processed += len(page.object_list)
+        progress_bar.update(n_processed)
+
+if __name__ == '__main__':
+    dry = '--dry' in sys.argv
+    reindex_quickfiles(dry=dry)

setup.cfg

@@ -5,8 +5,10 @@
 # E301: expected 1 blank line, found 0
 # E302: expected 2 blank lines, found 0
 # E266: too many leading '#' for block comment
+# E402: module level import not at top of file
+# W503: line break before binary operator
 [flake8]
-ignore = E501,E127,E128,E265,E301,E302,E266,E731,N803,N806
+ignore = E501,E127,E128,E265,E301,E302,E266,E731,N803,N806,E402,W503
 max-line-length = 100
 exclude = .git,.ropeproject,.tox,docs,setup.py,env,venv,models/migrations,tests/*,scripts/*,src,website/uploads/,website/static/**/*,website/settings/*,framework/forms/*,addons/*/tests/*,env,venv,node_modules,admin/static/*,osf/migrations
 

tasks/__init__.py

@@ -289,12 +289,20 @@ def test_module(ctx, module=None, numprocesses=None, nocapture=False, params=Non
     if not numprocesses:
         from multiprocessing import cpu_count
         numprocesses = cpu_count()
+    numprocesses = int(numprocesses)
     # NOTE: Subprocess to compensate for lack of thread safety in the httpretty module.
     # https://github.com/gabrielfalcao/HTTPretty/issues/209#issue-54090252
-    if nocapture:
-        args = []
-    else:
-        args = ['-s']
+    args = [
+        '--cov-report', 'term-missing',
+        '--cov', 'admin',
+        '--cov', 'addons',
+        '--cov', 'api',
+        '--cov', 'framework',
+        '--cov', 'osf',
+        '--cov', 'website',
+    ]
+    if not nocapture:
+        args += ['-s']
     if numprocesses > 1:
         args += ['-n {}'.format(numprocesses), '--max-slave-restart=0']
     modules = [module] if isinstance(module, basestring) else module