diff --git a/scripts/analytics/addon_snapshot.py b/scripts/analytics/addon_snapshot.py index b4d849f7fbe5..5d23d56ca7e4 100644 --- a/scripts/analytics/addon_snapshot.py +++ b/scripts/analytics/addon_snapshot.py @@ -4,10 +4,10 @@ from modularodm import Q from website.app import init_app -from website.models import Node -from osf.models import OSFUser as User +from osf.models import OSFUser as User, AbstractNode as Node from framework.mongo.utils import paginated from scripts.analytics.base import SnapshotAnalytics +from website.settings import ADDONS_AVAILABLE logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -31,12 +31,12 @@ def get_enabled_authorized_linked(user_settings_list, has_external_account, shor # osfstorage and wiki don't have user_settings, so always assume they're enabled, authorized, linked if short_name == 'osfstorage' or short_name == 'wiki': num_enabled = num_authorized = num_linked = User.find( - Q('is_registered', 'eq', True) & - Q('password', 'ne', None) & - Q('merged_by', 'eq', None) & - Q('date_disabled', 'eq', None) & - Q('date_confirmed', 'ne', None) - ).count() + Q('is_registered', 'eq', True) & + Q('password', 'ne', None) & + Q('merged_by', 'eq', None) & + Q('date_disabled', 'eq', None) & + Q('date_confirmed', 'ne', None) + ).count() elif short_name == 'forward': num_enabled = num_authorized = ForwardNodeSettings.find().count() @@ -72,8 +72,6 @@ def collection_name(self): def get_events(self, date=None): super(AddonSnapshot, self).get_events(date) - from addons.base.models import BaseNodeSettings - from website.settings import ADDONS_AVAILABLE counts = [] addons_available = {k: v for k, v in [(addon.short_name, addon) for addon in ADDONS_AVAILABLE]} diff --git a/scripts/analytics/institution_summary.py b/scripts/analytics/institution_summary.py index c163fc86772a..7a8db782051c 100644 --- a/scripts/analytics/institution_summary.py +++ b/scripts/analytics/institution_summary.py @@ -4,9 +4,8 @@ from dateutil.parser import parse from datetime import datetime, timedelta -from osf.models import OSFUser as User +from osf.models import OSFUser as User, AbstractNode as Node, Institution from website.app import init_app -from website.models import Node, Institution from scripts.analytics.base import SummaryAnalytics @@ -50,7 +49,7 @@ def get_events(self, date): project_public_query = project_query & public_query project_private_query = project_query & private_query count = { - 'institution':{ + 'institution': { 'id': institution._id, 'name': institution.name, }, @@ -58,7 +57,7 @@ def get_events(self, date): 'total': User.find(user_query).count(), }, 'nodes': { - 'total':AbstractNode.find_by_institutions(institution, node_query).count(), + 'total': AbstractNode.find_by_institutions(institution, node_query).count(), 'public': AbstractNode.find_by_institutions(institution, node_public_query).count(), 'private': AbstractNode.find_by_institutions(institution, node_private_query).count(), }, diff --git a/scripts/analytics/node_log_events.py b/scripts/analytics/node_log_events.py index 42f854c85321..20e2531d0095 100644 --- a/scripts/analytics/node_log_events.py +++ b/scripts/analytics/node_log_events.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from website.app import init_app -from website.project.model import NodeLog +from osf.models import NodeLog from framework.mongo.utils import paginated from scripts.analytics.base import EventAnalytics diff --git a/scripts/analytics/user_summary.py b/scripts/analytics/user_summary.py index 7f945c5a86f7..cb724e3f0d9e 100644 --- a/scripts/analytics/user_summary.py +++ b/scripts/analytics/user_summary.py @@ -5,9 +5,8 @@ from modularodm import Q -from osf.models import OSFUser as User +from osf.models import OSFUser as User, NodeLog from website.app import init_app -from website.models import NodeLog from framework.mongo.utils import paginated from scripts.analytics.base import SummaryAnalytics diff --git a/scripts/approve_embargo_terminations.py b/scripts/approve_embargo_terminations.py index 58c402ea79fe..d266f0401e3c 100644 --- a/scripts/approve_embargo_terminations.py +++ b/scripts/approve_embargo_terminations.py @@ -9,17 +9,18 @@ Makes the Embargoed Node and its components public. """ -import datetime import logging -import sys +import django from django.utils import timezone from django.db import transaction from modularodm import Q +django.setup() from framework.celery_tasks import app as celery_app -from website import models, settings +from osf import models +from website import settings from website.app import init_app from scripts import utils as scripts_utils diff --git a/scripts/approve_registrations.py b/scripts/approve_registrations.py index 9a031e5ccf72..aadf91306d5c 100644 --- a/scripts/approve_registrations.py +++ b/scripts/approve_registrations.py @@ -3,16 +3,18 @@ """ import logging -import datetime +import django from django.utils import timezone from django.db import transaction from modularodm import Q +django.setup() from framework.celery_tasks import app as celery_app +from osf import models from website.app import init_app -from website import models, settings +from website import settings from scripts import utils as scripts_utils @@ -61,4 +63,3 @@ def run_main(dry_run=True): if not dry_run: scripts_utils.add_file_logger(logger, __file__) main(dry_run=dry_run) - diff --git a/scripts/embargo_registrations.py b/scripts/embargo_registrations.py index 42c1f23d418e..95ddfebcc152 100644 --- a/scripts/embargo_registrations.py +++ b/scripts/embargo_registrations.py @@ -4,17 +4,18 @@ """ import logging -import datetime +import django from django.utils import timezone from django.db import transaction from modularodm import Q +django.setup() from framework.celery_tasks import app as celery_app from website.app import init_app from website import models, settings -from website.project.model import NodeLog +from osf.models import NodeLog from scripts import utils as scripts_utils diff --git a/scripts/generate_sitemap.py b/scripts/generate_sitemap.py index 2d521f8cd691..f2db7880f41a 100644 --- a/scripts/generate_sitemap.py +++ b/scripts/generate_sitemap.py @@ -4,23 +4,19 @@ import boto3 import datetime import gzip -import math import os import shutil import sys -import urllib import urlparse import xml import django django.setup() -from django.db import transaction import logging from framework import sentry from framework.celery_tasks import app as celery_app -from osf.models import OSFUser, AbstractNode, Registration -from osf.models.preprint_service import PreprintService +from osf.models import OSFUser, AbstractNode, PreprintService from scripts import utils as script_utils from website import settings from website.app import init_app @@ -31,12 +27,12 @@ class Progress(object): def __init__(self, bar_len=50): self.bar_len = bar_len - + def start(self, total, prefix): self.total = total self.count = 0 self.prefix = prefix - + def increment(self, inc=1): self.count += inc filled_len = int(round(self.bar_len * self.count / float(self.total))) @@ -136,7 +132,7 @@ def write_sitemap_index(self): for f in range(self.sitemap_count): sitemap = doc.createElement('sitemap') sitemap_index.appendChild(sitemap) - + loc = doc.createElement('loc') sitemap.appendChild(loc) loc_text = self.doc.createTextNode(urlparse.urljoin(settings.DOMAIN, 'sitemaps/sitemap_{}.xml.gz'.format(str(f)))) @@ -194,7 +190,7 @@ def generate(self): progress.stop() # AbstractNode urls (Nodes and Registrations, no colelctions) - objs = AbstractNode.objects.filter(is_public=True, is_deleted=False, retraction_id__isnull=True).exclude(type="osf.collection") + objs = AbstractNode.objects.filter(is_public=True, is_deleted=False, retraction_id__isnull=True).exclude(type="osf.collection") progress.start(objs.count(), 'NODE: ') for obj in objs.iterator(): try: @@ -221,12 +217,14 @@ def generate(self): # Preprint file urls try: file_config = settings.SITEMAP_PREPRINT_FILE_CONFIG - file_config['loc'] = urlparse.urljoin(settings.DOMAIN, - os.path.join('project', - obj.primary_file.node._id, # Parent node id + file_config['loc'] = urlparse.urljoin( + settings.DOMAIN, + os.path.join( + 'project', + obj.primary_file.node._id, # Parent node id 'files', 'osfstorage', - obj.primary_file._id, # Preprint file deep_url + obj.primary_file._id, # Preprint file deep_url '?action=download' ) ) diff --git a/scripts/osfstorage/files_audit.py b/scripts/osfstorage/files_audit.py index fbd4028af179..b12a7b131884 100644 --- a/scripts/osfstorage/files_audit.py +++ b/scripts/osfstorage/files_audit.py @@ -18,14 +18,13 @@ import pyrax -from modularodm import Q from boto.glacier.layer2 import Layer2 from pyrax.exceptions import NoSuchObject from framework.celery_tasks import app as celery_app from website.app import init_app -from website.files import models +from osf.models import FileVersion from scripts import utils as scripts_utils from scripts.osfstorage import utils as storage_utils @@ -122,12 +121,12 @@ def ensure_backups(version, dry_run): def glacier_targets(): - return models.FileVersion.objects.filter(location__has_key='object', metadata__archive__isnull=True) + return FileVersion.objects.filter(location__has_key='object', metadata__archive__isnull=True) def parity_targets(): # TODO: Add metadata.parity information from wb so we do not need to check remote services - return models.FileVersion.objects.filter(location__has_key='object') + return FileVersion.objects.filter(location__has_key='object') # & metadata__parity__isnull=True diff --git a/scripts/osfstorage/glacier_audit.py b/scripts/osfstorage/glacier_audit.py index b3a2f5b3570d..4b78793f0d4e 100644 --- a/scripts/osfstorage/glacier_audit.py +++ b/scripts/osfstorage/glacier_audit.py @@ -17,7 +17,7 @@ from framework.celery_tasks import app as celery_app from website.app import init_app -from website.files import models +from osf.models import FileVersion from scripts import utils as scripts_utils from scripts.osfstorage import settings as storage_settings @@ -66,7 +66,7 @@ def get_job(vault, job_id=None): def get_targets(date): - return models.FileVersion.find( + return FileVersion.find( Q('date_created', 'lt', date - DELTA_DATE) & Q('status', 'ne', 'cached') & Q('metadata.archive', 'exists', True) & diff --git a/scripts/osfstorage/usage_audit.py b/scripts/osfstorage/usage_audit.py index c011eeeb6b01..d89a188a3626 100644 --- a/scripts/osfstorage/usage_audit.py +++ b/scripts/osfstorage/usage_audit.py @@ -20,16 +20,14 @@ from osf.models import TrashedFile from website import mails -from website.models import User from website.app import init_app -from website.project.model import Node from scripts import utils as scripts_utils # App must be init'd before django models are imported init_app(set_backends=True, routes=False) -from osf.models import StoredFileNode, TrashedFileNode, FileVersion +from osf.models import StoredFileNode, FileVersion, OSFUser as User, AbstractNode as Node logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) diff --git a/scripts/populate_conferences.py b/scripts/populate_conferences.py index 9c251c25896c..ee573f4f8f54 100644 --- a/scripts/populate_conferences.py +++ b/scripts/populate_conferences.py @@ -4,14 +4,15 @@ import os import sys +import django from modularodm import Q from modularodm.exceptions import ModularOdmException +django.setup() -from framework.auth.core import User +from osf.models import Conference, OSFUser as User from website import settings from website.app import init_app -from website.conferences.model import Conference from datetime import datetime diff --git a/scripts/populate_preprint_providers.py b/scripts/populate_preprint_providers.py index deaa972cedea..22cb7ab6a184 100644 --- a/scripts/populate_preprint_providers.py +++ b/scripts/populate_preprint_providers.py @@ -13,7 +13,7 @@ import django django.setup() -from website.models import Subject, PreprintProvider, NodeLicense +from osf.models import Subject, PreprintProvider, NodeLicense logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) diff --git a/scripts/refresh_addon_tokens.py b/scripts/refresh_addon_tokens.py index 9c1cc8623a1a..fd7f235554c9 100644 --- a/scripts/refresh_addon_tokens.py +++ b/scripts/refresh_addon_tokens.py @@ -6,9 +6,11 @@ import time from django.utils import timezone +import django from modularodm import Q from oauthlib.oauth2 import OAuth2Error from dateutil.relativedelta import relativedelta +django.setup() from framework.celery_tasks import app as celery_app @@ -18,7 +20,7 @@ from addons.box.models import Provider as Box from addons.googledrive.models import GoogleDriveProvider from addons.mendeley.models import Mendeley -from website.oauth.models import ExternalAccount +from osf.models import ExternalAccount logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) diff --git a/scripts/retract_registrations.py b/scripts/retract_registrations.py index 1cb6aea95ba4..9dcc5401f3a8 100644 --- a/scripts/retract_registrations.py +++ b/scripts/retract_registrations.py @@ -2,16 +2,18 @@ import logging +import django from django.db import transaction from django.utils import timezone from modularodm import Q +django.setup() from framework.auth import Auth from framework.celery_tasks import app as celery_app from website.app import init_app from website import models, settings -from website.project.model import NodeLog +from osf.models import NodeLog from scripts import utils as scripts_utils diff --git a/scripts/send_queued_mails.py b/scripts/send_queued_mails.py index d519718a7c89..a91f4c42c976 100644 --- a/scripts/send_queued_mails.py +++ b/scripts/send_queued_mails.py @@ -1,13 +1,16 @@ import logging +import django from django.db import transaction from django.utils import timezone from modularodm import Q +django.setup() from framework.celery_tasks import app as celery_app +from osf.models.queued_mail import QueuedMail from website.app import init_app -from website import mails, settings +from website import settings from scripts.utils import add_file_logger @@ -46,7 +49,7 @@ def main(dry_run=True): def find_queued_mails_ready_to_be_sent(): - return mails.QueuedMail.find( + return QueuedMail.find( Q('send_at', 'lt', timezone.now()) & Q('sent_at', 'eq', None) ) @@ -55,7 +58,7 @@ def find_queued_mails_ready_to_be_sent(): def pop_and_verify_mails_for_each_user(user_queue): for user_emails in user_queue.values(): mail = user_emails[0] - mails_past_week = mails.QueuedMail.find( + mails_past_week = QueuedMail.find( Q('user', 'eq', mail.user) & Q('sent_at', 'gt', timezone.now() - settings.WAIT_BETWEEN_MAILS) ) diff --git a/scripts/triggered_mails.py b/scripts/triggered_mails.py index f0e49b5ad273..5ff6e33612ab 100644 --- a/scripts/triggered_mails.py +++ b/scripts/triggered_mails.py @@ -6,8 +6,9 @@ from framework.celery_tasks import app as celery_app from osf.models import OSFUser as User +from osf.models.queued_mail import NO_LOGIN_TYPE, NO_LOGIN, QueuedMail, queue_mail from website.app import init_app -from website import mails, settings +from website import settings from scripts.utils import add_file_logger @@ -22,9 +23,9 @@ def main(dry_run=True): logger.warn('Email of type no_login queued to {0}'.format(user.username)) if not dry_run: with transaction.atomic(): - mails.queue_mail( + queue_mail( to_addr=user.username, - mail=mails.NO_LOGIN, + mail=NO_LOGIN, send_at=timezone.now(), user=user, fullname=user.fullname, @@ -36,7 +37,7 @@ def find_inactive_users_with_no_inactivity_email_sent_or_queued(): (Q('date_last_login', 'lt', timezone.now() - settings.NO_LOGIN_WAIT_TIME) & Q('tags__name', 'ne', 'osf4m')) | (Q('date_last_login', 'lt', timezone.now() - settings.NO_LOGIN_OSF4M_WAIT_TIME) & Q('tags__name', 'eq', 'osf4m')) ) - inactive_emails = mails.QueuedMail.find(Q('email_type', 'eq', mails.NO_LOGIN_TYPE)) + inactive_emails = QueuedMail.find(Q('email_type', 'eq', NO_LOGIN_TYPE)) #This is done to prevent User query returns comparison to User, as equality fails #on datetime fields due to pymongo rounding. Instead here _id is compared. @@ -52,4 +53,3 @@ def run_main(dry_run=True): if not dry_run: add_file_logger(logger, __file__) main(dry_run=dry_run) - diff --git a/website/templates/emails/digest.txt.mako b/website/templates/emails/digest.txt.mako index ce08952d56e4..515d82325c7f 100644 --- a/website/templates/emails/digest.txt.mako +++ b/website/templates/emails/digest.txt.mako @@ -1,9 +1,9 @@ -<% from website.models import Node %> +<% from osf.models import AbstractNode%> <% def print_message(d, indent=0): message = '' for key in d['children']: - message += '\t' * indent + ' - ' + Node.load(key).title + ':' + message += '\t' * indent + ' - ' + AbstractNode.load(key).title + ':' if d['children'][key]['messages']: for m in d['children'][key]['messages']: message += '\n' +'\t' * (indent+1) + ' - '+ m @@ -22,7 +22,7 @@ From the Open Science Framework <%def name="build_message(d, indent=0)"> %for key in d['children']: - ${'\t' * indent + Node.load(key).title + ':'} + ${'\t' * indent + AbstractNode.load(key).title + ':'} %if d['children'][key]['messages']: %for m in d['children'][key]['messages']: ${'\t' * indent + '- ' + m['message'] + ' ' + m['timestamp'].strftime("%H:%M")} diff --git a/website/templates/emails/transactional.html.mako b/website/templates/emails/transactional.html.mako index 481cd07f0d27..5a66b4c9a34b 100644 --- a/website/templates/emails/transactional.html.mako +++ b/website/templates/emails/transactional.html.mako @@ -12,9 +12,9 @@

${node_title} - <% from website.project.model import Node %> - %if Node.load(node_id).parent_node: - in ${Node.load(node_id).parent_node.title} + <% from osf.models import AbstractNode %> + %if AbstractNode.load(node_id).parent_node: + in ${AbstractNode.load(node_id).parent_node.title} %endif