move purge_data.py and accessdb.py inside pybossa folder (#797)

* move purge_data.py and accessdb.py inside pybossa folder

* fix unit test

create_bulk_tasks.py

@@ -9,18 +9,16 @@
 """
 
 import argparse
-import pandas as pd
 import json
 from datetime import datetime
-import numpy as np
 from random import randrange
 import logging
 from logging.handlers import TimedRotatingFileHandler
 import sys
 
-from accessdb import AccessDatabase
+from pybossa.accessdb import AccessDatabase
 import psycopg2
-from pybossa.core import db, create_app
+from pybossa.core import create_app
 
 root_logger = logging.getLogger()
 hdlr = TimedRotatingFileHandler("databulk.log", when="D", backupCount=10)

pybossa/jobs.py

@@ -47,7 +47,6 @@
 from pybossa.model.webhook import Webhook
 from pybossa.util import with_cache_disabled, publish_channel, \
     mail_with_enabled_users
-from purge_data import purge_task_data
 
 MINUTE = 60
 IMPORT_TASKS_TIMEOUT = (20 * MINUTE)
@@ -233,7 +232,6 @@ def project_export(_id):
 
 def get_project_jobs(queue):
     """Return a list of jobs based on user type."""
-    from pybossa.core import project_repo
     from pybossa.cache import projects as cached_projects
     timeout = current_app.config.get('TIMEOUT')
     if queue == 'super':
@@ -448,7 +446,6 @@ def warm_cache():  # pragma: no cover
     with app.request_context(DUMMY_ENVIRON):
         import pybossa.cache.projects as cached_projects
         import pybossa.cache.categories as cached_cat
-        import pybossa.cache.project_stats as stats
         from pybossa.util import rank
         from pybossa.core import user_repo
 
@@ -1555,6 +1552,7 @@ def respond_csv():
 def perform_completed_tasks_cleanup():
     from sqlalchemy.sql import text
     from pybossa.core import db
+    from pybossa.purge_data import purge_task_data
 
     valid_days = [days[0] for days in current_app.config.get('COMPLETED_TASK_CLEANUP_DAYS', [(None, None)]) if days[0]]
     if not valid_days:

purge_data.py → pybossa/purge_data.py

@@ -8,10 +8,10 @@
 import sys
 import psycopg2
 
-from accessdb import AccessDatabase
+from pybossa.accessdb import AccessDatabase
 
 root_logger = logging.getLogger()
-hdlr = TimedRotatingFileHandler("purgedata.log", when="D", backupCount=10)
+hdlr = TimedRotatingFileHandler("../purgedata.log", when="D", backupCount=10)
 formatter = logging.Formatter(
     "%(asctime)s %(levelname)s : %(message)s %(module)s %(lineno)s", "%Y-%m-%d %H:%M:%S"
 )

test/test_jobs/__init__.py

@@ -24,8 +24,6 @@
 from nose.tools import assert_raises
 from test import with_context, Test
 from test.factories import ProjectFactory, TaskFactory, TaskRunFactory
-from accessdb import AccessDatabase
-from sqlalchemy.sql import text
 
 
 def jobs():
@@ -252,7 +250,7 @@ def test_completed_tasks_cleanup_scheduled_with_weekly_queue(self, non_contr, in
             assert job['queue'] == "weekly"
 
     @with_context
-    @patch('pybossa.jobs.purge_task_data')
+    @patch('pybossa.purge_data.purge_task_data')
     def test_completed_tasks_cleanup(self, mock_purge_tasks):
         """Test completed_tasks_cleanup deletes tasks qualify for deletion."""
 
@@ -301,7 +299,7 @@ def test_saturday_4pm_date(self):
         assert saturday.strftime("%Y-%m-%d %H:%M:%S") == "2026-01-31 16:00:00"
 
     @with_context
-    @patch('pybossa.jobs.purge_task_data')
+    @patch('pybossa.purge_data.purge_task_data')
     def test_completed_tasks_cleanup_bad_config(self, mock_purge_tasks):
         """Test completed_tasks_cleanup deletes tasks qualify for deletion."""
 
@@ -311,7 +309,7 @@ def test_completed_tasks_cleanup_bad_config(self, mock_purge_tasks):
         assert not mock_purge_tasks.called
 
     @with_context
-    @patch('pybossa.jobs.purge_task_data')
+    @patch('pybossa.purge_data.purge_task_data')
     def test_completed_tasks_cleanup_bad_project_config(self, mock_purge_tasks):
         """Test completed_tasks_cleanup deletes tasks qualify for deletion."""
 
@@ -327,7 +325,7 @@ def test_completed_tasks_cleanup_bad_project_config(self, mock_purge_tasks):
     # archive tables from test db upon testing complete for future test runs to be successful
     # mock_purge_tasks can be removed with task data cleanup and archive happening in actual
     # @with_context
-    # @patch('pybossa.jobs.purge_task_data')
+    # @patch('pybossa.purge_data.purge_task_data')
     # def test_completed_tasks_cleanup(self, mock_purge_tasks):
     #     """Test completed_tasks_cleanup deletes tasks qualify for deletion."""