From 21c068e4474d839aa6c25e7a4f5ec29c7a949db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 12:20:32 +0200 Subject: [PATCH 01/42] New migration to support external User IDs. --- .../8ce9b3da799e_add_user_external_id.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 alembic/versions/8ce9b3da799e_add_user_external_id.py diff --git a/alembic/versions/8ce9b3da799e_add_user_external_id.py b/alembic/versions/8ce9b3da799e_add_user_external_id.py new file mode 100644 index 0000000000..6acc3ad643 --- /dev/null +++ b/alembic/versions/8ce9b3da799e_add_user_external_id.py @@ -0,0 +1,24 @@ +"""Add user external ID + +Revision ID: 8ce9b3da799e +Revises: 4f12d8650050 +Create Date: 2016-07-27 12:12:46.392252 + +""" + +# revision identifiers, used by Alembic. +revision = '8ce9b3da799e' +down_revision = '4f12d8650050' + +from alembic import op +import sqlalchemy as sa + +field = 'external_uid' + + +def upgrade(): + op.add_column('task_run', sa.Column(field, sa.String)) + + +def downgrade(): + op.drop_column('task_run', field) From b0a372bbdc4fc672bec2aa8ca0a5ed9a9b35cc1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 12:22:35 +0200 Subject: [PATCH 02/42] Add external_uid column. --- pybossa/model/task_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pybossa/model/task_run.py b/pybossa/model/task_run.py index f0ab5789ed..6715f85215 100644 --- a/pybossa/model/task_run.py +++ b/pybossa/model/task_run.py @@ -47,6 +47,8 @@ class TaskRun(db.Model, DomainObject): finish_time = Column(Text, default=make_timestamp) timeout = Column(Integer) calibration = Column(Integer) + #: External User ID + external_uid = Column(Text) #: Value of the answer. info = Column(JSON) '''General writable field that should be used by clients to record results\ From ca619f53583cbc4f02170f1f1c8b0316308250e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 14:57:23 +0200 Subject: [PATCH 03/42] Allow get a task for an external uid. --- pybossa/api/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pybossa/api/__init__.py b/pybossa/api/__init__.py index f3f1e83540..6f900509e5 100644 --- a/pybossa/api/__init__.py +++ b/pybossa/api/__init__.py @@ -137,9 +137,13 @@ def _retrieve_new_task(project_id): offset = 0 user_id = None if current_user.is_anonymous() else current_user.id user_ip = request.remote_addr if current_user.is_anonymous() else None + external_uid = None + if user_ip is None and user_id is None: + external_uid = request.args.get('external_uid') task = sched.new_task(project_id, project.info.get('sched'), user_id, user_ip, + external_uid, offset) return task From a25c87f98f2701a7b6781c0759346023ff7664eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:01:31 +0200 Subject: [PATCH 04/42] Get external User ID. --- pybossa/api/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pybossa/api/__init__.py b/pybossa/api/__init__.py index 6f900509e5..057fbe90d1 100644 --- a/pybossa/api/__init__.py +++ b/pybossa/api/__init__.py @@ -137,9 +137,7 @@ def _retrieve_new_task(project_id): offset = 0 user_id = None if current_user.is_anonymous() else current_user.id user_ip = request.remote_addr if current_user.is_anonymous() else None - external_uid = None - if user_ip is None and user_id is None: - external_uid = request.args.get('external_uid') + external_uid = request.args.get('external_uid') task = sched.new_task(project_id, project.info.get('sched'), user_id, user_ip, From e9ee22fadbe33ac2a197f504e95305c600502596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:05:26 +0200 Subject: [PATCH 05/42] Add external_uid to the schedulers. --- pybossa/sched.py | 103 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/pybossa/sched.py b/pybossa/sched.py index 9473174914..18592e63e2 100644 --- a/pybossa/sched.py +++ b/pybossa/sched.py @@ -26,7 +26,8 @@ session = db.slave_session -def new_task(project_id, sched, user_id=None, user_ip=None, offset=0): +def new_task(project_id, sched, user_id=None, user_ip=None, + external_uid=None, offset=0): """Get a new task by calling the appropriate scheduler function.""" sched_map = { 'default': get_depth_first_task, @@ -34,10 +35,11 @@ def new_task(project_id, sched, user_id=None, user_ip=None, offset=0): 'depth_first': get_depth_first_task, 'incremental': get_incremental_task} scheduler = sched_map.get(sched, sched_map['default']) - return scheduler(project_id, user_id, user_ip, offset=offset) + return scheduler(project_id, user_id, user_ip, external_uid, offset=offset) -def get_breadth_first_task(project_id, user_id=None, user_ip=None, offset=0): +def get_breadth_first_task(project_id, user_id=None, user_ip=None, + external_uid=None, offset=0): """Get a new task which have the least number of task runs. It excludes the current user. @@ -46,7 +48,7 @@ def get_breadth_first_task(project_id, user_id=None, user_ip=None, offset=0): (this is not a big issue as all it means is that you may end up with some tasks run more than is strictly needed!) """ - if user_id and not user_ip: + if user_id and not user_ip and not external_uid: sql = text(''' SELECT task.id, COUNT(task_run.task_id) AS taskcount FROM task @@ -62,19 +64,35 @@ def get_breadth_first_task(project_id, user_id=None, user_ip=None, offset=0): else: if not user_ip: # pragma: no cover user_ip = '127.0.0.1' - sql = text(''' - SELECT task.id, COUNT(task_run.task_id) AS taskcount - FROM task - LEFT JOIN task_run ON (task.id = task_run.task_id) - WHERE NOT EXISTS - (SELECT 1 FROM task_run WHERE project_id=:project_id AND - user_ip=:user_ip AND task_id=task.id) - AND task.project_id=:project_id AND task.state !='completed' - group by task.id ORDER BY taskcount, id ASC LIMIT 10; - ''') + if user_ip and not external_uid: + sql = text(''' + SELECT task.id, COUNT(task_run.task_id) AS taskcount + FROM task + LEFT JOIN task_run ON (task.id = task_run.task_id) + WHERE NOT EXISTS + (SELECT 1 FROM task_run WHERE project_id=:project_id AND + user_ip=:user_ip AND task_id=task.id) + AND task.project_id=:project_id AND task.state !='completed' + group by task.id ORDER BY taskcount, id ASC LIMIT 10; + ''') + rows = session.execute(sql, + dict(project_id=project_id, user_ip=user_ip)) + + if external_uid and not user_ip: + sql = text(''' + SELECT task.id, COUNT(task_run.task_id) AS taskcount + FROM task + LEFT JOIN task_run ON (task.id = task_run.task_id) + WHERE NOT EXISTS + (SELECT 1 FROM task_run WHERE project_id=:project_id AND + external_uid=:external_uid AND task_id=task.id) + AND task.project_id=:project_id AND task.state !='completed' + group by task.id ORDER BY taskcount, id ASC LIMIT 10; + ''') + rows = session.execute(sql, + dict(project_id=project_id, + external_uid=external_uid)) - rows = session.execute(sql, - dict(project_id=project_id, user_ip=user_ip)) task_ids = [x[0] for x in rows] total_remaining = len(task_ids) - offset if total_remaining <= 0: @@ -82,22 +100,26 @@ def get_breadth_first_task(project_id, user_id=None, user_ip=None, offset=0): return session.query(Task).get(task_ids[offset]) -def get_depth_first_task(project_id, user_id=None, user_ip=None, offset=0): +def get_depth_first_task(project_id, user_id=None, user_ip=None, + external_uid=None, offset=0): """Get a new task for a given project.""" - candidate_task_ids = get_candidate_task_ids(project_id, user_id, user_ip) + candidate_task_ids = get_candidate_task_ids(project_id, user_id, + user_ip, external_uid) total_remaining = len(candidate_task_ids) - offset if total_remaining <= 0: return None return session.query(Task).get(candidate_task_ids[offset]) -def get_incremental_task(project_id, user_id=None, user_ip=None, offset=0): +def get_incremental_task(project_id, user_id=None, user_ip=None, + external_uid=None, offset=0): """Get a new task for a given project with its last given answer. It is an important strategy when dealing with large tasks, as transcriptions. """ - candidate_task_ids = get_candidate_task_ids(project_id, user_id, user_ip) + candidate_task_ids = get_candidate_task_ids(project_id, user_id, user_ip, + external_uid) total_remaining = len(candidate_task_ids) if total_remaining == 0: return None @@ -116,10 +138,12 @@ def get_incremental_task(project_id, user_id=None, user_ip=None, offset=0): return task -def get_candidate_task_ids(project_id, user_id=None, user_ip=None): +def get_candidate_task_ids(project_id, user_id=None, user_ip=None, + external_uid=None): """Get all available tasks for a given project and user.""" rows = None - if user_id and not user_ip: + data = None + if user_id and not user_ip and not external_uid: query = text(''' SELECT id FROM task WHERE NOT EXISTS (SELECT task_id FROM task_run WHERE @@ -129,20 +153,33 @@ def get_candidate_task_ids(project_id, user_id=None, user_ip=None): ORDER BY priority_0 DESC, id ASC LIMIT 10''') rows = session.execute(query, dict(project_id=project_id, user_id=user_id)) + data = [t.id for t in rows] else: if not user_ip: user_ip = '127.0.0.1' - query = text(''' - SELECT id FROM task WHERE NOT EXISTS - (SELECT task_id FROM task_run WHERE - project_id=:project_id AND user_ip=:user_ip - AND task_id=task.id) - AND project_id=:project_id AND state !='completed' - ORDER BY priority_0 DESC, id ASC LIMIT 10''') - rows = session.execute(query, dict(project_id=project_id, - user_ip=user_ip)) - - return [t.id for t in rows] + if user_ip and not external_uid: + query = text(''' + SELECT id FROM task WHERE NOT EXISTS + (SELECT task_id FROM task_run WHERE + project_id=:project_id AND user_ip=:user_ip + AND task_id=task.id) + AND project_id=:project_id AND state !='completed' + ORDER BY priority_0 DESC, id ASC LIMIT 10''') + rows = session.execute(query, dict(project_id=project_id, + user_ip=user_ip)) + data = [t.id for t in rows] + if external_uid and not user_ip: + query = text(''' + SELECT id FROM task WHERE NOT EXISTS + (SELECT task_id FROM task_run WHERE + project_id=:project_id AND external_uid=:external_uid + AND task_id=task.id) + AND project_id=:project_id AND state !='completed' + ORDER BY priority_0 DESC, id ASC LIMIT 10''') + rows = session.execute(query, dict(project_id=project_id, + external_uid=external_uid)) + data = [t.id for t in rows] + return data def sched_variants(): From eddad51c18cba1d62d263b0ef0bc9860048cf9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:27:26 +0200 Subject: [PATCH 06/42] Tests for external User ID. --- test/factories/__init__.py | 2 +- test/factories/taskrun_factory.py | 7 +++++ test/test_sched.py | 45 ++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/test/factories/__init__.py b/test/factories/__init__.py index adf89ccb2d..c8949ad906 100644 --- a/test/factories/__init__.py +++ b/test/factories/__init__.py @@ -62,7 +62,7 @@ def _build(cls, model_class, *args, **kwargs): from blogpost_factory import BlogpostFactory from category_factory import CategoryFactory from task_factory import TaskFactory -from taskrun_factory import TaskRunFactory, AnonymousTaskRunFactory +from taskrun_factory import TaskRunFactory, AnonymousTaskRunFactory, ExternalUidTaskRunFactory from user_factory import UserFactory from auditlog_factory import AuditlogFactory from webhook_factory import WebhookFactory diff --git a/test/factories/taskrun_factory.py b/test/factories/taskrun_factory.py index 2869cd19c2..759f77981a 100644 --- a/test/factories/taskrun_factory.py +++ b/test/factories/taskrun_factory.py @@ -45,3 +45,10 @@ class AnonymousTaskRunFactory(TaskRunFactory): user_id = None user_ip = '127.0.0.1' info = 'yes' + +class ExternalUidTaskRunFactory(TaskRunFactory): + user = None + user_id = None + user_ip = '127.0.0.1' + external_uid = '1xa' + info = 'yes' diff --git a/test/test_sched.py b/test/test_sched.py index a113a966a0..e852f3300c 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -28,7 +28,9 @@ from pybossa.model.user import User from pybossa.model.task_run import TaskRun from pybossa.model.category import Category -from factories import TaskFactory, ProjectFactory, TaskRunFactory, AnonymousTaskRunFactory, UserFactory +from pybossa.core import task_repo +from factories import TaskFactory, ProjectFactory, TaskRunFactory, UserFactory +from factories import AnonymousTaskRunFactory, ExternalUidTaskRunFactory import pybossa @@ -78,6 +80,47 @@ def test_anonymous_02_gets_different_tasks(self): for at in assigned_tasks: assert self.is_unique(at['id'], assigned_tasks), err_msg + @with_context + def test_external_uid_02_gets_different_tasks(self): + """ Test SCHED newtask returns N different Tasks + for a external User ID.""" + assigned_tasks = [] + # Get a Task until scheduler returns None + project = ProjectFactory.create() + tasks = TaskFactory.create_batch(3, project=project, info={}) + url = 'api/project/%s/newtask?external_uid=%s' % (project.id, '1xa') + res = self.app.get(url) + data = json.loads(res.data) + while data.get('info') is not None: + # Save the assigned task + assigned_tasks.append(data) + + task = db.session.query(Task).get(data['id']) + # Submit an Answer for the assigned task + tr = ExternalUidTaskRunFactory.create(project=project, task=task) + res = self.app.get(url) + data = json.loads(res.data) + + # Check if we received the same number of tasks that the available ones + assert len(assigned_tasks) == len(tasks), len(assigned_tasks) + # Check if all the assigned Task.id are equal to the available ones + err_msg = "Assigned Task not found in DB Tasks" + for at in assigned_tasks: + assert self.is_task(at['id'], tasks), err_msg + # Check that there are no duplicated tasks + err_msg = "One Assigned Task is duplicated" + for at in assigned_tasks: + assert self.is_unique(at['id'], assigned_tasks), err_msg + # Check that there are task runs saved with the external UID + answers = task_repo.filter_task_runs_by(external_uid='1xa') + print answers + err_msg = "There should be the same amount of task_runs than tasks" + assert len(answers) == len(assigned_tasks), err_msg + assigned_tasks_ids = sorted([at['id'] for at in assigned_tasks]) + task_run_ids = sorted([a.task_id for a in answers]) + err_msg = "There should be an answer for each assigned task" + assert assigned_tasks_ids == task_run_ids, err_msg + @with_context def test_anonymous_03_respects_limit_tasks(self): """ Test SCHED newtask respects the limit of 30 TaskRuns per Task""" From 71d635c8bbca02317fc2142f1a0d0fb55a809745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:27:52 +0200 Subject: [PATCH 07/42] Only get the external_uid if it's included. --- pybossa/sched.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybossa/sched.py b/pybossa/sched.py index 18592e63e2..a6242a8dc5 100644 --- a/pybossa/sched.py +++ b/pybossa/sched.py @@ -168,7 +168,7 @@ def get_candidate_task_ids(project_id, user_id=None, user_ip=None, rows = session.execute(query, dict(project_id=project_id, user_ip=user_ip)) data = [t.id for t in rows] - if external_uid and not user_ip: + else: query = text(''' SELECT id FROM task WHERE NOT EXISTS (SELECT task_id FROM task_run WHERE From 8665a456edbb9c887957d1db10f318b5dcc7232f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:35:56 +0200 Subject: [PATCH 08/42] New test. --- test/test_sched.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/test_sched.py b/test/test_sched.py index e852f3300c..ba750fd1b7 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -156,6 +156,45 @@ def test_anonymous_03_respects_limit_tasks(self): for tr in t.task_runs: assert self.is_unique(tr.user_ip, t.task_runs), err_msg + @with_context + def test_external_uid_03_respects_limit_tasks(self): + """ Test SCHED newtask respects the limit of 30 TaskRuns per Task for + external user id""" + assigned_tasks = [] + # Get Task until scheduler returns None + url = 'api/project/1/newtask?external_uid=%s' % '1xa' + for i in range(10): + res = self.app.get(url) + data = json.loads(res.data) + + while data.get('info') is not None: + # Check that we received a Task + assert data.get('info'), data + + # Save the assigned task + assigned_tasks.append(data) + + # Submit an Answer for the assigned task + tr = TaskRun(project_id=data['project_id'], task_id=data['id'], + user_ip="127.0.0.1", + external_uid='newUser' + str(i), + info={'answer': 'Yes'}) + db.session.add(tr) + db.session.commit() + res = self.app.get(url) + data = json.loads(res.data) + + # Check if there are 30 TaskRuns per Task + tasks = db.session.query(Task).filter_by(project_id=1).all() + for t in tasks: + assert len(t.task_runs) == 10, len(t.task_runs) + # Check that all the answers are from different IPs + err_msg = "There are two or more Answers from same IP" + for t in tasks: + for tr in t.task_runs: + assert self.is_unique(tr.external_uid, t.task_runs), err_msg + + @with_context def test_user_01_newtask(self): """ Test SCHED newtask returns a Task for John Doe User""" From 430e6d10900692e9169775e5c010348a93a8f364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:36:08 +0200 Subject: [PATCH 09/42] Refactor. --- pybossa/sched.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pybossa/sched.py b/pybossa/sched.py index a6242a8dc5..01b23c637c 100644 --- a/pybossa/sched.py +++ b/pybossa/sched.py @@ -77,8 +77,7 @@ def get_breadth_first_task(project_id, user_id=None, user_ip=None, ''') rows = session.execute(sql, dict(project_id=project_id, user_ip=user_ip)) - - if external_uid and not user_ip: + else: sql = text(''' SELECT task.id, COUNT(task_run.task_id) AS taskcount FROM task From ce46f0f3ba14668c55de0bb9dabf54f45ddb2c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:40:52 +0200 Subject: [PATCH 10/42] New test. --- test/test_sched.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/test/test_sched.py b/test/test_sched.py index ba750fd1b7..2f74af4b6c 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -354,6 +354,57 @@ def test_task_preloading(self): res = self.app.get('api/project/1/newtask?offset=11') assert json.loads(res.data) == {}, res.data + @with_context + def test_task_preloading_external_uid(self): + """Test TASK Pre-loading for external user IDs works""" + # Del previous TaskRuns + self.create() + self.del_task_runs() + + assigned_tasks = [] + # Get Task until scheduler returns None + url = 'api/project/1/newtask?external_uid=2xb' + res = self.app.get(url) + task1 = json.loads(res.data) + # Check that we received a Task + assert task1.get('info'), task1 + # Pre-load the next task for the user + res = self.app.get(url + '&offset=1') + task2 = json.loads(res.data) + # Check that we received a Task + assert task2.get('info'), task2 + # Check that both tasks are different + assert task1.get('id') != task2.get('id'), "Tasks should be different" + ## Save the assigned task + assigned_tasks.append(task1) + assigned_tasks.append(task2) + + # Submit an Answer for the assigned and pre-loaded task + for t in assigned_tasks: + tr = dict(project_id=t['project_id'], + task_id=t['id'], info={'answer': 'No'}, + external_uid='2xb') + tr = json.dumps(tr) + + self.app.post('/api/taskrun', data=tr) + # Get two tasks again + res = self.app.get(url) + task3 = json.loads(res.data) + # Check that we received a Task + assert task3.get('info'), task1 + # Pre-load the next task for the user + res = self.app.get(url + '&offset=1') + task4 = json.loads(res.data) + # Check that we received a Task + assert task4.get('info'), task2 + # Check that both tasks are different + assert task3.get('id') != task4.get('id'), "Tasks should be different" + assert task1.get('id') != task3.get('id'), "Tasks should be different" + assert task2.get('id') != task4.get('id'), "Tasks should be different" + # Check that a big offset returns None + res = self.app.get(url + '&offset=11') + assert json.loads(res.data) == {}, res.data + @with_context def test_task_priority(self): """Test SCHED respects priority_0 field""" From 645949a75a299d37b3f20c6174ad2299654096dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 15:43:19 +0200 Subject: [PATCH 11/42] New tests. --- test/test_sched.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/test_sched.py b/test/test_sched.py index 2f74af4b6c..144eae2bcb 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -440,6 +440,38 @@ def test_task_priority(self): err_msg = "Task.priority_0 should be the 1" assert task1.get('priority_0') == 1, err_msg + @with_context + def test_task_priority_external_uid(self): + """Test SCHED respects priority_0 field for externa uid""" + # Del previous TaskRuns + self.create() + self.del_task_runs() + + # By default, tasks without priority should be ordered by task.id (FIFO) + tasks = db.session.query(Task).filter_by(project_id=1).order_by('id').all() + url = 'api/project/1/newtask?external_uid=342' + res = self.app.get(url) + task1 = json.loads(res.data) + # Check that we received a Task + err_msg = "Task.id should be the same" + assert task1.get('id') == tasks[0].id, err_msg + + # Now let's change the priority to a random task + import random + t = random.choice(tasks) + # Increase priority to maximum + t.priority_0 = 1 + db.session.add(t) + db.session.commit() + # Request again a new task + res = self.app.get(url) + task1 = json.loads(res.data) + # Check that we received a Task + err_msg = "Task.id should be the same" + assert task1.get('id') == t.id, err_msg + err_msg = "Task.priority_0 should be the 1" + assert task1.get('priority_0') == 1, err_msg + def _add_task_run(self, app, task, user=None): tr = AnonymousTaskRunFactory.create(project=app, task=task) From 4e849161e27f98723dd17a3ee691c4615c950dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 16:01:23 +0200 Subject: [PATCH 12/42] Add secret key. --- alembic/versions/8ce9b3da799e_add_user_external_id.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alembic/versions/8ce9b3da799e_add_user_external_id.py b/alembic/versions/8ce9b3da799e_add_user_external_id.py index 6acc3ad643..4137e7384d 100644 --- a/alembic/versions/8ce9b3da799e_add_user_external_id.py +++ b/alembic/versions/8ce9b3da799e_add_user_external_id.py @@ -18,7 +18,9 @@ def upgrade(): op.add_column('task_run', sa.Column(field, sa.String)) + op.add_column('project', sa.Column('secret_key', sa.String)) def downgrade(): op.drop_column('task_run', field) + op.drop_column('project', sa.Column('secret_key', sa.String)) From 7e916a7f59f8903f2c30243760c943951cf62907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 16:01:33 +0200 Subject: [PATCH 13/42] Use JWT for securing projects. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 33c57a75b0..f3250084d6 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,8 @@ "google-api-python-client>=1.5.0, <1.6.0", "Flask-Assets", "jsmin", - "libsass" + "libsass", + "python-jose" ] setup( From 89493679cbb41232edbd42160d2193af7e74658c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 17:21:08 +0200 Subject: [PATCH 14/42] Populate the project secret key with an MD5 secret. --- alembic/versions/8ce9b3da799e_add_user_external_id.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alembic/versions/8ce9b3da799e_add_user_external_id.py b/alembic/versions/8ce9b3da799e_add_user_external_id.py index 4137e7384d..4533991681 100644 --- a/alembic/versions/8ce9b3da799e_add_user_external_id.py +++ b/alembic/versions/8ce9b3da799e_add_user_external_id.py @@ -19,8 +19,10 @@ def upgrade(): op.add_column('task_run', sa.Column(field, sa.String)) op.add_column('project', sa.Column('secret_key', sa.String)) + query = 'update project set secret_key=md5(random()::text);' + op.execute(query) def downgrade(): op.drop_column('task_run', field) - op.drop_column('project', sa.Column('secret_key', sa.String)) + op.drop_column('project', 'secret_key') From b2ec484905dc62f5f06f789be9f7c8f3626fa124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 17:33:03 +0200 Subject: [PATCH 15/42] Don't return secret_key via the API. --- pybossa/api/project.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pybossa/api/project.py b/pybossa/api/project.py index 3d1a1e9259..391b24e294 100644 --- a/pybossa/api/project.py +++ b/pybossa/api/project.py @@ -45,7 +45,8 @@ class ProjectAPI(APIBase): __class__ = Project reserved_keys = set(['id', 'created', 'updated', 'completed', 'contacted', - 'published']) + 'published', 'secret_key']) + private_keys = set(['secret_key']) def _create_instance_from_request(self, data): inst = super(ProjectAPI, self)._create_instance_from_request(data) @@ -71,3 +72,9 @@ def _forbidden_attributes(self, data): if key == 'published': raise Forbidden('You cannot publish a project via the API') raise BadRequest("Reserved keys in payload") + + def _select_attributes(self, data): + for key in self.private_keys: + if data.get(key): + del data[key] + return data From 4c79fa4105329afa5c57f6d6de92518926184a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 17:37:58 +0200 Subject: [PATCH 16/42] Add endpoint to authenticate a project. It uses its secret key. The server returns JWT that can be used to authenticate each API call. --- pybossa/api/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pybossa/api/__init__.py b/pybossa/api/__init__.py index 057fbe90d1..84d575296c 100644 --- a/pybossa/api/__init__.py +++ b/pybossa/api/__init__.py @@ -30,6 +30,7 @@ """ import json +from jose import jwt from flask import Blueprint, request, abort, Response, make_response from flask.ext.login import current_user from werkzeug.exceptions import NotFound @@ -184,3 +185,25 @@ def user_progress(project_id=None, short_name=None): return abort(404) else: # pragma: no cover return abort(404) + + +@jsonpify +@blueprint.route('/auth/project//token') +@crossdomain(origin='*', headers=cors_headers) +@ratelimit(limit=ratelimits.get('LIMIT'), per=ratelimits.get('PER')) +def auth_jwt_project(short_name): + """Create a JWT for a project via its secret KEY.""" + project_secret_key = None + if 'Authorization' in request.headers: + project_secret_key = request.headers.get('Authorization') + if project_secret_key: + project = project_repo.get_by_shortname(short_name) + if project and project.secret_key == project_secret_key: + token = jwt.encode({'short_name': short_name, + 'project_id': project.id}, + project.secret_key, algorithm='HS256') + return token + else: + return abort(404) + else: + return abort(403) From a224562977e8859af3ec2be0f8934a597b25f0d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 27 Jul 2016 17:38:07 +0200 Subject: [PATCH 17/42] Add secret key to projects. --- pybossa/model/project.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pybossa/model/project.py b/pybossa/model/project.py index 2cfc0f9248..d12af5b5a9 100644 --- a/pybossa/model/project.py +++ b/pybossa/model/project.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.mutable import MutableDict from pybossa.core import db, signer -from pybossa.model import DomainObject, make_timestamp +from pybossa.model import DomainObject, make_timestamp, make_uuid from pybossa.model.task import Task from pybossa.model.task_run import TaskRun from pybossa.model.category import Category @@ -58,6 +58,8 @@ class Project(db.Model, DomainObject): published = Column(Boolean, nullable=False, default=False) # If the project is featured featured = Column(Boolean, nullable=False, default=False) + # Secret key for project + secret_key = Column(Text, default=make_uuid) # If the project owner has been emailed contacted = Column(Boolean, nullable=False, default=False) #: Project owner_id From 9c0920054ed24dcb748d74b645df9270dbb8a4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Mon, 1 Aug 2016 12:03:16 +0200 Subject: [PATCH 18/42] Add support to JWT authorization. --- pybossa/api/__init__.py | 9 +++++++- pybossa/auth/__init__.py | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/pybossa/api/__init__.py b/pybossa/api/__init__.py index 84d575296c..ec1bc25352 100644 --- a/pybossa/api/__init__.py +++ b/pybossa/api/__init__.py @@ -30,7 +30,7 @@ """ import json -from jose import jwt +import jwt from flask import Blueprint, request, abort, Response, make_response from flask.ext.login import current_user from werkzeug.exceptions import NotFound @@ -53,6 +53,7 @@ from result import ResultAPI from pybossa.core import project_repo, task_repo from pybossa.contributions_guard import ContributionsGuard +from pybossa.auth import jwt_authorize_project blueprint = Blueprint('api', __name__) @@ -110,6 +111,12 @@ def new_task(project_id): """Return a new task for a project.""" # Check if the request has an arg: try: + if request.args.get('external_uid'): + project = project_repo.get(project_id) + resp = jwt_authorize_project(project, + request.headers.get('Authorization')) + if resp != True: + return resp task = _retrieve_new_task(project_id) # If there is a task for the user, return it if task is not None: diff --git a/pybossa/auth/__init__.py b/pybossa/auth/__init__.py index 0161ba907b..ac08a34ea0 100644 --- a/pybossa/auth/__init__.py +++ b/pybossa/auth/__init__.py @@ -21,6 +21,10 @@ from flask.ext.login import current_user from pybossa.core import task_repo, project_repo, result_repo +import jwt +from flask import jsonify +from jwt import exceptions + import project import task import taskrun @@ -89,3 +93,44 @@ def _authorizer_for(resource_name): if resource_name in ('project', 'task', 'taskrun'): kwargs.update({'result_repo': result_repo}) return _auth_classes[resource_name](**kwargs) + + +def handle_error(error): + """Return authentication error in JSON.""" + resp = jsonify(error) + resp.status_code = 401 + return resp + + +def jwt_authorize_project(project, payload): + """Authorize the project for the payload.""" + try: + if payload is None: + return handle_error({'code': 'invalid_header', + 'description': 'Missing Authorization header'}) + parts = payload.split() + + if parts[0].lower() != 'bearer': + return handle_error({'code': 'invalid_header', + 'description': 'Authorization header \ + must start with Bearer'}) + elif len(parts) == 1: + return handle_error({'code': 'invalid_header', + 'description': 'Token not found'}) + elif len(parts) > 2: + return handle_error({'code': 'invalid_header', + 'description': 'Authorization header must \ + be Bearer + \\s + token'}) + + data = jwt.decode(parts[1], + project.secret_key, + 'H256') + if (data['project_id'] == project.id + and data['short_name'] == project.short_name): + return True + else: + return handle_error({'code': 'Wrong project', + 'description': 'Signature verification failed'}) + except exceptions.DecodeError: + return handle_error({'code': 'Decode error', + 'description': 'Signature verification failed'}) From 725dc2cae27add57756d858141b571b9437e56d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Mon, 1 Aug 2016 12:22:02 +0200 Subject: [PATCH 19/42] Refactor. --- pybossa/api/__init__.py | 20 ++++++++++++++------ pybossa/auth/__init__.py | 1 + 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pybossa/api/__init__.py b/pybossa/api/__init__.py index ec1bc25352..cdf44eddce 100644 --- a/pybossa/api/__init__.py +++ b/pybossa/api/__init__.py @@ -111,13 +111,11 @@ def new_task(project_id): """Return a new task for a project.""" # Check if the request has an arg: try: - if request.args.get('external_uid'): - project = project_repo.get(project_id) - resp = jwt_authorize_project(project, - request.headers.get('Authorization')) - if resp != True: - return resp task = _retrieve_new_task(project_id) + + if type(task) is Response: + return task + # If there is a task for the user, return it if task is not None: guard = ContributionsGuard(sentinel.master) @@ -131,14 +129,24 @@ def new_task(project_id): def _retrieve_new_task(project_id): + project = project_repo.get(project_id) + if project is None: raise NotFound + if not project.allow_anonymous_contributors and current_user.is_anonymous(): info = dict( error="This project does not allow anonymous contributors") error = model.task.Task(info=info) return error + + if request.args.get('external_uid'): + resp = jwt_authorize_project(project, + request.headers.get('Authorization')) + if resp != True: + return resp + if request.args.get('offset'): offset = int(request.args.get('offset')) else: diff --git a/pybossa/auth/__init__.py b/pybossa/auth/__init__.py index ac08a34ea0..e3dd52f5d3 100644 --- a/pybossa/auth/__init__.py +++ b/pybossa/auth/__init__.py @@ -99,6 +99,7 @@ def handle_error(error): """Return authentication error in JSON.""" resp = jsonify(error) resp.status_code = 401 + print error return resp From 0e385a8de94f0c461dcb45373afdf4f4bb93bec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Mon, 1 Aug 2016 14:14:20 +0200 Subject: [PATCH 20/42] Use another jwt library. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f3250084d6..86c1bef985 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ "Flask-Assets", "jsmin", "libsass", - "python-jose" + "pyjwt" ] setup( From 5fc24400324fd79ff9388fc43cfd5a188c7aa26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Mon, 1 Aug 2016 17:14:12 +0200 Subject: [PATCH 21/42] Update tests to use properly JWT tokens. --- test/test_sched.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/test/test_sched.py b/test/test_sched.py index 144eae2bcb..0b2bfb3d94 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -28,7 +28,7 @@ from pybossa.model.user import User from pybossa.model.task_run import TaskRun from pybossa.model.category import Category -from pybossa.core import task_repo +from pybossa.core import task_repo, project_repo from factories import TaskFactory, ProjectFactory, TaskRunFactory, UserFactory from factories import AnonymousTaskRunFactory, ExternalUidTaskRunFactory import pybossa @@ -39,6 +39,18 @@ def setUp(self): super(TestSched, self).setUp() self.endpoints = ['project', 'task', 'taskrun'] + + def get_headers_jwt(self, project): + """Return headesr JWT token.""" + # Get JWT token + url = 'api/auth/project/%s/token' % project.short_name + + res = self.app.get(url, headers={'Authorization': project.secret_key}) + + authorization_token = 'Bearer %s' % res.data + + return {'Authorization': authorization_token} + # Tests @with_context def test_anonymous_01_newtask(self): @@ -88,8 +100,12 @@ def test_external_uid_02_gets_different_tasks(self): # Get a Task until scheduler returns None project = ProjectFactory.create() tasks = TaskFactory.create_batch(3, project=project, info={}) + + headers = self.get_headers_jwt(project) + url = 'api/project/%s/newtask?external_uid=%s' % (project.id, '1xa') - res = self.app.get(url) + + res = self.app.get(url, headers=headers) data = json.loads(res.data) while data.get('info') is not None: # Save the assigned task @@ -98,7 +114,7 @@ def test_external_uid_02_gets_different_tasks(self): task = db.session.query(Task).get(data['id']) # Submit an Answer for the assigned task tr = ExternalUidTaskRunFactory.create(project=project, task=task) - res = self.app.get(url) + res = self.app.get(url, headers=headers) data = json.loads(res.data) # Check if we received the same number of tasks that the available ones @@ -363,13 +379,15 @@ def test_task_preloading_external_uid(self): assigned_tasks = [] # Get Task until scheduler returns None + project = project_repo.get(1) + headers = self.get_headers_jwt(project) url = 'api/project/1/newtask?external_uid=2xb' - res = self.app.get(url) + res = self.app.get(url, headers=headers) task1 = json.loads(res.data) # Check that we received a Task assert task1.get('info'), task1 # Pre-load the next task for the user - res = self.app.get(url + '&offset=1') + res = self.app.get(url + '&offset=1', headers=headers) task2 = json.loads(res.data) # Check that we received a Task assert task2.get('info'), task2 @@ -388,12 +406,12 @@ def test_task_preloading_external_uid(self): self.app.post('/api/taskrun', data=tr) # Get two tasks again - res = self.app.get(url) + res = self.app.get(url, headers=headers) task3 = json.loads(res.data) # Check that we received a Task assert task3.get('info'), task1 # Pre-load the next task for the user - res = self.app.get(url + '&offset=1') + res = self.app.get(url + '&offset=1', headers=headers) task4 = json.loads(res.data) # Check that we received a Task assert task4.get('info'), task2 @@ -402,7 +420,7 @@ def test_task_preloading_external_uid(self): assert task1.get('id') != task3.get('id'), "Tasks should be different" assert task2.get('id') != task4.get('id'), "Tasks should be different" # Check that a big offset returns None - res = self.app.get(url + '&offset=11') + res = self.app.get(url + '&offset=11', headers=headers) assert json.loads(res.data) == {}, res.data @with_context @@ -449,8 +467,10 @@ def test_task_priority_external_uid(self): # By default, tasks without priority should be ordered by task.id (FIFO) tasks = db.session.query(Task).filter_by(project_id=1).order_by('id').all() + project = project_repo.get(1) + headers = self.get_headers_jwt(project) url = 'api/project/1/newtask?external_uid=342' - res = self.app.get(url) + res = self.app.get(url, headers=headers) task1 = json.loads(res.data) # Check that we received a Task err_msg = "Task.id should be the same" @@ -464,7 +484,7 @@ def test_task_priority_external_uid(self): db.session.add(t) db.session.commit() # Request again a new task - res = self.app.get(url) + res = self.app.get(url, headers=headers) task1 = json.loads(res.data) # Check that we received a Task err_msg = "Task.id should be the same" From 097d214c032381e964f7b68f494c466daa809dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 09:44:54 +0200 Subject: [PATCH 22/42] Ensure that TaskRuns with external UID have a valid JWT --- pybossa/api/task_run.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pybossa/api/task_run.py b/pybossa/api/task_run.py index 4be159ac68..fdc111513f 100644 --- a/pybossa/api/task_run.py +++ b/pybossa/api/task_run.py @@ -23,7 +23,7 @@ """ import json -from flask import request +from flask import request, Response from flask.ext.login import current_user from pybossa.model.task_run import TaskRun from werkzeug.exceptions import Forbidden, BadRequest @@ -32,6 +32,7 @@ from pybossa.util import get_user_id_or_ip from pybossa.core import task_repo, sentinel from pybossa.contributions_guard import ContributionsGuard +from pybossa.auth import jwt_authorize_project class TaskRunAPI(APIBase): @@ -61,6 +62,12 @@ def _validate_project_and_task(self, taskrun, task): raise Forbidden('Invalid task_id') if (task.project_id != taskrun.project_id): raise Forbidden('Invalid project_id') + if taskrun.external_uid: + resp = jwt_authorize_project(task.project, + request.headers.get('Authorization')) + msg = json.loads(resp.data)['description'] + if type(resp) == Response: + raise Forbidden(msg) def _ensure_task_was_requested(self, task, guard): if not guard.check_task_stamped(task, get_user_id_or_ip()): From 38b64f7e17d0134d5a60f6e2256c74beb76c7bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 10:04:33 +0200 Subject: [PATCH 23/42] Tests for posting task runs with external UID. --- pybossa/api/task_run.py | 2 +- test/test_sched.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pybossa/api/task_run.py b/pybossa/api/task_run.py index fdc111513f..887b971e06 100644 --- a/pybossa/api/task_run.py +++ b/pybossa/api/task_run.py @@ -65,8 +65,8 @@ def _validate_project_and_task(self, taskrun, task): if taskrun.external_uid: resp = jwt_authorize_project(task.project, request.headers.get('Authorization')) - msg = json.loads(resp.data)['description'] if type(resp) == Response: + msg = json.loads(resp.data)['description'] raise Forbidden(msg) def _ensure_task_was_requested(self, task, guard): diff --git a/test/test_sched.py b/test/test_sched.py index 0b2bfb3d94..196cc7c553 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -404,7 +404,7 @@ def test_task_preloading_external_uid(self): external_uid='2xb') tr = json.dumps(tr) - self.app.post('/api/taskrun', data=tr) + res = self.app.post('/api/taskrun', data=tr, headers=headers) # Get two tasks again res = self.app.get(url, headers=headers) task3 = json.loads(res.data) From 9847c853f31140b1faadfcf495a9549745d75cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 10:50:42 +0200 Subject: [PATCH 24/42] Tests for jwt auth project api endpoint. --- test/test_api/test_jwt.py | 76 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/test_api/test_jwt.py diff --git a/test/test_api/test_jwt.py b/test/test_api/test_jwt.py new file mode 100644 index 0000000000..48273fcbce --- /dev/null +++ b/test/test_api/test_jwt.py @@ -0,0 +1,76 @@ +# -*- coding: utf8 -*- +# This file is part of PyBossa. +# +# Copyright (C) 2016 SciFabric LTD. +# +# PyBossa is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyBossa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with PyBossa. If not, see . +import json +from default import flask_app, with_context +from mock import patch, Mock +from test_api import TestAPI +from factories import ProjectFactory +from pybossa.auth import jwt_authorize_project + + +class TestJwtAPI(TestAPI): + + @with_context + def test_jwt_existing_project(self): + """Test JWT for non existing project works.""" + project = ProjectFactory.create() + url = '/api/auth/project/%s/token' % project.short_name + resp = self.app.get(url) + err_msg = "It should return a 403 as no Authorization headers." + assert resp.status_code == 403, err_msg + + url = '/api/auth/project/nonexisting/token' + resp = self.app.get(url) + err_msg = "It should return a 403 as no Authorization headers." + assert resp.status_code == 403, err_msg + + @with_context + def test_jwt_with_auth_headers(self): + """Test JWT with Auth headers.""" + project = ProjectFactory.create() + headers = {'Authorization': project.secret_key} + url = '/api/auth/project/%s/token' % project.short_name + resp = self.app.get(url, headers=headers) + + err_msg = "It should get the token" + assert resp.status_code == 200, err_msg + bearer = "Bearer %s" % resp.data + data = jwt_authorize_project(project, bearer) + assert data, err_msg + + @with_context + def test_jwt_with_auth_headers_nonproject(self): + """Test JWT with Auth headers but no project.""" + project = ProjectFactory.create() + headers = {'Authorization': project.secret_key} + url = '/api/auth/project/nnon/token' + resp = self.app.get(url, headers=headers) + + err_msg = "It should return 404 as project does not exist" + assert resp.status_code == 404, err_msg + + @with_context + def test_jwt_with_auth_headers_wrong_secret(self): + """Test JWT with Auth headers but wrong project secret.""" + project = ProjectFactory.create() + headers = {'Authorization': 'foobar'} + url = '/api/auth/project/%s/token' + resp = self.app.get(url, headers=headers) + + err_msg = "It should return 404 as project does not exist" + assert resp.status_code == 404, err_msg From 964fc986b240a1b594e5aa4e7945e18528ba54ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 11:48:06 +0200 Subject: [PATCH 25/42] Tests for jwt authentication method. --- test/test_authentication.py | 82 ++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/test/test_authentication.py b/test/test_authentication.py index cc10ddf522..6fed7fa08a 100644 --- a/test/test_authentication.py +++ b/test/test_authentication.py @@ -16,7 +16,12 @@ # You should have received a copy of the GNU Affero General Public License # along with PyBossa. If not, see . -from default import Test +import jwt +from default import Test, with_context +from pybossa.auth import jwt_authorize_project +from pybossa.auth.errcodes import * +from factories import ProjectFactory +from mock import patch class TestAuthentication(Test): @@ -27,3 +32,78 @@ def test_api_authenticate(self): res = self.app.get('/?api_key=%s' % self.api_key) assert ' Date: Tue, 2 Aug 2016 11:48:16 +0200 Subject: [PATCH 26/42] Refactor. --- pybossa/auth/__init__.py | 22 +++++++--------------- pybossa/auth/errcodes.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 pybossa/auth/errcodes.py diff --git a/pybossa/auth/__init__.py b/pybossa/auth/__init__.py index e3dd52f5d3..690acd10f8 100644 --- a/pybossa/auth/__init__.py +++ b/pybossa/auth/__init__.py @@ -20,6 +20,7 @@ from flask import abort from flask.ext.login import current_user from pybossa.core import task_repo, project_repo, result_repo +from pybossa.auth.errcodes import * import jwt from flask import jsonify @@ -99,7 +100,6 @@ def handle_error(error): """Return authentication error in JSON.""" resp = jsonify(error) resp.status_code = 401 - print error return resp @@ -107,21 +107,15 @@ def jwt_authorize_project(project, payload): """Authorize the project for the payload.""" try: if payload is None: - return handle_error({'code': 'invalid_header', - 'description': 'Missing Authorization header'}) + return handle_error(INVALID_HEADER_MISSING) parts = payload.split() if parts[0].lower() != 'bearer': - return handle_error({'code': 'invalid_header', - 'description': 'Authorization header \ - must start with Bearer'}) + return handle_error(INVALID_HEADER_BEARER) elif len(parts) == 1: - return handle_error({'code': 'invalid_header', - 'description': 'Token not found'}) + return handle_error(INVALID_HEADER_TOKEN) elif len(parts) > 2: - return handle_error({'code': 'invalid_header', - 'description': 'Authorization header must \ - be Bearer + \\s + token'}) + return handle_error(INVALID_HEADER_BEARER_TOKEN) data = jwt.decode(parts[1], project.secret_key, @@ -130,8 +124,6 @@ def jwt_authorize_project(project, payload): and data['short_name'] == project.short_name): return True else: - return handle_error({'code': 'Wrong project', - 'description': 'Signature verification failed'}) + return handle_error(WRONG_PROJECT_SIGNATURE) except exceptions.DecodeError: - return handle_error({'code': 'Decode error', - 'description': 'Signature verification failed'}) + return handle_error(DECODE_ERROR_SIGNATURE) diff --git a/pybossa/auth/errcodes.py b/pybossa/auth/errcodes.py new file mode 100644 index 0000000000..e87ede0138 --- /dev/null +++ b/pybossa/auth/errcodes.py @@ -0,0 +1,37 @@ +# -*- coding: utf8 -*- +# This file is part of PyBossa. +# +# Copyright (C) 2016 SciFabric LTD. +# +# PyBossa is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# PyBossa is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with PyBossa. If not, see . + +INVALID_HEADER_MISSING = {'code': 'invalid_header', + 'description': 'Missing Authorization header'} + +INVALID_HEADER_BEARER = {'code': 'invalid_header', + 'description': 'Authorization header \ + must start with Bearer'} + +INVALID_HEADER_TOKEN = {'code': 'invalid_header', + 'description': 'Token not found'} + +INVALID_HEADER_BEARER_TOKEN = {'code': 'invalid_header', + 'description': 'Authorization header must \ + be Bearer + \\s + token'} + +WRONG_PROJECT_SIGNATURE = {'code': 'Wrong project', + 'description': 'Signature verification failed'} + +DECODE_ERROR_SIGNATURE = {'code': 'Decode error', + 'description': 'Signature verification failed'} From c38615a3f1118852d905c67831c794a7105ef79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 11:58:47 +0200 Subject: [PATCH 27/42] Test external uid. --- test/test_api/test_taskrun_api.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/test_api/test_taskrun_api.py b/test/test_api/test_taskrun_api.py index 695556cac2..2722a73c0c 100644 --- a/test/test_api/test_taskrun_api.py +++ b/test/test_api/test_taskrun_api.py @@ -488,6 +488,38 @@ def test_taskrun_post_requires_newtask_first_anonymous(self): success = self.app.post('/api/taskrun', data=datajson) assert success.status_code == 200, success.data + def test_taskrun_post_requires_newtask_first_external_uid(self): + """Test API TaskRun post fails if task was not previously requested for + external user""" + project = ProjectFactory.create() + url = '/api/auth/project/%s/token' % project.short_name + headers = {'Authorization': project.secret_key} + token = self.app.get(url, headers=headers) + headers['Authorization'] = 'Bearer %s' % token.data + task = TaskFactory.create(project=project) + data = dict( + project_id=project.id, + task_id=task.id, + info='my task result', + external_uid='1xa') + datajson = json.dumps(data) + fail = self.app.post('/api/taskrun', data=datajson, headers=headers) + err = json.loads(fail.data) + + assert fail.status_code == 403, fail.status_code + assert err['status'] == 'failed', err + assert err['status_code'] == 403, err + assert err['exception_msg'] == 'You must request a task first!', err + assert err['exception_cls'] == 'Forbidden', err + assert err['target'] == 'taskrun', err + + # Succeeds after requesting a task + self.app.get('/api/project/%s/newtask?external_uid=1xa' % project.id, + headers=headers) + success = self.app.post('/api/taskrun', data=datajson, headers=headers) + assert success.status_code == 200, success.data + + @with_context def test_taskrun_post_requires_newtask_first_authenticated(self): From 255ad775c08af4367d9c97773a8b31c6862655fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 12:08:18 +0200 Subject: [PATCH 28/42] Test also external uid. --- test/test_api/test_taskrun_api.py | 63 +++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/test/test_api/test_taskrun_api.py b/test/test_api/test_taskrun_api.py index 2722a73c0c..1566947dfc 100644 --- a/test/test_api/test_taskrun_api.py +++ b/test/test_api/test_taskrun_api.py @@ -463,6 +463,69 @@ def test_taskrun_authenticated_post(self, guard): assert tmp.status_code == 403, tmp.data + @with_context + @patch('pybossa.api.task_run.ContributionsGuard') + def test_taskrun_authenticated_external_uid_post(self, guard): + """Test API TaskRun creation and auth for authenticated external uid""" + guard.return_value = mock_contributions_guard(True) + project = ProjectFactory.create() + url = '/api/auth/project/%s/token' % project.short_name + headers = {'Authorization': project.secret_key} + token = self.app.get(url, headers=headers) + headers['Authorization'] = 'Bearer %s' % token.data + + task = TaskFactory.create(project=project) + data = dict( + project_id=project.id, + task_id=task.id, + info='my task result', + external_uid='1xa') + + # With wrong project_id + data['project_id'] = 100000000000000000 + datajson = json.dumps(data) + url = '/api/taskrun?api_key=%s' % project.owner.api_key + tmp = self.app.post(url, data=datajson, headers=headers) + err_msg = "This post should fail as the project_id is wrong" + err = json.loads(tmp.data) + assert tmp.status_code == 403, err_msg + assert err['status'] == 'failed', err_msg + assert err['status_code'] == 403, err_msg + assert err['exception_msg'] == 'Invalid project_id', err_msg + assert err['exception_cls'] == 'Forbidden', err_msg + assert err['target'] == 'taskrun', err_msg + + # With wrong task_id + data['project_id'] = task.project_id + data['task_id'] = 100000000000000000000 + datajson = json.dumps(data) + tmp = self.app.post(url, data=datajson, headers=headers) + err_msg = "This post should fail as the task_id is wrong" + err = json.loads(tmp.data) + assert tmp.status_code == 403, err_msg + assert err['status'] == 'failed', err_msg + assert err['status_code'] == 403, err_msg + assert err['exception_msg'] == 'Invalid task_id', err_msg + assert err['exception_cls'] == 'Forbidden', err_msg + assert err['target'] == 'taskrun', err_msg + + # Now with everything fine + data = dict( + project_id=task.project_id, + task_id=task.id, + user_id=project.owner.id, + info='my task result', + external_uid='1xa') + datajson = json.dumps(data) + tmp = self.app.post(url, data=datajson, headers=headers) + r_taskrun = json.loads(tmp.data) + assert tmp.status_code == 200, r_taskrun + + # If the user tries again it should be forbidden + tmp = self.app.post(url, data=datajson, headers=headers) + assert tmp.status_code == 403, tmp.data + + def test_taskrun_post_requires_newtask_first_anonymous(self): """Test API TaskRun post fails if task was not previously requested for anonymous user""" From 3692e42cfb2449d7b6b108ed26353b0efec7c911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 12:12:27 +0200 Subject: [PATCH 29/42] Increase coverage. --- test/test_api/test_taskrun_api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_api/test_taskrun_api.py b/test/test_api/test_taskrun_api.py index 1566947dfc..276b67eb3e 100644 --- a/test/test_api/test_taskrun_api.py +++ b/test/test_api/test_taskrun_api.py @@ -517,6 +517,11 @@ def test_taskrun_authenticated_external_uid_post(self, guard): info='my task result', external_uid='1xa') datajson = json.dumps(data) + # But without authentication + tmp = self.app.post(url, data=datajson) + r_taskrun = json.loads(tmp.data) + assert tmp.status_code == 403, r_taskrun + tmp = self.app.post(url, data=datajson, headers=headers) r_taskrun = json.loads(tmp.data) assert tmp.status_code == 200, r_taskrun From 9225ab7033ab8069948f5bff0bc9ec03fd5a362f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 12:19:40 +0200 Subject: [PATCH 30/42] Increase coverage. --- test/test_authentication.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/test_authentication.py b/test/test_authentication.py index 6fed7fa08a..9086e532fc 100644 --- a/test/test_authentication.py +++ b/test/test_authentication.py @@ -19,9 +19,10 @@ import jwt from default import Test, with_context from pybossa.auth import jwt_authorize_project +from pybossa.auth import handle_error as handle_error_upstream from pybossa.auth.errcodes import * from factories import ProjectFactory -from mock import patch +from mock import patch, MagicMock class TestAuthentication(Test): @@ -40,6 +41,18 @@ def handle_error(error): class TestJwtAuthorization(Test): + @patch('pybossa.auth.jsonify') + def test_handle_error(self, mymock): + """Test handle error method.""" + resp = MagicMock() + resp.status_code = 0 + resp.data = INVALID_HEADER_TOKEN + mymock.return_value = resp + + tmp = handle_error_upstream(INVALID_HEADER_TOKEN) + assert tmp.status_code == 401 + assert tmp.data == INVALID_HEADER_TOKEN, tmp.data + @patch('pybossa.auth.handle_error') def test_jwt_authorize_project_no_payload(self, mymock): From 5568ef82bd02b2a59296efa98777295afb8a6b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 12:24:07 +0200 Subject: [PATCH 31/42] Increase coverage. --- test/test_api/test_taskrun_api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_api/test_taskrun_api.py b/test/test_api/test_taskrun_api.py index 276b67eb3e..e304571861 100644 --- a/test/test_api/test_taskrun_api.py +++ b/test/test_api/test_taskrun_api.py @@ -25,6 +25,7 @@ from pybossa.repositories import ProjectRepository, TaskRepository from pybossa.repositories import ResultRepository from pybossa.core import db +from pybossa.auth.errcodes import * project_repo = ProjectRepository(db) task_repo = TaskRepository(db) @@ -581,6 +582,12 @@ def test_taskrun_post_requires_newtask_first_external_uid(self): assert err['exception_cls'] == 'Forbidden', err assert err['target'] == 'taskrun', err + # Succeeds after requesting a task + res = self.app.get('/api/project/%s/newtask?external_uid=1xa' % project.id) + assert res.status_code == 401 + assert json.loads(res.data) == INVALID_HEADER_MISSING + + # Succeeds after requesting a task self.app.get('/api/project/%s/newtask?external_uid=1xa' % project.id, headers=headers) From e2dd16965fce76f02ff7fd77c513a609b90f20a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 12:36:01 +0200 Subject: [PATCH 32/42] Increase coverage. --- test/test_sched.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/test_sched.py b/test/test_sched.py index 196cc7c553..30f5309cb9 100644 --- a/test/test_sched.py +++ b/test/test_sched.py @@ -544,9 +544,14 @@ def test_get_default_task_anonymous(self): @with_context def test_get_breadth_first_task_user(self): user = self.create_users()[0] - self._test_get_breadth_first_task(user) + self._test_get_breadth_first_task(user=user) - def _test_get_breadth_first_task(self, user=None): + @with_context + def test_get_breadth_first_task_external_user(self): + self._test_get_breadth_first_task(external_uid='234') + + + def _test_get_breadth_first_task(self, user=None, external_uid=None): self.del_task_runs() if user: short_name = 'xyzuser' @@ -582,6 +587,11 @@ def _test_get_breadth_first_task(self, user=None): out = pybossa.sched.get_breadth_first_task(projectid, owner.id) assert out.id == taskid, out + # now check we get task without task runs as a external uid + out = pybossa.sched.get_breadth_first_task(projectid, + external_uid=external_uid) + assert out.id == taskid, out + # now check that offset works out1 = pybossa.sched.get_breadth_first_task(projectid) From f9183a08cdc4cff01049d77bd85bcbf324f11670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 13:04:13 +0200 Subject: [PATCH 33/42] Docs for new v2.3.0 --- doc/api.rst | 46 ++++++++++++++++++++++++++++++++++++++++ doc/changelog/index.rst | 2 ++ doc/changelog/v2.3.0.rst | 22 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 doc/changelog/v2.3.0.rst diff --git a/doc/api.rst b/doc/api.rst index 4ac3791abd..354d28a316 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -407,6 +407,52 @@ desired:: Where 'provider' will be any of the third parties supported, i.e. 'twitter', 'facebook' or 'google'. +Using your own user database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since version v2.3.0 PYBOSSA supports external User IDs. This means that you can +easily use your own database of users without having to registering them in the +PYBOSSA server. As a benefit, you will be able to track your own users within the +PYBOSSA server providing a very simple and easy experience for them. + +A typical case for this would be for example a native phone app (Android, iOS or Windows). + +Usually phone apps have their own user base. With this in mind, you can add a crowdsourcing +feature to your phone app by just using PYBOSSA in the following way. + +First, create a project. When you create a project in PYBOSSA the system will create for +you a *secret key*. This secret key will be used by your phone app to authenticate all +the requests and avoid other users to send data to your project via external user API. + +Now your phone app will have to authenticate to the server to get tasks and post task runs. + +To do it, all you have to do is to create an HTTP Request with an Authorization Header like this:: + + HEADERS Authorization: project.secret_key + GET http://{pybossa-site-url}/api/auth/project/short_name/token + +That request will return a JWT token for you. With that token, you will be able to start +requesting tasks for your user base passing again an authorization header. Imagine a user +from your database is identified like this: '1xa':: + + HEADERS Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.TJVA95OrM7E2cBab30RMHrHDcEfxjoYZgeFONFh7HgQ + GET http://{pybossa-site-url}/api/{project.id}/newtask?external_uid=1xa + + +That will return a task for the user ID 1xa that belongs to your database but not to +PYBOSSA. Then, once the user has completed the task you will be able to submit it like +this:: + + HEADERS Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWV9.TJVA95OrM7E2cBab30RMHrHDcEfxjoYZgeFONFh7HgQ + POST http://{pybossa-site-url}/api/taskrun + + +.. note:: + The TaskRun object needs to have the external_uid field filled with 1xa. + +As simple as that! + + Example Usage ------------- diff --git a/doc/changelog/index.rst b/doc/changelog/index.rst index 5ac8baa81d..b853aecd79 100644 --- a/doc/changelog/index.rst +++ b/doc/changelog/index.rst @@ -1,6 +1,7 @@ Changelog ========= +* v2.3.0_ * v2.2.1_ * v2.2.0_ * v2.1.0_ @@ -22,6 +23,7 @@ Changelog * v1.1.0_ * v0.2.3_ +.. _v2.3.0: v2.3.0.html .. _v2.2.1: v2.2.1.html .. _v2.2.0: v2.2.0.html .. _v2.1.0: v2.1.0.html diff --git a/doc/changelog/v2.3.0.rst b/doc/changelog/v2.3.0.rst new file mode 100644 index 0000000000..26ff9ee146 --- /dev/null +++ b/doc/changelog/v2.3.0.rst @@ -0,0 +1,22 @@ +================ +Changelog v2.3.0 +================ + + +This new version adds a few cool features to PYBOSSA. Basically, it allows to use +PYBOSSA backend as your crowdsourcing engine for native iOS and Android phone apps. + +The idea is that those apps, usually have their own user base, with their own IDs. + +As a result, you don't want to force your user base to register again in another +service just to help you with your crowdsourcing research. Therefore, PYBOSSA comes +to the rescue allowing you to login those users in a PYBOSSA project using a secure +token (JWT). + +The process is really simple, you create a PYBOSSA project, you copy the secret key +created by PYBOSSA for your project and you use it to authenticate your requests. Then +when a user sends a Task Run you pass your authentication token and your internal user +ID. As simple as that. PYBOSSA will handle everything as usual. + + * Add support for external User IDs. + * Add JWT authentication for projects. From fb4c31cca72f1bcf07c98d79c777d9f9f6a612b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 14:34:53 +0200 Subject: [PATCH 34/42] Allow project owner to update the secret key. --- pybossa/view/projects.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pybossa/view/projects.py b/pybossa/view/projects.py index 7f6e29a099..75b955810d 100644 --- a/pybossa/view/projects.py +++ b/pybossa/view/projects.py @@ -34,6 +34,7 @@ from pybossa.core import (uploader, signer, sentinel, json_exporter, csv_exporter, importer, sentinel) +from pybossa.model import make_uuid from pybossa.model.project import Project from pybossa.model.category import Category from pybossa.model.task import Task @@ -1675,3 +1676,28 @@ def results(short_name): "n_results": n_results} return render_template('/projects/results.html', **template_args) + +@blueprint.route('//resetsecretkey', methods=['POST']) +@login_required +def reset_secret_key(short_name): + """ + Reset Project key. + + Returns a Jinja2 template. + + """ + + (project, owner, n_tasks, n_task_runs, + overall_progress, last_activity, + n_results) = project_by_shortname(short_name) + + title = project_title(project, "Results") + + ensure_authorized_to('update', project) + + project.secret_key = make_uuid() + project_repo.update(project) + cached_projects.delete_project(short_name) + msg = gettext('New secret key generated') + flash(msg, 'success') + return redirect(url_for('.update', short_name=short_name)) From 8786c7afe7c5c53202a5754ef36fb17f2145982c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Tue, 2 Aug 2016 15:56:29 +0200 Subject: [PATCH 35/42] Fixes. --- pybossa/view/projects.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pybossa/view/projects.py b/pybossa/view/projects.py index 75b955810d..f5bbbc9909 100644 --- a/pybossa/view/projects.py +++ b/pybossa/view/projects.py @@ -1677,14 +1677,13 @@ def results(short_name): return render_template('/projects/results.html', **template_args) + @blueprint.route('//resetsecretkey', methods=['POST']) @login_required def reset_secret_key(short_name): """ Reset Project key. - Returns a Jinja2 template. - """ (project, owner, n_tasks, n_task_runs, From c6f24a4ab1ae436c280ebf4279508e41ba669325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 10:31:21 +0200 Subject: [PATCH 36/42] New tests. --- test/test_view/test_project_passwords.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_view/test_project_passwords.py b/test/test_view/test_project_passwords.py index d20063b4d5..b14a42f583 100644 --- a/test/test_view/test_project_passwords.py +++ b/test/test_view/test_project_passwords.py @@ -27,7 +27,10 @@ def configure_mock_current_user_from(user, mock): def is_anonymous(): return user is None + def is_authenticated(): + return True mock.is_anonymous.return_value = is_anonymous() + mock.is_authenticated.return_value = True mock.admin = user.admin if user != None else None mock.id = user.id if user != None else None return mock @@ -218,3 +221,19 @@ def test_normal_auth_used_if_no_password_protected(self, fake_authorizer): self.app.get('/project/%s' % project.short_name, follow_redirects=True) assert fake_authorizer.called == True + + def test_get_reset_project_secret_key(self): + """Test GET project reset key method works.""" + project = ProjectFactory.create() + url = '/project/%s/resetsecretkey' % project.short_name + res = self.app.get(url) + assert res.status_code == 405, res.status_code + + def test_reset_project_secret_key(self): + """Test project reset key method works.""" + project = ProjectFactory.create() + url = '/project/%s/resetsecretkey' % project.short_name + res = self.app.post(url, follow_redirects=True) + assert res.status_code == 200, res.status_code + err_msg = "User should be redirected to sign in." + assert "Sign in" in res.data, err_msg From 7182966d71a71e38cd9e10debc2af20b2f3911ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 10:51:05 +0200 Subject: [PATCH 37/42] New tests for resetting the secret key. --- test/test_web.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/test/test_web.py b/test/test_web.py index 4f9793422e..96d0812aa1 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -893,7 +893,7 @@ def test_11_a_create_application_errors(self, mock): @patch('pybossa.ckan.requests.get') @patch('pybossa.view.projects.uploader.upload_file', return_value=True) @patch('pybossa.forms.validator.requests.get') - def test_12_update_application(self, Mock, mock, mock_webhook): + def test_12_update_project(self, Mock, mock, mock_webhook): """Test WEB update project works""" html_request = FakeResponse(text=json.dumps(self.pkg_json_not_found), status_code=200, @@ -3713,3 +3713,41 @@ def test_results_with_values_and_template(self): result_repo.update(result) res = self.app.get(url, follow_redirects=True) assert "The results" in res.data, res.data + + @with_context + def test_update_project_secret_key_owner(self): + """Test update project secret key owner.""" + self.register() + self.new_project() + + project = project_repo.get(1) + + old_key = project.secret_key + + url = "/project/%s/resetsecretkey" % project.short_name + + res = self.app.post(url, follow_redirects=True) + + project = project_repo.get(1) + + err_msg = "A new key should be generated" + assert "New secret key generated" in res.data, err_msg + assert old_key != project.secret_key, err_msg + + @with_context + def test_update_project_secret_key_not_owner(self): + """Test update project secret key not owner.""" + self.register() + self.new_project() + self.signout() + + self.register(email="juan@juan.com", name="juanjuan") + + project = project_repo.get(1) + + url = "/project/%s/resetsecretkey" % project.short_name + + res = self.app.post(url, follow_redirects=True) + + assert res.status_code == 403, res.status_code + From be4223f1e7e18099c8c5bc82ad0c276bcdbe09ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 10:51:33 +0200 Subject: [PATCH 38/42] Use new branch for reset project key. --- pybossa/themes/default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybossa/themes/default b/pybossa/themes/default index cb86ecf29f..596472f6bf 160000 --- a/pybossa/themes/default +++ b/pybossa/themes/default @@ -1 +1 @@ -Subproject commit cb86ecf29f80a8b313e928c6a8e3096cea0cffef +Subproject commit 596472f6bf78b04f9aa6b34a782eb38b71a61433 From 30d68782111ef710a6b82550e90f05219adce962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 11:25:57 +0200 Subject: [PATCH 39/42] Fixes. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 86c1bef985..20bfc63aa2 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( name = 'pybossa', - version = '2.2.0', + version = '2.3.0', packages = find_packages(), install_requires = requirements, # only needed when installing directly from setup.py (PyPi, eggs?) and pointing to e.g. a git repo. From 1bce666aa6237dcd0999cb6d0779dda165387ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 11:26:22 +0200 Subject: [PATCH 40/42] Fixes. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 20bfc63aa2..05635d3402 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,6 @@ # metadata for upload to PyPI author = 'SciFabric LTD', - # TODO: change author_email = 'info@scifabric.com', description = 'Open Source CrowdSourcing framework', long_description = '''PyBossa is an open source crowdsourcing solution for volunteer computing, thinking and sensing ''', From d02a5ac3a415215bff1aed4e6b5bd168ab59ece7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 11:33:43 +0200 Subject: [PATCH 41/42] Fixes. --- doc/api.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 354d28a316..47cf76aec5 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -424,6 +424,12 @@ First, create a project. When you create a project in PYBOSSA the system will cr you a *secret key*. This secret key will be used by your phone app to authenticate all the requests and avoid other users to send data to your project via external user API. + +.. note:: + + We highly recommend using SSL on your server to secure all the process. You can use + Let's Encrypt certificates for free. Check their `documentation. `_ + Now your phone app will have to authenticate to the server to get tasks and post task runs. To do it, all you have to do is to create an HTTP Request with an Authorization Header like this:: @@ -453,8 +459,8 @@ this:: As simple as that! -Example Usage -------------- +Command line Example Usage of the API +------------------------------------- Create a Project object: From 168f490abb3a12830169920f4500eda2459e44a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Wed, 3 Aug 2016 11:57:48 +0200 Subject: [PATCH 42/42] Use latest version. --- pybossa/themes/default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pybossa/themes/default b/pybossa/themes/default index 596472f6bf..23ef3c10af 160000 --- a/pybossa/themes/default +++ b/pybossa/themes/default @@ -1 +1 @@ -Subproject commit 596472f6bf78b04f9aa6b34a782eb38b71a61433 +Subproject commit 23ef3c10affe5db85d02cb60f60bb5dae1a591f9