diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index d2e6383ea..213d348d2 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -8,6 +8,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 + with: + submodules: recursive - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/asreview/webapp/tests/test_api/test_projects.py b/asreview/webapp/tests/test_api/test_projects.py index 1bbaf6622..716e6d18a 100644 --- a/asreview/webapp/tests/test_api/test_projects.py +++ b/asreview/webapp/tests/test_api/test_projects.py @@ -1,5 +1,6 @@ import inspect import time +from pathlib import Path from typing import Union import pytest @@ -9,10 +10,10 @@ import asreview.webapp.tests.utils.crud as crud import asreview.webapp.tests.utils.misc as misc from asreview.project import ASReviewProject +from asreview.utils import asreview_path from asreview.webapp import DB from asreview.webapp.authentication.models import Project from asreview.webapp.tests.utils.misc import current_app_is_authenticated -from asreview.webapp.tests.utils.misc import retrieve_project_url_github # NOTE: I don't see a plugin that can be used for testing # purposes @@ -23,7 +24,15 @@ + "asreview/master/tests/demo_data/generic_labels.csv" }, ] -IMPORT_PROJECT_URLS = retrieve_project_url_github() + + +def _asreview_file_archive(): + return list( + Path("asreview", "webapp", "tests", "asreview-project-file-archive").glob( + "*/asreview-project-*-startreview.asreview" + ) + ) + # NOTE: the setup fixture entails: a FlaskClient, 1 user (signed in), # and a project of this user OR a project from an unauthenticated app. @@ -70,9 +79,16 @@ def test_try_upgrade_a_modern_project(setup): # Test upgrading a v0.x project def test_upgrade_an_old_project(setup): client, user, _ = setup - # get an old version from github - old_project_url = retrieve_project_url_github("v0.19") - project = misc.copy_github_project_into_asreview_folder(old_project_url) + + asreview_v0_file = ( + "asreview/webapp/tests/asreview-project-file-archive/" + "v0.19/asreview-project-v0-19-startreview.asreview" + ) + + project = ASReviewProject.load( + open(asreview_v0_file, "rb"), asreview_path(), safe_import=True + ) + # we need to make sure this new, old-style project can be found # under current user if the app is authenticated if current_app_is_authenticated(): @@ -85,14 +101,13 @@ def test_upgrade_an_old_project(setup): # Test importing old projects, verify ids -@pytest.mark.parametrize("url", IMPORT_PROJECT_URLS) -def test_import_project_files(setup, url): +@pytest.mark.parametrize("fp", _asreview_file_archive()) +def test_import_project_files(setup, fp): client, user, first_project = setup # import project - status_code, data = au.import_project(client, url) + status_code, data = au.import_project(client, fp) # get contents asreview folder folders = set(misc.get_folders_in_asreview_path()) - # asserts assert len(folders) == 2 assert status_code == 200 assert isinstance(data, dict) diff --git a/asreview/webapp/tests/test_extensions/test_auth_tool.py b/asreview/webapp/tests/test_extensions/test_auth_tool.py index 9288e77c4..decf5025f 100644 --- a/asreview/webapp/tests/test_extensions/test_auth_tool.py +++ b/asreview/webapp/tests/test_extensions/test_auth_tool.py @@ -7,6 +7,7 @@ import pytest import asreview.entry_points.auth_tool as tool +from asreview import ASReviewProject from asreview.entry_points.auth_tool import AuthTool from asreview.state.sql_converter import upgrade_asreview_project_file from asreview.utils import asreview_path @@ -48,12 +49,25 @@ def import_2_unauthenticated_projects(with_upgrade=True): files from github and copies them in the asreview folder. To use them in tests they need to be upgraded. Both projects are returned.""" - # get 2 unauthenticated projects - url1 = misc.retrieve_project_url_github("v0.19") - url2 = misc.retrieve_project_url_github("v0.18") - # import projects - proj1 = misc.copy_github_project_into_asreview_folder(url1) - proj2 = misc.copy_github_project_into_asreview_folder(url2) + + asreview_v0_file = ( + "asreview/webapp/tests/asreview-project-file-archive/" + "v0.18/asreview-project-v0-18-startreview.asreview" + ) + + proj1 = ASReviewProject.load( + open(asreview_v0_file, "rb"), asreview_path(), safe_import=True + ) + + asreview_v0_file = ( + "asreview/webapp/tests/asreview-project-file-archive/" + "v0.19/asreview-project-v0-19-startreview.asreview" + ) + + proj2 = ASReviewProject.load( + open(asreview_v0_file, "rb"), asreview_path(), safe_import=True + ) + if with_upgrade: # update these projects to a 1.x-ish config upgrade_asreview_project_file(proj1.project_path) @@ -189,7 +203,7 @@ def test_auth_tool_add_users_interact(client_auth): auth_tool = get_auth_tool_object(Namespace(json=None)) # build interactive input answers = interactive_user_data() - with patch('builtins.input', side_effect=answers): + with patch("builtins.input", side_effect=answers): auth_tool.add_users() # assert we now have a users assert crud.count_users() == 1 @@ -207,7 +221,7 @@ def test_auth_tool_add_users_interact_incorr_email(client_auth, capsys): answers = interactive_user_data() # add in a faulty email address answers.insert(1, "abcd@") - with patch('builtins.input', side_effect=answers): + with patch("builtins.input", side_effect=answers): auth_tool.add_users() _, err = capsys.readouterr() assert "Entered email address is not recognized" in err @@ -227,7 +241,7 @@ def test_auth_tool_add_users_interact_incorr_name(client_auth, capsys): answers = interactive_user_data() # add in a name that is too short answers.insert(2, "ab") - with patch('builtins.input', side_effect=answers): + with patch("builtins.input", side_effect=answers): auth_tool.add_users() _, err = capsys.readouterr() assert "Full name must contain more than 2" in err @@ -247,7 +261,7 @@ def test_auth_tool_add_users_interact_incorr_passw(client_auth, capsys): answers = interactive_user_data() # add in a name that is too short answers.insert(4, "1111") - with patch('builtins.input', side_effect=answers): + with patch("builtins.input", side_effect=answers): auth_tool.add_users() _, err = capsys.readouterr() assert "Use 8 or more characters with a mix" in err @@ -268,11 +282,9 @@ def test_validity_function_valid(capsys): correct = "a" hint = "Test hint" # run function with patched input - with patch('builtins.input', side_effect=[correct]): + with patch("builtins.input", side_effect=[correct]): # run validity function - auth_tool._ensure_valid_value_for( - "test", lambda x: x == correct, hint=hint - ) + auth_tool._ensure_valid_value_for("test", lambda x: x == correct, hint=hint) out, err = capsys.readouterr() assert not bool(out) assert not bool(err) @@ -291,11 +303,9 @@ def test_validity_function_invalid(capsys): incorrect = "b" hint = "Test hint" # run function with patched input - with patch('builtins.input', side_effect=[incorrect, correct]): + with patch("builtins.input", side_effect=[incorrect, correct]): # run validity function - auth_tool._ensure_valid_value_for( - "test", lambda x: x == correct, hint=hint - ) + auth_tool._ensure_valid_value_for("test", lambda x: x == correct, hint=hint) out, err = capsys.readouterr() assert not bool(out) assert err == hint @@ -403,10 +413,7 @@ def test_list_projects_with_json(client_no_auth, capsys): # create two projects _, data1 = au.create_project(client_no_auth, "test1") _, data2 = au.create_project(client_no_auth, "test2") - data = { - data1.get("id"): data1, - data2.get("id"): data2 - } + data = {data1.get("id"): data1, data2.get("id"): data2} # get auth_tool object auth_tool = get_auth_tool_object(Namespace(json=True)) # run function @@ -458,8 +465,7 @@ def test_link_project_with_json_string(client_auth, capsys): # check database and check if the users own the correct project assert crud.count_projects() == 2 project_dict = { - proj["owner_id"]: proj - for proj in json.loads(json.loads(json_string)) + proj["owner_id"]: proj for proj in json.loads(json.loads(json_string)) } for user in [user1, user2]: expected_proj = project_dict[user.id] @@ -483,7 +489,7 @@ def test_link_projects_interactively(client_auth): # create AuthTool object auth_tool = get_auth_tool_object(Namespace(json=None)) # run function with patched input - with patch('builtins.input', side_effect=[user.id, user.id]): + with patch("builtins.input", side_effect=[user.id, user.id]): # link project to user auth_tool.link_projects() # check database again @@ -509,7 +515,7 @@ def test_link_projects_interactively_with_typo(client_auth): # create AuthTool object auth_tool = get_auth_tool_object(Namespace(json=None)) # run function with patched input (there is a wrong id in there) - with patch('builtins.input', side_effect=[user.id, str(-5), user.id]): + with patch("builtins.input", side_effect=[user.id, str(-5), user.id]): # link project to user auth_tool.link_projects() # check database again @@ -519,12 +525,7 @@ def test_link_projects_interactively_with_typo(client_auth): # Test failure of anything related to projects if a project is older # than version 0.x. @pytest.mark.parametrize( - "method", - [ - "_generate_project_links", - "list_projects", - "link_projects" - ] + "method", ["_generate_project_links", "list_projects", "link_projects"] ) def test_projects_with_0x_projects(client_auth, method): # import projects diff --git a/asreview/webapp/tests/utils/api_utils.py b/asreview/webapp/tests/utils/api_utils.py index af3ce1991..31a01786f 100644 --- a/asreview/webapp/tests/utils/api_utils.py +++ b/asreview/webapp/tests/utils/api_utils.py @@ -1,8 +1,6 @@ import random import time -from io import BytesIO from typing import Union -from urllib.request import urlopen from flask.testing import FlaskClient @@ -223,12 +221,11 @@ def upgrade_project(client: FlaskClient, project: Union[Project, ASReviewProject return process_response(response) -def import_project(client: FlaskClient, url: str): - with urlopen(url) as project_file: - response = client.post( - "/api/projects/import_project", - data={"file": (BytesIO(project_file.read()), "project.asreview")}, - ) +def import_project(client: FlaskClient, asreview_file): + response = client.post( + "/api/projects/import_project", + data={"file": (open(asreview_file, "rb"), "project.asreview")}, + ) return process_response(response) diff --git a/asreview/webapp/tests/utils/misc.py b/asreview/webapp/tests/utils/misc.py index 58b1aed40..18c86e8fd 100644 --- a/asreview/webapp/tests/utils/misc.py +++ b/asreview/webapp/tests/utils/misc.py @@ -1,15 +1,11 @@ -import io import json import random import re from pathlib import Path from typing import Union -from urllib.request import urlopen -import requests from flask import current_app -from asreview.project import ASReviewProject from asreview.utils import asreview_path @@ -78,48 +74,6 @@ def choose_project_algorithms(): return data -def retrieve_project_url_github(version=None): - """Retrieve .asreview file(s) url from asreview-project-files-testing - GitHub repository. When version is not None, the function resturns - a single URL, otherwise a list containing URLs.""" - - repo = "asreview/asreview-project-files-testing" - repo_api_url = f"https://api.github.com/repos/{repo}/git/trees/master" - repo_url = f"https://github.com/{repo}/blob/master" - file_type = "startreview.asreview?raw=true" - - json_file = json.loads(urlopen(repo_api_url).read().decode("utf-8"))["tree"] - - version_tags = [] - project_urls = [] - - for file in json_file: - if file["type"] == "tree": - version_tags.append(file["path"]) - - for tag in version_tags: - file_version = f"/{tag}/asreview-project-{tag.replace('.', '-')}-" - url = repo_url + file_version + file_type - - if version is None: - project_urls.append(url) - else: - return url - - return project_urls - - -def copy_github_project_into_asreview_folder(url): - """This function copies a, on Github stored, ASReview project - into the asreview folder.""" - response = requests.get(url) - return ASReviewProject.load( - io.BytesIO(response.content), - asreview_path(), - safe_import=True - ) - - def get_folders_in_asreview_path(): """This function returns the amount of folders located in the asreview folder."""