From d0718c553888ace04b917c64cb75c6cc696d0e81 Mon Sep 17 00:00:00 2001 From: Yam Mesicka Date: Wed, 12 Aug 2020 06:54:06 +0300 Subject: [PATCH 01/13] Change DB schema to support multiple files, add migration --- devops/dev_bootstrap.sh | 4 +- lms/lmsdb/bootstrap.py | 83 ++++++++++++++++++++++++++++++++++++----- lms/lmsdb/models.py | 12 ++++-- 3 files changed, 84 insertions(+), 15 deletions(-) diff --git a/devops/dev_bootstrap.sh b/devops/dev_bootstrap.sh index 92c2aea7..f99bd158 100755 --- a/devops/dev_bootstrap.sh +++ b/devops/dev_bootstrap.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eux +set -x SCRIPT_FILE_PATH=$(readlink -f "${0}") SCRIPT_FOLDER=$(dirname "${SCRIPT_FILE_PATH}") @@ -47,4 +47,4 @@ $pip_exec install -r "${MAIN_FOLDER}/dev_requirements.txt" echo "Creating local SQLite DB" $python_exec "${DB_BOOTSTRAP_FILE_PATH}" -set +eux +set +x diff --git a/lms/lmsdb/bootstrap.py b/lms/lmsdb/bootstrap.py index bd9f09b2..2f64760f 100644 --- a/lms/lmsdb/bootstrap.py +++ b/lms/lmsdb/bootstrap.py @@ -1,9 +1,10 @@ import sys import logging -from typing import Type +from typing import Any, Optional, Type from peewee import ( # type: ignore - Field, Model, OperationalError, ProgrammingError, TextField, + Entity, Expression, Field, ForeignKeyField, Model, OP, + OperationalError, ProgrammingError, SQL, TextField, ) from playhouse.migrate import migrate # type: ignore @@ -21,8 +22,11 @@ def _migrate_column_in_table_if_needed( table: Type[Model], field_instance: Field, + *, + field_name: Optional[str] = None, + **kwargs: Any, ) -> bool: - column_name = field_instance.name + column_name = field_name or field_instance.name table_name = table.__name__.lower() cols = {col.name for col in db_config.database.get_columns(table_name)} @@ -34,14 +38,33 @@ def _migrate_column_in_table_if_needed( migrator = db_config.get_migrator_instance() with db_config.database.transaction(): migrate(migrator.add_column( - table_name, - field_instance.name, - field_instance, + table=table_name, + column_name=column_name, + field=field_instance, + **kwargs, )) db_config.database.commit() return True +def _migrate_copy_column(table: Type[Model], source: str, dest: str) -> bool: + table_name = table.__name__.lower() + migrator = db_config.get_migrator_instance() + with db_config.database.transaction(): + ( + db_config.database.execute_sql( + migrator.make_context() + .literal('UPDATE ').sql(Entity(table_name)) + .literal(' SET ').sql( + Expression( + Entity(dest), OP.EQ, SQL(' solution_id'), flat=True, + ), + ).query()[0] + ) + ) + return True + + def _rename_column_in_table_if_needed( table: Type[Model], old_column_name: str, @@ -111,10 +134,23 @@ def _drop_column_from_module_if_needed( log.info(f'Drop {column_name} field in {table}') migrator = db_config.get_migrator_instance() with db_config.database.transaction(): - migrate(migrator.drop_column( - table_name, - column_name, - )) + migrate(migrator.drop_column(table_name, column_name)) + db_config.database.commit() + return True + + +def _drop_constraint_if_needed(table: Type[Model], column_name: str) -> bool: + table_name = table.__name__.lower() + cols = {col.name for col in db_config.database.get_columns(table_name)} + + if column_name not in cols: + log.info(f'Column {column_name} not exists in {table}') + return False + + log.info(f'Drop foreign key on {table}.{column_name}') + migrator = db_config.get_migrator_instance() + with db_config.database.transaction(): + migrate(migrator.drop_constraint(table_name, column_name)) db_config.database.commit() return True @@ -239,6 +275,32 @@ def _add_solution_state_if_needed(): ) +def _multiple_files_migration() -> bool: + db = db_config.database + c = models.Comment + f = models.SolutionFile + s = models.Solution + solution_cols = {col.name for col in db.get_columns(s.__name__.lower())} + if 'json_data_str' not in solution_cols: + log.info('Skipping multiple files migration.') + return False + + with models.database.connection_context(): + solutions = s.select(s.id, s.id, '/main.py', s.json_data_str) + f.insert_from(solutions, [f.id, f.solution, f.path, f.code]).execute() + # _rename_column_in_table_if_needed(c, 'solution', 'file') + # _drop_constraint_if_needed(c, 'file') + file = ForeignKeyField(f, field=f.id, backref='comments', null=True) + _migrate_column_in_table_if_needed(c, file, field_name='file') + solutions = c.select(s.id, s.id, '/main.py', s.json_data_str) + f.insert_from(solutions, [f.id, f.solution, f.path, f.code]) + _migrate_copy_column(c, dest='file', source='solution_id') + _drop_column_from_module_if_needed(c, 'solution_id') + _drop_column_from_module_if_needed(s, 'json_data_str') + log.info('Successfully migrated multiple files.') + return True + + def main(): with models.database.connection_context(): models.database.create_tables(models.ALL_MODELS, safe=True) @@ -256,6 +318,7 @@ def main(): _upgrade_notifications_if_needed() _add_solution_state_if_needed() _add_indices_if_needed() + _multiple_files_migration() text_fixer.fix_texts() import_tests.load_tests_from_path('/app_dir/notebooks-tests') diff --git a/lms/lmsdb/models.py b/lms/lmsdb/models.py index 601eb9b2..64560e41 100644 --- a/lms/lmsdb/models.py +++ b/lms/lmsdb/models.py @@ -122,7 +122,7 @@ def __str__(self): @pre_save(sender=User) def on_save_handler(model_class, instance, created): - """Hashes password on creation/save""" + """Hash password on creation/save.""" # If password changed then it won't start with hash's method prefix is_password_changed = not instance.password.startswith('pbkdf2:sha256') @@ -215,7 +215,7 @@ class Exercise(BaseModel): subject = CharField() date = DateTimeField() users = ManyToManyField(User, backref='exercises') - is_archived = BooleanField(index=True) + is_archived = BooleanField(default=False, index=True) due_date = DateTimeField(null=True) notebook_num = IntegerField(default=0) order = IntegerField(default=0, index=True) @@ -473,6 +473,12 @@ def left_in_exercise(cls, exercise: Exercise) -> int: return int(response['checked'] * 100 / response['submitted']) +class SolutionFile(BaseModel): + path = TextField() + solution = ForeignKeyField(Solution, backref='files') + code = TextField() + + class ExerciseTest(BaseModel): exercise = ForeignKeyField(model=Exercise, unique=True) code = TextField() @@ -598,7 +604,7 @@ class Comment(BaseModel): timestamp = DateTimeField(default=datetime.now) line_number = IntegerField(constraints=[Check('line_number >= 1')]) comment = ForeignKeyField(CommentText) - solution = ForeignKeyField(Solution) + file = ForeignKeyField(Solution, backref='comments') is_auto = BooleanField(default=False) @classmethod From b08d8d5eae83bbb5157194d4c9348311cc6c02e5 Mon Sep 17 00:00:00 2001 From: Yam Mesicka Date: Wed, 12 Aug 2020 07:04:39 +0300 Subject: [PATCH 02/13] Fix flake8 errors --- lms/lmsdb/bootstrap.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lms/lmsdb/bootstrap.py b/lms/lmsdb/bootstrap.py index 2f64760f..557c4bb4 100644 --- a/lms/lmsdb/bootstrap.py +++ b/lms/lmsdb/bootstrap.py @@ -59,7 +59,7 @@ def _migrate_copy_column(table: Type[Model], source: str, dest: str) -> bool: Expression( Entity(dest), OP.EQ, SQL(' solution_id'), flat=True, ), - ).query()[0] + ).query()[0], ) ) return True @@ -288,8 +288,6 @@ def _multiple_files_migration() -> bool: with models.database.connection_context(): solutions = s.select(s.id, s.id, '/main.py', s.json_data_str) f.insert_from(solutions, [f.id, f.solution, f.path, f.code]).execute() - # _rename_column_in_table_if_needed(c, 'solution', 'file') - # _drop_constraint_if_needed(c, 'file') file = ForeignKeyField(f, field=f.id, backref='comments', null=True) _migrate_column_in_table_if_needed(c, file, field_name='file') solutions = c.select(s.id, s.id, '/main.py', s.json_data_str) From a31de178ee05e71b9c3127c72fdc62020fe3ae9d Mon Sep 17 00:00:00 2001 From: Yam Mesicka Date: Wed, 12 Aug 2020 23:35:37 +0300 Subject: [PATCH 03/13] Finish migration preparations --- lms/lmsdb/bootstrap.py | 25 ++++++++++++++++++++++++- lms/lmsdb/models.py | 1 - 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lms/lmsdb/bootstrap.py b/lms/lmsdb/bootstrap.py index 557c4bb4..0cae43ad 100644 --- a/lms/lmsdb/bootstrap.py +++ b/lms/lmsdb/bootstrap.py @@ -65,6 +65,24 @@ def _migrate_copy_column(table: Type[Model], source: str, dest: str) -> bool: return True +def _drop_not_null(table: Type[Model], column_name: str) -> bool: + table_name = table.__name__.lower() + migrator = db_config.get_migrator_instance() + with db_config.database.transaction(): + migrate(migrator.drop_not_null(table_name, column_name)) + db_config.database.commit() + return True + + +def _add_not_null(table: Type[Model], column_name: str) -> bool: + table_name = table.__name__.lower() + migrator = db_config.get_migrator_instance() + with db_config.database.transaction(): + migrate(migrator.add_not_null(table_name, column_name)) + db_config.database.commit() + return True + + def _rename_column_in_table_if_needed( table: Type[Model], old_column_name: str, @@ -279,7 +297,11 @@ def _multiple_files_migration() -> bool: db = db_config.database c = models.Comment f = models.SolutionFile - s = models.Solution + + class Solution(models.Solution): + json_data_str = TextField(column_name='json_data_str') + s = Solution + solution_cols = {col.name for col in db.get_columns(s.__name__.lower())} if 'json_data_str' not in solution_cols: log.info('Skipping multiple files migration.') @@ -293,6 +315,7 @@ def _multiple_files_migration() -> bool: solutions = c.select(s.id, s.id, '/main.py', s.json_data_str) f.insert_from(solutions, [f.id, f.solution, f.path, f.code]) _migrate_copy_column(c, dest='file', source='solution_id') + _add_not_null(c, 'file') _drop_column_from_module_if_needed(c, 'solution_id') _drop_column_from_module_if_needed(s, 'json_data_str') log.info('Successfully migrated multiple files.') diff --git a/lms/lmsdb/models.py b/lms/lmsdb/models.py index 64560e41..a86e5976 100644 --- a/lms/lmsdb/models.py +++ b/lms/lmsdb/models.py @@ -284,7 +284,6 @@ class Solution(BaseModel): default=0, constraints=[Check('grade <= 100'), Check('grade >= 0')], ) submission_timestamp = DateTimeField(index=True) - json_data_str = TextField() @property def is_checked(self): From a106db503a4bb5de0b4d8f3abc4818c91ce0f107 Mon Sep 17 00:00:00 2001 From: Yam Mesicka Date: Thu, 13 Aug 2020 06:05:42 +0300 Subject: [PATCH 04/13] Make schema support multiple files, fix minor bugs * Stop flake8 messages about asserts * Add hash to each solution for comparision * Further enhance the migration script * Create upload model * Make the view work again (with single file) * Fix minor bug in the notification's HTML --- .flake8 | 7 +- lms/extractors/base.py | 2 +- lms/lmsdb/bootstrap.py | 47 ++++++++-- lms/lmsdb/models.py | 100 ++++++++++++---------- lms/lmstests/public/unittests/services.py | 5 +- lms/lmsweb/views.py | 58 ++++++++----- lms/models/upload.py | 10 +++ lms/templates/navbar.html | 4 +- lms/templates/view.html | 58 +++++++------ 9 files changed, 182 insertions(+), 109 deletions(-) create mode 100644 lms/models/upload.py diff --git a/.flake8 b/.flake8 index a7278435..d9cf486a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,6 @@ [flake8] per-file-ignores = - lms/tests/*.py:S101 lms/lmstests/sandbox/flake8/defines.py:E501 - lms/tests/test_exercise_unit_tests.py:Q001,S101 - lms/tests/test_extractor.py:W293,S101 -ignore=I100,I201,W503 + lms/tests/test_exercise_unit_tests.py:Q001 + lms/tests/test_extractor.py:W293 +ignore=I100,S101,I201,W503 diff --git a/lms/extractors/base.py b/lms/extractors/base.py index d0723eec..b16a6c7b 100644 --- a/lms/extractors/base.py +++ b/lms/extractors/base.py @@ -29,7 +29,7 @@ def _convert_to_text(code: CodeFile) -> str: if code and isinstance(code, bytes): return code.decode(errors='replace') - assert isinstance(code, str) # noqa: S101 + assert isinstance(code, str) return code @classmethod diff --git a/lms/lmsdb/bootstrap.py b/lms/lmsdb/bootstrap.py index 0cae43ad..7a8a51c9 100644 --- a/lms/lmsdb/bootstrap.py +++ b/lms/lmsdb/bootstrap.py @@ -12,6 +12,7 @@ from lms.lmsdb import models from lms.lmstests.public.flake8 import text_fixer from lms.lmstests.public.unittests import import_tests +from lms.models import upload logging.basicConfig(stream=sys.stdout, level=logging.INFO, @@ -243,6 +244,27 @@ def _add_index_if_needed( db_config.database.commit() +def _drop_index_if_needed( + table: Type[Model], + field_instance: Field, + foreign_key: bool = False, +) -> None: + table_name = table.__name__.lower() + suffix = "_id" if foreign_key else "" + column_name = f'{table_name}_{field_instance.name}{suffix}' + migrator = db_config.get_migrator_instance() + log.info(f"Drop index from '{column_name}' field in '{table_name}'") + with db_config.database.transaction(): + try: + migrate(migrator.drop_index(table_name, column_name)) + except (OperationalError, ProgrammingError) as e: + if 'already exists' in str(e): + log.info('Index already exists.') + else: + raise + db_config.database.commit() + + def _add_indices_if_needed(): table_field_pairs = ( (models.Notification, models.Notification.created), @@ -293,15 +315,26 @@ def _add_solution_state_if_needed(): ) +def _update_solution_hashes(s): + log.info('Updating solution hashes. Might take a while.') + for solution in s: + solution.upload_hash = upload.create_hash(solution.json_data_str) + solution.save() + + def _multiple_files_migration() -> bool: db = db_config.database - c = models.Comment f = models.SolutionFile class Solution(models.Solution): json_data_str = TextField(column_name='json_data_str') + upload_hash = TextField(null=True) s = Solution + class Comment(models.Comment): + file = ForeignKeyField(f, backref='comments', null=True) + c = Comment + solution_cols = {col.name for col in db.get_columns(s.__name__.lower())} if 'json_data_str' not in solution_cols: log.info('Skipping multiple files migration.') @@ -310,12 +343,16 @@ class Solution(models.Solution): with models.database.connection_context(): solutions = s.select(s.id, s.id, '/main.py', s.json_data_str) f.insert_from(solutions, [f.id, f.solution, f.path, f.code]).execute() - file = ForeignKeyField(f, field=f.id, backref='comments', null=True) - _migrate_column_in_table_if_needed(c, file, field_name='file') + + _drop_index_if_needed(c, c.file, foreign_key=True) + _migrate_column_in_table_if_needed(c, c.file, field_name='file_id') + _migrate_column_in_table_if_needed(s, s.upload_hash) solutions = c.select(s.id, s.id, '/main.py', s.json_data_str) f.insert_from(solutions, [f.id, f.solution, f.path, f.code]) - _migrate_copy_column(c, dest='file', source='solution_id') - _add_not_null(c, 'file') + _update_solution_hashes(s) + _migrate_copy_column(c, dest='file_id', source='solution_id') + _add_not_null(c, 'file_id') + _add_not_null(s, 'upload_hash') _drop_column_from_module_if_needed(c, 'solution_id') _drop_column_from_module_if_needed(s, 'json_data_str') log.info('Successfully migrated multiple files.') diff --git a/lms/lmsdb/models.py b/lms/lmsdb/models.py index a86e5976..fa3780fd 100644 --- a/lms/lmsdb/models.py +++ b/lms/lmsdb/models.py @@ -1,9 +1,10 @@ import enum +import hashlib import random import secrets import string from datetime import datetime -from typing import Any, Dict, Iterable, Optional, Tuple, Type, Union +from typing import Any, Dict, Iterable, Optional, Tuple, Type, Union, cast from flask_login import UserMixin # type: ignore from peewee import ( # type: ignore @@ -178,12 +179,12 @@ def of( @classmethod def send( - cls, - user: User, - kind: int, - message: str, - related_id: Optional[int] = None, - action_url: Optional[str] = None, + cls, + user: User, + kind: int, + message: str, + related_id: Optional[int] = None, + action_url: Optional[str] = None, ) -> 'Notification': return cls.create(**{ cls.user.name: user, @@ -196,9 +197,9 @@ def send( @post_save(sender=Notification) def on_notification_saved( - model_class: Type[Notification], - instance: Notification, - created: datetime, + model_class: Type[Notification], + instance: Notification, + created: datetime, ): # sqlite supports delete query with order # but when we use postgres, peewee is stupid @@ -264,8 +265,9 @@ def active_solutions(cls) -> Iterable[str]: ) @classmethod - def to_choices(cls) -> Tuple[Tuple[str, str], ...]: - return tuple((choice.name, choice.value) for choice in tuple(cls)) + def to_choices(cls: enum.EnumMeta) -> Tuple[Tuple[str, str], ...]: + choices = cast(Iterable[enum.Enum], tuple(cls)) + return tuple((choice.name, choice.value) for choice in choices) class Solution(BaseModel): @@ -284,6 +286,7 @@ class Solution(BaseModel): default=0, constraints=[Check('grade <= 100'), Check('grade >= 0')], ) submission_timestamp = DateTimeField(index=True) + upload_hash = TextField() @property def is_checked(self): @@ -303,10 +306,6 @@ def set_state(self, new_state: SolutionState, **kwargs) -> bool: updated = changes.execute() == 1 return updated - @property - def code(self): - return self.json_data_str - def ordered_versions(self) -> Iterable['Solution']: return Solution.select().where( Solution.exercise == self.exercise, @@ -318,7 +317,7 @@ def test_results(self) -> Iterable[dict]: @classmethod def of_user( - cls, user_id: int, with_archived: bool = False, + cls, user_id: int, with_archived: bool = False, ) -> Iterable[Dict[str, Any]]: db_exercises = Exercise.get_objects(fetch_archived=with_archived) exercises = Exercise.as_dicts(db_exercises) @@ -344,15 +343,15 @@ def comments(self): @classmethod def solution_exists( - cls, - exercise: Exercise, - solver: User, - json_data_str: str, + cls, + exercise: Exercise, + solver: User, + upload_hash: str, ): return cls.select().where( cls.exercise == exercise, cls.solver == solver, - cls.json_data_str == json_data_str, + cls.upload_hash == upload_hash, ).exists() @classmethod @@ -360,15 +359,22 @@ def create_solution( cls, exercise: Exercise, solver: User, - json_data_str='', + upload_hash: str, + files: Dict[str, str], ) -> 'Solution': instance = cls.create(**{ cls.exercise.name: exercise, cls.solver.name: solver, cls.submission_timestamp.name: datetime.now(), - cls.json_data_str.name: json_data_str, + cls.upload_hash.name: upload_hash, }) + files_details = [ + SolutionFile(path, content) + for path, content in files.items() + ] + SolutionFile.bulk_create(files_details) + # update old solutions for this exercise other_solutions: Iterable[Solution] = cls.select().where( cls.exercise == exercise, @@ -408,8 +414,8 @@ def _base_next_unchecked(cls): ) def mark_as_checked( - self, - by: Optional[Union[User, int]] = None, + self, + by: Optional[Union[User, int]] = None, ) -> bool: return self.set_state( Solution.STATES.DONE, @@ -513,10 +519,10 @@ class ExerciseTestName(BaseModel): @classmethod def create_exercise_test_name( - cls, - exercise_test: ExerciseTest, - test_name: str, - pretty_test_name: str, + cls, + exercise_test: ExerciseTest, + test_name: str, + pretty_test_name: str, ): instance, created = cls.get_or_create(**{ cls.exercise_test.name: exercise_test, @@ -555,11 +561,11 @@ class SolutionExerciseTestExecution(BaseModel): @classmethod def create_execution_result( - cls, - solution: Solution, - test_name: str, - user_message: str, - staff_message: str, + cls, + solution: Solution, + test_name: str, + user_message: str, + staff_message: str, ): exercise_test_name = ExerciseTestName.get_exercise_test( exercise=solution.exercise, @@ -589,7 +595,7 @@ class CommentText(BaseModel): @classmethod def create_comment( - cls, text: str, flake_key: Optional[str] = None, + cls, text: str, flake_key: Optional[str] = None, ) -> 'CommentText': instance, _ = CommentText.get_or_create( **{CommentText.text.name: text}, @@ -603,23 +609,23 @@ class Comment(BaseModel): timestamp = DateTimeField(default=datetime.now) line_number = IntegerField(constraints=[Check('line_number >= 1')]) comment = ForeignKeyField(CommentText) - file = ForeignKeyField(Solution, backref='comments') + file = ForeignKeyField(SolutionFile, backref='comments') is_auto = BooleanField(default=False) @classmethod def create_comment( - cls, - commenter: User, - line_number: int, - comment_text: CommentText, - solution: Solution, - is_auto: bool, + cls, + commenter: User, + line_number: int, + comment_text: CommentText, + file: SolutionFile, + is_auto: bool, ) -> 'Comment': return cls.get_or_create( commenter=commenter, line_number=line_number, comment=comment_text, - solution=solution, + file=file, is_auto=is_auto, ) @@ -628,15 +634,19 @@ def by_solution(cls, solution_id: int): fields = [ cls.id, cls.line_number, cls.is_auto, CommentText.id.alias('comment_id'), CommentText.text, + SolutionFile.id.alias('file_id'), User.fullname.alias('author_name'), ] return ( cls .select(*fields) + .join(SolutionFile) + .join(Solution) + .switch() .join(CommentText) .switch() .join(User) - .where(cls.solution == solution_id) + .where(cls.file.solution == solution_id) ) @classmethod diff --git a/lms/lmstests/public/unittests/services.py b/lms/lmstests/public/unittests/services.py index 3af32168..5beb37e5 100644 --- a/lms/lmstests/public/unittests/services.py +++ b/lms/lmstests/public/unittests/services.py @@ -62,7 +62,10 @@ def _run_tests_on_solution(self): return junit_results def _generate_python_code(self) -> str: - user_code = self._solution.code + # FIX: Multiple files + assert self._solution is not None + user_code = '\n'.join(file.code for file in self._solution.files) + assert self._exercise_auto_test is not None test_code = self._exercise_auto_test.code return f'{test_code}\n\n{user_code}' diff --git a/lms/lmsweb/views.py b/lms/lmsweb/views.py index 57c3eae0..d6667ff0 100644 --- a/lms/lmsweb/views.py +++ b/lms/lmsweb/views.py @@ -18,15 +18,15 @@ from werkzeug.utils import redirect from lms.lmsdb.models import ( - ALL_MODELS, Comment, CommentText, Exercise, RoleOptions, Solution, User, - database, + ALL_MODELS, Comment, CommentText, Exercise, RoleOptions, + Solution, SolutionFile, User, database, ) import lms.extractors.base as extractor from lms.lmstests.public.flake8 import tasks as flake8_tasks from lms.lmstests.public.unittests import tasks as unittests_tasks from lms.lmstests.public.identical_tests import tasks as identical_tests_tasks from lms.lmsweb import config, routes, webapp -from lms.models import notifications, solutions +from lms.models import notifications, solutions, upload login_manager = LoginManager() login_manager.init_app(webapp) @@ -299,7 +299,7 @@ def send_(): @webapp.route('/upload', methods=['POST']) @login_required -def upload(): +def upload_page(): user_id = current_user.id user = User.get_or_none(User.id == user_id) # should never happen if user is None: @@ -311,7 +311,9 @@ def upload(): if not file: return fail(422, 'No file was given') - exercises = list(extractor.Extractor(file.read())) + file_content = file.read() + file_hash = upload.create_hash(file_content) + exercises = list(extractor.Extractor(file_content)) if not exercises: msg = 'No exercises were found in the notebook' desc = 'did you use Upload ? (example: Upload 1)' @@ -327,14 +329,15 @@ def upload(): continue if Solution.solution_exists( - exercise=exercise, - solver=user, - json_data_str=code, + exercise=exercise, + solver=user, + upload_hash=file_hash, ): continue solution = Solution.create_solution( exercise=exercise, solver=user, + upload_hash=file_hash, json_data_str=code, ) flake8_tasks.run_flake8_on_solution.apply_async(args=(solution.id,)) @@ -351,8 +354,9 @@ def upload(): @webapp.route(f'{routes.SOLUTIONS}/') +@webapp.route(f'{routes.SOLUTIONS}//') @login_required -def view(solution_id): +def view(solution_id: int, file_id: Optional[int]): solution = Solution.get_or_none(Solution.id == solution_id) if solution is None: return fail(404, 'Solution does not exist.') @@ -365,8 +369,10 @@ def view(solution_id): versions = solution.ordered_versions() test_results = solution.test_results() is_manager = current_user.role.is_manager + view_params = { 'solution': model_to_dict(solution), + 'files': solution.files, 'is_manager': is_manager, 'role': current_user.role.name.lower(), 'versions': versions, @@ -418,22 +424,28 @@ def _common_comments(exercise_id=None, user_id=None): query = CommentText.filter(**{ CommentText.flake8_key.name: None, }).select(CommentText.id, CommentText.text).join(Comment) + if exercise_id is not None: - query = (query - .join(Solution) - .join(Exercise) - .where(Exercise.id == exercise_id) - ) + query = ( + query + .join(SolutionFile) + .join(Solution) + .join(Exercise) + .where(Exercise.id == exercise_id) + ) + if user_id is not None: - query = (query - .filter(Comment.commenter == user_id) - ) - - query = (query - .group_by(CommentText.id) - .order_by(fn.Count(CommentText.id).desc()) - .limit(5) - ) + query = ( + query + .filter(Comment.commenter == user_id) + ) + + query = ( + query + .group_by(CommentText.id) + .order_by(fn.Count(CommentText.id).desc()) + .limit(5) + ) return tuple(query.dicts()) diff --git a/lms/models/upload.py b/lms/models/upload.py new file mode 100644 index 00000000..8164570c --- /dev/null +++ b/lms/models/upload.py @@ -0,0 +1,10 @@ +import hashlib + + +def create_hash(file_content: bytes) -> str: + if not isinstance(file_content, bytes): + file_content = file_content.encode('utf-8') + + hashed_code = hashlib.blake2b(digest_size=20) + hashed_code.update(file_content) + return hashed_code.hexdigest() diff --git a/lms/templates/navbar.html b/lms/templates/navbar.html index 44c55545..a7c0981b 100644 --- a/lms/templates/navbar.html +++ b/lms/templates/navbar.html @@ -15,7 +15,7 @@ 0
-
-
- 
{{- solution['json_data_str'] | trim(' ') -}}
-
- {% if test_results %} -
-

בדיקות אוטומטיות

-
    - {%- for test_result in test_results %} -
  1. -
    - {{ test_result.pretty_test_name }} -
    - שגיאה: - - 
    {{ test_result.user_message }}
    -     - {% if is_manager %} - שגיאת סגל: - - 
    {{ test_result.staff_message }}
    -     - {% endif %} -
    2. - {% endfor -%} -
-
- {% endif %} + {%- for file in files %} +
+
+ 
{{- file.code | trim(' ') -}}
+ {% if test_results %} +
+

בדיקות אוטומטיות

+
    + {%- for test_result in test_results %} +
  1. +
    + {{ test_result.pretty_test_name }} +
    + שגיאה: + + 
    {{ test_result.user_message }}
    +     + {% if is_manager %} + שגיאת סגל: + + 
    {{ test_result.staff_message }}
    +     + {% endif %} +
    2. + {% endfor -%} +
+
+ {% endif %} +
+ {% endfor -%} {% if is_manager %}