From 3d90b939d7f2ce43aaf430e9afdc9f045140a44e Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Fri, 1 Jun 2018 16:44:15 -0700 Subject: [PATCH 01/60] Fixed get_key docstring. --- server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server.py b/server.py index 517ffff..a186b42 100644 --- a/server.py +++ b/server.py @@ -272,9 +272,7 @@ def get_key(key): """ This is the key parser for the course names - :param campus: (str) The campus to retrieve data from :param key: (str) The unparsed string containing the course name - :return match_obj.groups(): (list) the string for the regex match """ k = key.split(' ') From e3d141c6d0155c52c1ecb4322ac573894165ccd1 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sat, 2 Jun 2018 13:33:13 -0700 Subject: [PATCH 02/60] Minor cleanup in server.py --- server.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/server.py b/server.py index a186b42..3111de1 100644 --- a/server.py +++ b/server.py @@ -10,18 +10,23 @@ from tinydb import TinyDB from maya import when, MayaInterval + # Quart config def add_cors_headers(response): response.headers['Access-Control-Allow-Origin'] = '*' return response -application = Flask(__name__, - template_folder="../frontend/templates", static_folder='../frontend/static') + +application = Flask( + __name__, + template_folder="../frontend/templates", + static_folder='../frontend/static' +) application.after_request(add_cors_headers) DB_ROOT = 'db/' -CAMPUS_LIST = {'fh':'201911', 'da':'201912', 'test':'test'} +CAMPUS_LIST = {'fh': '201911', 'da': '201912', 'test': 'test'} COURSE_PATTERN = r'[FD]0*(\d*\w?)\.?\d*([YWZH])?' DAYS_PATTERN = f"^{'(M|T|W|Th|F|S|U)?'*7}$" @@ -115,7 +120,8 @@ def api_many(campus): courses = get_many(db=db, data=data, filters=filters) if not courses: # null case from get_one (invalid param or filter) - return 'Error! Could not find one or more course selectors in database', 404 + return 'Error! Could not find one or more course ' \ + 'selectors in database', 404 json = jsonify({'courses': courses}) return json, 200 @@ -175,9 +181,6 @@ def filter_courses(filters: ty.Dict[str, ty.Any], course): This is a helper called by get_one() that filters a set of classes based on some filter conditionals - This is a helper called by get_one() that filters a set of classes - based on some filter conditionals - Be careful with these as they can be limiting on the data, often returning as 404 when one of the courses does not pass the filter. Additionally, filters like status and types can be extremely From 3ea2e631384857326d9737bb1065b7af91ccb61b Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sat, 2 Jun 2018 13:40:15 -0700 Subject: [PATCH 03/60] Began implementing DataModel as a distinct class. --- model/__init__.py | 0 model/model.py | 270 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 model/__init__.py create mode 100644 model/model.py diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/model/model.py b/model/model.py new file mode 100644 index 0000000..b65c286 --- /dev/null +++ b/model/model.py @@ -0,0 +1,270 @@ +""" +This module contains the data model used by server.py. +""" +import os +import tinydb +import typing as ty +import string +import weakref + +DB_EXT = '.json' +SCHOOL_NAMES_BY_CODE = { + '1': 'FH', + '2': 'DA' +} +QUARTER_NAMES_BY_CODE = { + 1: 'summer', + 2: 'fall', + 3: 'winter', + 4: 'spring' +} + +# Type aliases + +COURSE_FIELD_T = str +COURSE_VALUE_T = str +CRN_T = str +COURSE_ID_T = str +DEPT_ID_T = str + +SECTION_ENTRY_T = ty.Dict[COURSE_FIELD_T, COURSE_VALUE_T] +SECTION_DATA_T = ty.List[SECTION_ENTRY_T] +COURSE_DATA_T = ty.Dict[CRN_T, SECTION_DATA_T] +DEPT_DATA_T = ty.Dict[str, ty.Dict[COURSE_ID_T, COURSE_DATA_T]] +# todo: Identify this ^^^ str here and what it means. +QTR_DATA_T = ty.Dict[DEPT_ID_T, DEPT_DATA_T] + + +class DataModel: + def __init__(self, db_dir: str): + """ + Initializer for DataModel that uses the passed db_dir path + to find database tables and store cached values. + :param db_dir: str path to directory. + """ + if not os.path.isdir(db_dir): + raise ValueError(f'Passed path is not a directory: {db_dir}') + self.db_dir: str = db_dir + self.quarter_instances = weakref.WeakValueDictionary() + + def register_quarter(self, quarter: QuarterView): + """ + Stores weak reference to passed quarter so that in the future, + the program can avoid creating duplicates of the same quarter. + :param quarter: QuarterView + :return: None + """ + if quarter.model is not self: + raise ValueError( + f'Passed quarter {quarter} has model {quarter.model}, ' + f'cannot register it with {self}') + if quarter.name in self.quarter_instances: + raise ValueError(f'Passed quarter {quarter} is a duplicate of ' + f'an existing quarter in {self}.') + self.quarter_instances[quarter.name] = quarter + + @property + def quarters(self) -> ty.Iterable[QuarterView]: + """ + Returns generator yielding quarter + :return: + """ + for file_name in os.listdir(self.db_dir): + name, ext = os.path.splitext(file_name) + if ext != DB_EXT: + continue # Ignore non-database files + if all(c in string.digits for c in name): + yield QuarterView.get_quarter(self, name) + + def __repr__(self) -> str: + return f'DataModel[{self.db_dir}]' + + +# The reason that the classes below are considered views is that they +# do not contain their own data, and multiple instances may be able to +# be created that all refer to the same data in the model. + + +class SchoolView: + """ + Represents a view onto data for a specific school. + """ + def __init__(self, model: DataModel, name: str): + if name not in SCHOOL_NAMES_BY_CODE.values(): + raise ValueError(f'Unexpected name received: {name} expected ' + f'name in {SCHOOL_NAMES_BY_CODE.values()}') + self.model = model + self.name = name + + @property + def quarters(self) -> ty.Iterable[QuarterView]: + """ + Iterates over and yields views for each quarter that is + associated with school. + :return: QuarterView + """ + for quarter in self.model.quarters: + if quarter.school == self.name: + yield quarter + + def __repr__(self) -> str: + return f'SchoolView[{self.name}]' + + +class QuarterView: + """ + Represents a view of a specific quarter, for a specific school. + """ + # In order to avoid loading into memory many duplicates of the same + # database file, QuarterViews will throw an exception if a + # duplicate QuarterView is created using the same model + name. + # + # To get a QuarterView of a specific quarter, in a specific model, + # get_quarter() factory method is intended to be used, rather than + # the constructor. + # + # The get_quarter() method will check if the model has a previously + # stored weak reference to a QuarterView with the same name, and + # return it if it exists. Otherwise, it creates a new QuarterView, + # which during its __init__ method, registers itself with + # the model. + + def __init__(self, model: DataModel, name: str): + """ + Instantiates a new QuarterView. + When instantiated, QuarterView will attempt to register + itself with its associated model. If a duplicate QuarterView + exists (with the same model and name) a ValueError is raised. + :param model: DataModel + :param name: str + :raises ValueError if QuarterView is a duplicate. + """ + self.model: DataModel = model + self.name: str = name + self._db: tinydb.TinyDB or None = None # Loaded lazily (use .db prop) + + # sanity check + if self.quarter_number not in range(1, 5): # 1, 2, 3, or 4 + raise ValueError(f'Invalid Quarter number: {self.quarter_number}' + f'for {self}') + + # Register QuarterView with model. If QuarterView is a + # duplicate, then something has gone wrong, and an exception + # is raised by the method. + model.register_quarter(self) + + # This factory method might equally have been implemented as an + # overwritten __new__ method, for more concise instantiation, but + # the potential for confusion and unintuitive behavior is + # considered to outweigh the benefit at the time of writing. + # This balance may change in the future. + @classmethod + def get_quarter(cls, model: DataModel, name: str) -> 'QuarterView': + """ + Returns the quarter of the passed name in the passed model. + If a QuarterView exists that has already been instantiated + with the passed name, the pre-existing QuarterView will be + returned. + Otherwise, a newly created QuarterView will be returned. + :param model: DataModel + :param name: str + :return: QuarterView + """ + try: + return model.quarter_instances[name] + except KeyError: + return cls(model, name) + + @property + def db(self) -> tinydb.TinyDB: + """ + Database getter; + Database will only be created when this property is accessed, + and then stored for future uses. + :return: TinyDB + """ + if not self._db: + if not os.path.exists(self.path): + raise ValueError( + f'Path does not exist for {self} in {self.model}') + self._db = tinydb.TinyDB(self.path) + return self._db + + @property + def year(self) -> int: + """ + Returns school year that quarter occurs in. + The school year starts with summer and ends with the end of + spring, and the number represents the calendar year in which + spring occurs. + :return: str + """ + return int(self.name[:4]) + + @property + def quarter_number(self) -> int: + """ + Returns integer indicating which quarter of the year the + quarter is. + 1: Summer + 2: Fall + 3: Winter + 4: Spring + :return: int in range(1, 4) inclusive. + """ + return int(self.name[4]) + + @property + def quarter_name(self) -> str: + """ + Returns name of the quarter of the year in which the + quarter occurs. + 1: Summer + 2: Fall + 3: Winter + 4: Spring + :return: str + """ + return QUARTER_NAMES_BY_CODE[self.quarter_number] + + @property + def school_name(self) -> str: + """ + Gets name of school as str. + :return: str 'FH' or 'DA' + """ + return SCHOOL_NAMES_BY_CODE[self.name[5]] + + @property + def school(self) -> SchoolView: + return SchoolView(self.model, self.name) + + @property + def path(self) -> str: + return os.path.join(self.model.db_dir, self.name) + + def __getitem__(self, dept_name: str) -> DepartmentView: + return self.db + + def __repr__(self) -> str: + return f'QuarterView[{self.name}]' + + +class DepartmentView: + """ + View onto data of a specific department + """ + + def __init__(self, model: DataModel, name: str, data: DEPT_DATA_T): + self.model = model + self.name = name + self.data = data + + +class InstructorView: + """ + Class handling access of instructor data. + """ + def __init__(self, model: DataModel, name: str): + self.model: DataModel = model + self.name: str = name From 4604b5d18dc572e7913e90dc493e29fb348c9d5d Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sat, 2 Jun 2018 14:51:11 -0700 Subject: [PATCH 04/60] Added initial tests for DataModel and reorganized test data. --- model/model.py | 12 +-- .../test_database.json | 0 tests/generate_test_data.py | 83 ++++++++++++++----- tests/model/__init__.py | 0 tests/model/test_model.py | 18 ++++ 5 files changed, 86 insertions(+), 27 deletions(-) rename {tests/test_db => test_resources}/test_database.json (100%) create mode 100644 tests/model/__init__.py create mode 100644 tests/model/test_model.py diff --git a/model/model.py b/model/model.py index b65c286..e58dcaf 100644 --- a/model/model.py +++ b/model/model.py @@ -46,8 +46,10 @@ def __init__(self, db_dir: str): raise ValueError(f'Passed path is not a directory: {db_dir}') self.db_dir: str = db_dir self.quarter_instances = weakref.WeakValueDictionary() + self.schools = {quarter.school_name: quarter.school + for quarter in self.quarters} - def register_quarter(self, quarter: QuarterView): + def register_quarter(self, quarter: 'QuarterView'): """ Stores weak reference to passed quarter so that in the future, the program can avoid creating duplicates of the same quarter. @@ -64,7 +66,7 @@ def register_quarter(self, quarter: QuarterView): self.quarter_instances[quarter.name] = quarter @property - def quarters(self) -> ty.Iterable[QuarterView]: + def quarters(self) -> ty.Iterable['QuarterView']: """ Returns generator yielding quarter :return: @@ -97,7 +99,7 @@ def __init__(self, model: DataModel, name: str): self.name = name @property - def quarters(self) -> ty.Iterable[QuarterView]: + def quarters(self) -> ty.Iterable['QuarterView']: """ Iterates over and yields views for each quarter that is associated with school. @@ -237,13 +239,13 @@ def school_name(self) -> str: @property def school(self) -> SchoolView: - return SchoolView(self.model, self.name) + return SchoolView(self.model, self.school_name) @property def path(self) -> str: return os.path.join(self.model.db_dir, self.name) - def __getitem__(self, dept_name: str) -> DepartmentView: + def __getitem__(self, dept_name: str) -> 'DepartmentView': return self.db def __repr__(self) -> str: diff --git a/tests/test_db/test_database.json b/test_resources/test_database.json similarity index 100% rename from tests/test_db/test_database.json rename to test_resources/test_database.json diff --git a/tests/generate_test_data.py b/tests/generate_test_data.py index c09ced0..5a865ab 100644 --- a/tests/generate_test_data.py +++ b/tests/generate_test_data.py @@ -1,37 +1,76 @@ import os +import shutil + from tinydb import TinyDB -TEST_DB_DIR = os.path.join(os.path.dirname(__file__), 'test_db') -test_database = TinyDB(os.path.join(TEST_DB_DIR, 'test_database.json')) +# No guarantees can be made about .path when this script is used, so +# use of settings.py is somewhat difficult. +TESTS_DIR = os.path.join(os.path.dirname(__file__)) +TEST_DB_DIR = os.path.join(TESTS_DIR, 'test_db') +TEST_RESOURCES_DIR = os.path.abspath( + os.path.join(TESTS_DIR, '..', 'test_resources')) +TEST_DATABASE_PATH = os.path.join(TEST_RESOURCES_DIR, 'test_database.json') +test_database = TinyDB(TEST_DATABASE_PATH) + + +def generate_data_py(): + with open(os.path.join(TEST_DB_DIR, 'data.py'), 'w') as file: + # test_sample_url_can_be_generated + data = {'dept': 'test_dept', 'course': 'test_course'} + + file.write(f"test_sample_url_can_be_generated_data = {data}\n") + + # test_get_one_dept + data = {'dept': 'CS'} + + dept = test_database.table(f"{data['dept']}").all() + + file.write(f"test_get_one_dept_data = {dept}\n") + + # test_get_one_dept_and_course + data = {'dept': 'CS', 'course': '2A'} + + dept = test_database.table(f"{data['dept']}").all() -with open(os.path.join(TEST_DB_DIR, 'data.py'), 'w') as file: - # test_sample_url_can_be_generated - data = {'dept': 'test_dept', 'course': 'test_course'} + course = next((e[f"{data['course']}"] for e in dept if + f"{data['course']}" in e)) - file.write(f"test_sample_url_can_be_generated_data = {data}\n") + file.write(f"test_get_one_dept_and_course_data = {course}\n") - # test_get_one_dept - data = {'dept': 'CS'} + # test_get_two_dept + data = {'courses': [{'dept': 'CS'}, {'dept': 'MATH'}]} - dept = test_database.table(f"{data['dept']}").all() + depts = [] + for i in data['courses']: + depts.append(test_database.table(i['dept']).all()) - file.write(f"test_get_one_dept_data = {dept}\n") + file.write(f"test_get_two_dept_data = {depts}\n") - # test_get_one_dept_and_course - data = {'dept': 'CS', 'course': '2A'} - dept = test_database.table(f"{data['dept']}").all() +def generate_data_model_test_files(): + copies = { + 'model_test_dir_a': ( + '000011', + '000012', + '000021', + ) + } + # create directories + for dir_ in copies: + abs_dir = os.path.join(TEST_RESOURCES_DIR, dir_) + os.makedirs(abs_dir, exist_ok=True) # Create path if needed. - course = next((e[f"{data['course']}"] for e in dept if f"{data['course']}" in e)) + # create test data copies + for copy_name in copies[dir_]: + destination = os.path.join(TEST_RESOURCES_DIR, dir_, copy_name) + destination += '.json' + shutil.copyfile(TEST_DATABASE_PATH, destination) - file.write(f"test_get_one_dept_and_course_data = {course}\n") - # test_get_two_dept - data = {'courses': [{'dept': 'CS'}, {'dept': 'MATH'}]} - filters = dict() +def generate_all(): + generate_data_py() + generate_data_model_test_files() - depts = [] - for i in data['courses']: - depts.append(test_database.table(i['dept']).all()) - file.write(f"test_get_two_dept_data = {depts}\n") +if __name__ == '__main__': + generate_all() diff --git a/tests/model/__init__.py b/tests/model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/model/test_model.py b/tests/model/test_model.py new file mode 100644 index 0000000..51dc468 --- /dev/null +++ b/tests/model/test_model.py @@ -0,0 +1,18 @@ +import os + +from unittest import TestCase + +from model.model import DataModel, QuarterView +from tests.generate_test_data import TEST_RESOURCES_DIR + + +class TestDataModel(TestCase): + def test_model_finds_all_tables(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + self.assertEqual(3, len([quarter for quarter in data.quarters])) + + def test_model_contains_correct_schools(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + self.assertEqual(2, len(data.schools)) + self.assertIn('FH', data.schools) + self.assertIn('DA', data.schools) From 87301d501fe993aed1c11cdf0421f665266055dd Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sat, 2 Jun 2018 20:21:05 -0700 Subject: [PATCH 05/60] Began implementing Departments + assorted fixes. --- {model => owl}/__init__.py | 0 {model => owl}/model.py | 120 +++++++++++++++++++++++++----------- settings.py | 2 +- tests/generate_test_data.py | 4 +- tests/model/test_model.py | 10 ++- tests/test_server.py | 2 +- 6 files changed, 97 insertions(+), 41 deletions(-) rename {model => owl}/__init__.py (100%) rename {model => owl}/model.py (67%) diff --git a/model/__init__.py b/owl/__init__.py similarity index 100% rename from model/__init__.py rename to owl/__init__.py diff --git a/model/model.py b/owl/model.py similarity index 67% rename from model/model.py rename to owl/model.py index e58dcaf..d7abb94 100644 --- a/model/model.py +++ b/owl/model.py @@ -46,8 +46,36 @@ def __init__(self, db_dir: str): raise ValueError(f'Passed path is not a directory: {db_dir}') self.db_dir: str = db_dir self.quarter_instances = weakref.WeakValueDictionary() + self.schools = {} # Populated by generate_data() + self.quarters = {} # Populated by generate_data() + + self.generate_data() + + def generate_data(self): + """ + Generates basic data for DataModel that is kept in memory for + duration of program or until generate_data is called again to + update stored data. + :return: None + """ + + def find_quarters() -> ty.Dict[str, 'QuarterView']: + """ + Returns dictionary of quarters by name. + :return: Dict[str code, QuarterView] + """ + quarters: ty.Dict[str, 'QuarterView'] = {} + for file_name in os.listdir(self.db_dir): + name, ext = os.path.splitext(file_name) + if ext != DB_EXT: + continue # Ignore non-database files + if all(c in string.digits for c in name): + quarters[name] = QuarterView.get_quarter(self, name) + return quarters + + self.quarters = find_quarters() self.schools = {quarter.school_name: quarter.school - for quarter in self.quarters} + for quarter in self.quarters.values()} def register_quarter(self, quarter: 'QuarterView'): """ @@ -58,33 +86,20 @@ def register_quarter(self, quarter: 'QuarterView'): """ if quarter.model is not self: raise ValueError( - f'Passed quarter {quarter} has model {quarter.model}, ' + f'Passed quarter {quarter} has owl {quarter.owl}, ' f'cannot register it with {self}') if quarter.name in self.quarter_instances: raise ValueError(f'Passed quarter {quarter} is a duplicate of ' f'an existing quarter in {self}.') self.quarter_instances[quarter.name] = quarter - @property - def quarters(self) -> ty.Iterable['QuarterView']: - """ - Returns generator yielding quarter - :return: - """ - for file_name in os.listdir(self.db_dir): - name, ext = os.path.splitext(file_name) - if ext != DB_EXT: - continue # Ignore non-database files - if all(c in string.digits for c in name): - yield QuarterView.get_quarter(self, name) - def __repr__(self) -> str: return f'DataModel[{self.db_dir}]' # The reason that the classes below are considered views is that they # do not contain their own data, and multiple instances may be able to -# be created that all refer to the same data in the model. +# be created that all refer to the same data in the owl. class SchoolView: @@ -99,15 +114,14 @@ def __init__(self, model: DataModel, name: str): self.name = name @property - def quarters(self) -> ty.Iterable['QuarterView']: + def quarters(self) -> ty.Dict[str, 'QuarterView']: """ Iterates over and yields views for each quarter that is associated with school. :return: QuarterView """ - for quarter in self.model.quarters: - if quarter.school == self.name: - yield quarter + return {name: quarter for name, quarter in self.model.quarters if + quarter.school_name == self.name} def __repr__(self) -> str: return f'SchoolView[{self.name}]' @@ -119,24 +133,24 @@ class QuarterView: """ # In order to avoid loading into memory many duplicates of the same # database file, QuarterViews will throw an exception if a - # duplicate QuarterView is created using the same model + name. + # duplicate QuarterView is created using the same owl + name. # - # To get a QuarterView of a specific quarter, in a specific model, + # To get a QuarterView of a specific quarter, in a specific owl, # get_quarter() factory method is intended to be used, rather than # the constructor. # - # The get_quarter() method will check if the model has a previously + # The get_quarter() method will check if the owl has a previously # stored weak reference to a QuarterView with the same name, and # return it if it exists. Otherwise, it creates a new QuarterView, # which during its __init__ method, registers itself with - # the model. + # the owl. def __init__(self, model: DataModel, name: str): """ Instantiates a new QuarterView. When instantiated, QuarterView will attempt to register - itself with its associated model. If a duplicate QuarterView - exists (with the same model and name) a ValueError is raised. + itself with its associated owl. If a duplicate QuarterView + exists (with the same owl and name) a ValueError is raised. :param model: DataModel :param name: str :raises ValueError if QuarterView is a duplicate. @@ -150,7 +164,7 @@ def __init__(self, model: DataModel, name: str): raise ValueError(f'Invalid Quarter number: {self.quarter_number}' f'for {self}') - # Register QuarterView with model. If QuarterView is a + # Register QuarterView with owl. If QuarterView is a # duplicate, then something has gone wrong, and an exception # is raised by the method. model.register_quarter(self) @@ -163,7 +177,7 @@ def __init__(self, model: DataModel, name: str): @classmethod def get_quarter(cls, model: DataModel, name: str) -> 'QuarterView': """ - Returns the quarter of the passed name in the passed model. + Returns the quarter of the passed name in the passed owl. If a QuarterView exists that has already been instantiated with the passed name, the pre-existing QuarterView will be returned. @@ -188,7 +202,7 @@ def db(self) -> tinydb.TinyDB: if not self._db: if not os.path.exists(self.path): raise ValueError( - f'Path does not exist for {self} in {self.model}') + f'Path does not exist for {self} in {self.owl}') self._db = tinydb.TinyDB(self.path) return self._db @@ -243,25 +257,59 @@ def school(self) -> SchoolView: @property def path(self) -> str: - return os.path.join(self.model.db_dir, self.name) + return os.path.join(self.model.db_dir, self.name) + DB_EXT - def __getitem__(self, dept_name: str) -> 'DepartmentView': - return self.db + @property + def departments(self): + return self.Departments(self) def __repr__(self) -> str: return f'QuarterView[{self.name}]' + class Departments: + """ + Helper class that handles access of department data. + """ + def __init__(self, quarter: 'QuarterView'): + self.quarter = quarter + + def __getitem__(self, dept_name: str) -> 'DepartmentQuarterView': + # screen department names that might otherwise access + # internal tables, defaults, etc. + if any(char not in string.ascii_letters for char in dept_name): + raise ValueError( + f'Invalid department name passed: {dept_name}') + if dept_name not in self.db.tables(): + raise ValueError( + f'Passed department name: {dept_name} does not' + f'exist in {self}.') + + dept_data: DEPT_DATA_T = self.db.table(dept_name).all() + return DepartmentQuarterView(self.quarter, dept_name, dept_data) + + @property + def db(self): + return self.quarter.db -class DepartmentView: + +class DepartmentQuarterView: """ - View onto data of a specific department + View onto data of a department's data for a specific quarter. """ - def __init__(self, model: DataModel, name: str, data: DEPT_DATA_T): - self.model = model + def __init__(self, quarter: QuarterView, name: str, data: DEPT_DATA_T): + self.quarter = quarter self.name = name self.data = data + # Sanity Check + if not 2 <= len(name) <= 4: + raise Warning(f'Odd department name received: {name}') + + @property + def model(self): + return self.quarter.model + class InstructorView: """ diff --git a/settings.py b/settings.py index df60686..5c25641 100644 --- a/settings.py +++ b/settings.py @@ -4,4 +4,4 @@ API_DIR = os.path.join(ROOT_DIR, 'owlapi') DB_DIR = os.path.join(ROOT_DIR, 'db') TEST_DIR = os.path.join(ROOT_DIR, 'tests') -TEST_DB_DIR = os.path.join(TEST_DIR, 'test_db') +TEST_RESOURCES_DIR = os.path.join(ROOT_DIR, 'test_resources') diff --git a/tests/generate_test_data.py b/tests/generate_test_data.py index 5a865ab..ab3b9a1 100644 --- a/tests/generate_test_data.py +++ b/tests/generate_test_data.py @@ -6,7 +6,7 @@ # No guarantees can be made about .path when this script is used, so # use of settings.py is somewhat difficult. TESTS_DIR = os.path.join(os.path.dirname(__file__)) -TEST_DB_DIR = os.path.join(TESTS_DIR, 'test_db') +TEST_DATA_DIR = os.path.join(TESTS_DIR, 'test_db') TEST_RESOURCES_DIR = os.path.abspath( os.path.join(TESTS_DIR, '..', 'test_resources')) TEST_DATABASE_PATH = os.path.join(TEST_RESOURCES_DIR, 'test_database.json') @@ -14,7 +14,7 @@ def generate_data_py(): - with open(os.path.join(TEST_DB_DIR, 'data.py'), 'w') as file: + with open(os.path.join(TEST_DATA_DIR, 'data.py'), 'w') as file: # test_sample_url_can_be_generated data = {'dept': 'test_dept', 'course': 'test_course'} diff --git a/tests/model/test_model.py b/tests/model/test_model.py index 51dc468..56cee13 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -2,7 +2,7 @@ from unittest import TestCase -from model.model import DataModel, QuarterView +from owl.model import DataModel, QuarterView from tests.generate_test_data import TEST_RESOURCES_DIR @@ -16,3 +16,11 @@ def test_model_contains_correct_schools(self): self.assertEqual(2, len(data.schools)) self.assertIn('FH', data.schools) self.assertIn('DA', data.schools) + + def test_department_view_can_be_retrieved_from_quarter(self): + # This test doesn't help much, but at least confirms that no + # errors are thrown. + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + self.assertEqual('ACTG', dept.name) diff --git a/tests/test_server.py b/tests/test_server.py index 9c26353..a68625d 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -14,7 +14,7 @@ 'does not exist, it can be generated using the ' 'generate_test_data.py script') from e -test_database = TinyDB(join(settings.TEST_DB_DIR, 'test_database.json')) +test_database = TinyDB(join(settings.TEST_RESOURCES_DIR, 'test_database.json')) class TestGenerateURL(TestCase): From 6928cb585ea441831edaa0d942c060bacf192c21 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sat, 2 Jun 2018 20:25:39 -0700 Subject: [PATCH 06/60] Fixed owl infestation. --- owl/model.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/owl/model.py b/owl/model.py index d7abb94..b7cf834 100644 --- a/owl/model.py +++ b/owl/model.py @@ -86,7 +86,7 @@ def register_quarter(self, quarter: 'QuarterView'): """ if quarter.model is not self: raise ValueError( - f'Passed quarter {quarter} has owl {quarter.owl}, ' + f'Passed quarter {quarter} has model {quarter.model}, ' f'cannot register it with {self}') if quarter.name in self.quarter_instances: raise ValueError(f'Passed quarter {quarter} is a duplicate of ' @@ -99,7 +99,7 @@ def __repr__(self) -> str: # The reason that the classes below are considered views is that they # do not contain their own data, and multiple instances may be able to -# be created that all refer to the same data in the owl. +# be created that all refer to the same data in the model. class SchoolView: @@ -133,24 +133,24 @@ class QuarterView: """ # In order to avoid loading into memory many duplicates of the same # database file, QuarterViews will throw an exception if a - # duplicate QuarterView is created using the same owl + name. + # duplicate QuarterView is created using the same model + name. # - # To get a QuarterView of a specific quarter, in a specific owl, + # To get a QuarterView of a specific quarter, in a specific model, # get_quarter() factory method is intended to be used, rather than # the constructor. # - # The get_quarter() method will check if the owl has a previously + # The get_quarter() method will check if the model has a previously # stored weak reference to a QuarterView with the same name, and # return it if it exists. Otherwise, it creates a new QuarterView, # which during its __init__ method, registers itself with - # the owl. + # the model. def __init__(self, model: DataModel, name: str): """ Instantiates a new QuarterView. When instantiated, QuarterView will attempt to register - itself with its associated owl. If a duplicate QuarterView - exists (with the same owl and name) a ValueError is raised. + itself with its associated model. If a duplicate QuarterView + exists (with the same model and name) a ValueError is raised. :param model: DataModel :param name: str :raises ValueError if QuarterView is a duplicate. @@ -164,7 +164,7 @@ def __init__(self, model: DataModel, name: str): raise ValueError(f'Invalid Quarter number: {self.quarter_number}' f'for {self}') - # Register QuarterView with owl. If QuarterView is a + # Register QuarterView with model. If QuarterView is a # duplicate, then something has gone wrong, and an exception # is raised by the method. model.register_quarter(self) @@ -177,7 +177,7 @@ def __init__(self, model: DataModel, name: str): @classmethod def get_quarter(cls, model: DataModel, name: str) -> 'QuarterView': """ - Returns the quarter of the passed name in the passed owl. + Returns the quarter of the passed name in the passed model. If a QuarterView exists that has already been instantiated with the passed name, the pre-existing QuarterView will be returned. @@ -202,7 +202,7 @@ def db(self) -> tinydb.TinyDB: if not self._db: if not os.path.exists(self.path): raise ValueError( - f'Path does not exist for {self} in {self.owl}') + f'Path does not exist for {self} in {self.model}') self._db = tinydb.TinyDB(self.path) return self._db @@ -311,6 +311,21 @@ def model(self): return self.quarter.model +class CourseQuarterView: + """ + View onto course data for a single quarter. + """ + def __init__( + self, + department: DepartmentQuarterView, + name: str, + data: COURSE_DATA_T + ): + self.department = department + self.name = name + self.data = data + + class InstructorView: """ Class handling access of instructor data. From 3d329dffd260167fa74655a3ec627f0e0b54cc7e Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sat, 2 Jun 2018 21:02:32 -0700 Subject: [PATCH 07/60] Added SectionQuarterView and other fixes. --- owl/model.py | 62 ++++++++++++++++++++++++++++++++++----- tests/model/test_model.py | 19 +++++++++++- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/owl/model.py b/owl/model.py index b7cf834..eeec84d 100644 --- a/owl/model.py +++ b/owl/model.py @@ -30,9 +30,7 @@ SECTION_ENTRY_T = ty.Dict[COURSE_FIELD_T, COURSE_VALUE_T] SECTION_DATA_T = ty.List[SECTION_ENTRY_T] COURSE_DATA_T = ty.Dict[CRN_T, SECTION_DATA_T] -DEPT_DATA_T = ty.Dict[str, ty.Dict[COURSE_ID_T, COURSE_DATA_T]] -# todo: Identify this ^^^ str here and what it means. -QTR_DATA_T = ty.Dict[DEPT_ID_T, DEPT_DATA_T] +DEPT_DATA_T = ty.Dict[COURSE_ID_T, COURSE_DATA_T] class DataModel: @@ -260,7 +258,7 @@ def path(self) -> str: return os.path.join(self.model.db_dir, self.name) + DB_EXT @property - def departments(self): + def departments(self) -> 'Departments': return self.Departments(self) def __repr__(self) -> str: @@ -284,11 +282,11 @@ def __getitem__(self, dept_name: str) -> 'DepartmentQuarterView': f'Passed department name: {dept_name} does not' f'exist in {self}.') - dept_data: DEPT_DATA_T = self.db.table(dept_name).all() + dept_data: DEPT_DATA_T = self.db.table(dept_name).all()[0] return DepartmentQuarterView(self.quarter, dept_name, dept_data) @property - def db(self): + def db(self) -> tinydb.TinyDB: return self.quarter.db @@ -297,7 +295,7 @@ class DepartmentQuarterView: View onto data of a department's data for a specific quarter. """ - def __init__(self, quarter: QuarterView, name: str, data: DEPT_DATA_T): + def __init__(self, quarter: 'QuarterView', name: str, data: DEPT_DATA_T): self.quarter = quarter self.name = name self.data = data @@ -310,6 +308,24 @@ def __init__(self, quarter: QuarterView, name: str, data: DEPT_DATA_T): def model(self): return self.quarter.model + @property + def courses(self): + return self.Courses(self) + + class Courses: + """ + Helper class for accessing courses within department. + """ + def __init__(self, department: 'DepartmentQuarterView'): + self.department = department + + def __getitem__(self, course_name: str): + return CourseQuarterView( + self.department, + course_name, + self.department.data[course_name] + ) + class CourseQuarterView: """ @@ -317,7 +333,7 @@ class CourseQuarterView: """ def __init__( self, - department: DepartmentQuarterView, + department: 'DepartmentQuarterView', name: str, data: COURSE_DATA_T ): @@ -325,6 +341,36 @@ def __init__( self.name = name self.data = data + @property + def sections(self): + return self.Sections(self) + + class Sections: + """ + Helper class for accessing sections within Course + """ + def __init__(self, course: 'CourseQuarterView'): + self.course = course + + def __getitem__(self, section_name: str): + return SectionQuarterView( + self.course, + section_name, + self.course.data[section_name] + ) + + +class SectionQuarterView: + def __init__( + self, + course: CourseQuarterView, + name: str, + data: SECTION_DATA_T + ): + self.course = course + self.name = name + self.data = data + class InstructorView: """ diff --git a/tests/model/test_model.py b/tests/model/test_model.py index 56cee13..bc6b488 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -2,7 +2,8 @@ from unittest import TestCase -from owl.model import DataModel, QuarterView +from owl.model import DataModel, DepartmentQuarterView, CourseQuarterView, \ + SectionQuarterView from tests.generate_test_data import TEST_RESOURCES_DIR @@ -24,3 +25,19 @@ def test_department_view_can_be_retrieved_from_quarter(self): quarter = data.quarters['000011'] dept = quarter.departments['ACTG'] self.assertEqual('ACTG', dept.name) + self.assertIsInstance(dept, DepartmentQuarterView) + + def test_courses_are_accessible_from_department(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + self.assertIsInstance(course, CourseQuarterView) + + def test_sections_are_accessible_from_course(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertIsInstance(section, SectionQuarterView) From 537e5e58628383a44c5e0e76847d6ac193153af7 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 3 Jun 2018 01:03:59 -0700 Subject: [PATCH 08/60] Implemented section properties and associated test methods. --- owl/model.py | 260 ++++++++++++++++++++++++++++++++++++-- tests/model/test_model.py | 234 +++++++++++++++++++++++++++++++++- 2 files changed, 483 insertions(+), 11 deletions(-) diff --git a/owl/model.py b/owl/model.py index eeec84d..717f58a 100644 --- a/owl/model.py +++ b/owl/model.py @@ -6,11 +6,14 @@ import typing as ty import string import weakref +import maya DB_EXT = '.json' +FH = 'FH' +DA = 'DA' SCHOOL_NAMES_BY_CODE = { - '1': 'FH', - '2': 'DA' + '1': FH, + '2': DA } QUARTER_NAMES_BY_CODE = { 1: 'summer', @@ -18,6 +21,41 @@ 3: 'winter', 4: 'spring' } +DAYS = 'M', 'T', 'W', 'Th', 'F', 'S', 'U' +DAYS_DECREASING_LEN = sorted(DAYS, key=lambda day: -len(day)) + +STANDARD_TYPE = 'standard' +ONLINE_TYPE = 'online' +HYBRID_TYPE = 'hybrid' + +SCHOOL_TYPE_CODES = { + FH: {'W': ONLINE_TYPE, 'Y': HYBRID_TYPE}, + DA: {'Z': ONLINE_TYPE, 'Y': HYBRID_TYPE} +} + +# Section fields +COURSE_ID_KEY = 'course' +CRN_KEY = 'CRN' +DESCRIPTION_KEY = 'desc' +STATUS_KEY = 'status' +DAYS_KEY = 'days' +TIME_KEY = 'time' +START_KEY = 'start' +END_KEY = 'end' +ROOM_KEY = 'room' +CAMPUS_KEY = 'campus' +UNITS_KEY = 'units' +INSTRUCTOR_KEY = 'instructor' +SEATS_KEY = 'seats' +WAIT_SEATS_KEY = 'wait_seats' +WAIT_CAP_KEY = 'wait_cap' + +# section keywords +ONLINE = 'ONLINE' +OPEN = 'Open' +WAITLIST = 'Waitlist' +FULL = 'Full' +TBA = 'TBA' # Type aliases @@ -33,6 +71,15 @@ DEPT_DATA_T = ty.Dict[COURSE_ID_T, COURSE_DATA_T] +class DataError(Exception): + """ + Raised when accessed data cannot be handled. + Distinct from ValueError because ValueError indicates invalid data + has been passed, while DataError indicates data that has been + retrieved from model. + """ + + class DataModel: def __init__(self, db_dir: str): """ @@ -121,6 +168,10 @@ def quarters(self) -> ty.Dict[str, 'QuarterView']: return {name: quarter for name, quarter in self.model.quarters if quarter.school_name == self.name} + @property + def type_codes(self) -> ty.Dict[str, str]: + return SCHOOL_TYPE_CODES[self.name] + def __repr__(self) -> str: return f'SchoolView[{self.name}]' @@ -345,6 +396,10 @@ def __init__( def sections(self): return self.Sections(self) + def __repr__(self) -> str: + return f'CourseQuarterView[dept: {self.department.name}, ' \ + f'course: {self.name}]' + class Sections: """ Helper class for accessing sections within Course @@ -352,25 +407,210 @@ class Sections: def __init__(self, course: 'CourseQuarterView'): self.course = course - def __getitem__(self, section_name: str): - return SectionQuarterView( - self.course, - section_name, - self.course.data[section_name] - ) + def __getitem__(self, section_name: str) -> 'SectionQuarterView': + # If section cannot be found, raise a more readable + # key error. + try: + data = self.course.data[section_name] + except KeyError as e: + raise KeyError(f'No Section exists in {self} with id: ' + f'{repr(section_name)}') from e + + # Get Section from data + section = SectionQuarterView(self.course, data) + assert section.crn == section_name, \ + f'section crn: {section.crn} does not match that requested: ' \ + + section_name + return section + + def __repr__(self): + return f'{self.course}.Sections' class SectionQuarterView: + # All these fields should be equal if multiple entries exist. + EQUAL_FIELDS = ( + COURSE_ID_KEY, CRN_KEY, DESCRIPTION_KEY, STATUS_KEY, UNITS_KEY, + INSTRUCTOR_KEY, SEATS_KEY, WAIT_SEATS_KEY, WAIT_CAP_KEY + ) + def __init__( self, course: CourseQuarterView, - name: str, data: SECTION_DATA_T ): self.course = course - self.name = name self.data = data + # Sanity check; Ensure assumptions made about data are true + for field_name in self.EQUAL_FIELDS: + if not all(entry[field_name] == self.data[0][field_name] for + entry in self.data): + raise ValueError( + f'{self}: {field_name} fields do not match in data') + + @property + def course_id(self) -> str: + """ + Returns full identity of section, as listed in database under + the key 'course' (a somewhat misleading field name). + :return: str + """ + return self.data[0][COURSE_ID_KEY] + + @property + def crn(self) -> str: + return self.data[0][CRN_KEY] + + @property + def description(self) -> str: + return self.data[0][DESCRIPTION_KEY] + + @property + def section_type(self) -> str: + return self.school.type_codes.get(self.course_id[-1], STANDARD_TYPE) + + @property + def days(self) -> ty.Set[str]: + """ + Gets set of days during which class/section meets. + :return: Set[str] + :raises DataError if non-online entry has 'TBA' or other value + in 'days' field. + """ + days = set() + for entry in self.data: + days |= self._unpack_entry_days(entry) + return days + + def _unpack_entry_days(self, entry: SECTION_ENTRY_T) -> ty.Set[str]: + if entry[ROOM_KEY] == ONLINE: + return set() + s = entry[DAYS_KEY] + if s == TBA: + raise DataError(f'{self} Entry days have not yet been entered') + days = set() + for day_code in DAYS_DECREASING_LEN: + if day_code in s: + s = s.replace(day_code, '') + days.add(day_code) + return days + + @property + def start_date(self) -> maya.MayaDT: + return maya.when(self.data[0][START_KEY]) + + @property + def end_date(self) -> maya.MayaDT: + return maya.when(self.data[0][END_KEY]) + + @property + def durations(self) -> ty.List['ClassDuration']: + """ + Gets list of meeting durations, for times over the week that + a section meets in person. + :return: List[ClassDuration] + :raises DataError if unexpected values, such as 'TBA' are found + in data entries that are not online. + """ + durations = [] + for entry in self.data: + room = entry[ROOM_KEY] + if room == ONLINE: + continue + start_s, end_s = entry[TIME_KEY].split('-') + start, end = maya.when(start_s), maya.when(end_s) + for day in self._unpack_entry_days(entry): + durations.append(ClassDuration(day, room, start, end)) + return durations + + @property + def rooms(self) -> ty.Set['str']: + rooms = set() + [rooms.add(duration.room) for duration in self.durations] + return rooms + + @property + def status(self) -> str: + return self.data[0][STATUS_KEY] + + @property + def campus(self) -> str: + return self.data[0][CAMPUS_KEY] + + @property + def units(self) -> float: + return float(self.data[0][UNITS_KEY]) + + @property + def instructor_name(self) -> str: + return self.data[0][INSTRUCTOR_KEY] + + @property + def open_seats_available(self) -> int: + return int(self.data[0][SEATS_KEY]) + + @property + def waitlist_seats_available(self) -> int: + return int(self.data[0][WAIT_SEATS_KEY]) + + @property + def waitlist_capacity(self) -> int: + return int(self.data[0][WAIT_CAP_KEY]) + + @property + def school_name(self) -> str: + return self.quarter.school_name + + @property + def school(self) -> SchoolView: + return self.quarter.school + + @property + def _online_data(self) -> SECTION_ENTRY_T or None: + for entry in self.data: + if entry[ROOM_KEY] == ONLINE: + return entry + + @property + def _offline_data(self) -> SECTION_ENTRY_T or None: + for entry in self.data: + if entry[ROOM_KEY] != ONLINE: + return entry + + @property + def department(self) -> DepartmentQuarterView: + return self.course.department + + @property + def quarter(self) -> QuarterView: + return self.department.quarter + + @property + def model(self) -> DataModel: + return self.quarter.model + + def __repr__(self) -> str: + return f'SectionQuarterView[cid: {self.course_id}, crn: {self.crn}]' + + +class ClassDuration: + """ + Class storing data about a specific meeting time for a class, on a + specific day. + """ + def __init__( + self, + day: str, + room: str, + start: maya.MayaDT, + end: maya.MayaDT + ): + self.day = day + self.room = room + self.start = start + self.end = end + class InstructorView: """ diff --git a/tests/model/test_model.py b/tests/model/test_model.py index bc6b488..51097eb 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -2,8 +2,11 @@ from unittest import TestCase +import maya + from owl.model import DataModel, DepartmentQuarterView, CourseQuarterView, \ - SectionQuarterView + SectionQuarterView, ClassDuration, \ + OPEN, STANDARD_TYPE, ONLINE_TYPE, HYBRID_TYPE from tests.generate_test_data import TEST_RESOURCES_DIR @@ -41,3 +44,232 @@ def test_sections_are_accessible_from_course(self): course = dept.courses['1A'] section = course.sections['40065'] self.assertIsInstance(section, SectionQuarterView) + + def test_section_course_id_returns_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual('ACTG F001A02Y', section.course_id) + + def test_section_crn_returns_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual('40065', section.crn) + + def test_section_description_returns_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual('FINANCIAL ACCOUNTING I', section.description) + + def test_section_status_returns_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['41130'] + self.assertEqual(OPEN, section.status) + + def test_section_days_returns_correctly_in_semi_online_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual({'T', 'Th'}, section.days) + + def test_section_days_property_raises_exception_in_online_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40018'] + self.assertEqual(set(), section.days) + + def test_section_days_returns_correctly_in_offline_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ENGL'] + course = dept.courses['1A'] + section = course.sections['40140'] + self.assertEqual({'M', 'W'}, section.days) + + def test_section_days_returns_correctly_in_single_day_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['76'] + section = course.sections['41440'] + self.assertEqual({'W'}, section.days) + + def test_section_days_returns_correctly_in_class_with_labs_a(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['2B'] + section = course.sections['40582'] + self.assertEqual({'T', 'Th'}, section.days) + + def test_section_days_returns_correctly_in_class_with_labs_b(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + self.assertEqual({'T', 'Th', 'F'}, section.days) + + def test_section_durations_return_correct_maya_date_times(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + durations = section.durations + self.assertEqual(4, len(durations)) + self.assertIsInstance(durations[0], ClassDuration) + self.assertIsInstance(durations[1], ClassDuration) + self.assertIsInstance(durations[2], ClassDuration) + self.assertIsInstance(durations[3], ClassDuration) + + # Check that each duration has expected times + for duration in durations: + if duration.day == 'Th': + self.assertEqual(maya.when('10:00AM'), duration.start) + self.assertEqual(maya.when('11:50AM'), duration.end) + elif duration.day == 'F': + self.assertEqual(maya.when('11:00 AM'), duration.start) + self.assertEqual(maya.when('11:50AM'), duration.end) + + def test_section_rooms_are_found_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + durations = section.durations + self.assertIsInstance(durations[0], ClassDuration) + self.assertIsInstance(durations[1], ClassDuration) + self.assertIsInstance(durations[2], ClassDuration) + self.assertIsInstance(durations[3], ClassDuration) + + # Check that each duration has expected times + for duration in durations: + if duration.day == 'Th': + self.assertEqual('4501', duration.room) + elif duration.day == 'F': + self.assertEqual('4501', duration.room) + elif duration.day == 'T' and \ + duration.start == maya.when('12:00PM'): + self.assertEqual('4718', duration.room) + + def test_section_rooms_are_found_correctly_for_multi_room_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + self.assertEqual({'4718', '4501'}, section.rooms) + + def test_section_rooms_are_found_correctly_for_single_room_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual({'3201'}, section.rooms) + + def test_section_rooms_are_found_correctly_for_online_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1C'] + section = course.sections['40022'] + self.assertEqual(set(), section.rooms) + + def test_section_end_returns_correct_maya_date(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ENGL'] + course = dept.courses['1A'] + section = course.sections['40140'] + end_date = section.end_date + self.assertIsInstance(end_date, maya.MayaDT) + + def test_section_type_is_determined_correctly_for_offline_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ENGL'] + course = dept.courses['1A'] + section = course.sections['40140'] + self.assertEqual(STANDARD_TYPE, section.section_type) + + def test_section_type_is_determined_correctly_for_online_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40018'] + self.assertEqual(ONLINE_TYPE, section.section_type) + + def test_section_type_is_determined_correctly_for_hybrid_class(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual(HYBRID_TYPE, section.section_type) + + def test_section_campus_is_found_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual('FH', section.campus) + + def test_section_units_are_found_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(5, section.units) + + def test_section_instructor_name_returns_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual('Drake', section.instructor_name) + + def test_section_seats_return_correctly_when_on_waitlist_status(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(0, section.open_seats_available) + + def test_section_wait_seats_return_correctly_when_on_waitlist_status(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(7, section.waitlist_seats_available) + + def test_section_waitlist_capacity_returns_correctly(self): + data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(15, section.waitlist_capacity) From bde366922cec4637f51900baa58404f12dfc245e Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sun, 3 Jun 2018 10:09:38 -0700 Subject: [PATCH 09/60] Updated gitignore to ignore generated test files. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 56ac861..791d0ff 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ frontend/static/README.md .pytest_cache/ tests/test_db/data.py !test_database.json + +# Generated test files +test_resources/data_model_test_dir* From 9a2a529e6852ddcc4e35ffa8bed349ffa92446c2 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sun, 3 Jun 2018 11:01:47 -0700 Subject: [PATCH 10/60] Added more documentation to model.py --- owl/model.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/owl/model.py b/owl/model.py index 717f58a..281e3ca 100644 --- a/owl/model.py +++ b/owl/model.py @@ -170,6 +170,14 @@ def quarters(self) -> ty.Dict[str, 'QuarterView']: @property def type_codes(self) -> ty.Dict[str, str]: + """ + Gets dictionary of course section type codes that are specific + to individual schools. + :return: Dict[ + str (course section id suffix), + str (course section type) + ] + """ return SCHOOL_TYPE_CODES[self.name] def __repr__(self) -> str: @@ -306,10 +314,19 @@ def school(self) -> SchoolView: @property def path(self) -> str: + """ + Gets path to database file which contains quarter data. + :return: str path + """ return os.path.join(self.model.db_dir, self.name) + DB_EXT @property def departments(self) -> 'Departments': + """ + Gets helper class that provides methods for accessing + departments and associated information. + :return: QuarterView.Departments + """ return self.Departments(self) def __repr__(self) -> str: @@ -361,6 +378,11 @@ def model(self): @property def courses(self): + """ + Gets helper class instance for accessing courses + within department. + :return: DepartmentQuarterView.Courses + """ return self.Courses(self) class Courses: @@ -394,6 +416,11 @@ def __init__( @property def sections(self): + """ + Gets helper class instance for accessing sections + within course. + :return: CourseQuarterView.Sections + """ return self.Sections(self) def __repr__(self) -> str: @@ -454,29 +481,59 @@ def course_id(self) -> str: """ Returns full identity of section, as listed in database under the key 'course' (a somewhat misleading field name). + 'Course' field usually contains department 2-4 letter code, + Followed by a number indicating Campus, Course id, + section index, and section type. + + ex: "ACTG F067.03W" + ^^^^ ^^^^ ^^^ + 1111 2333 445 + + 1: Department code + 2: School + 3: Course id + 4: Section index + 5: Section type ('standard', 'hybrid', or 'online) + :return: str """ return self.data[0][COURSE_ID_KEY] @property def crn(self) -> str: + """ + Gets course registration number for course. + :return: str of digits of length 5. + """ return self.data[0][CRN_KEY] @property def description(self) -> str: + """ + Gets description of course. + This usually takes the form of an all-capital course title, + ex: "ADVANCED TAX ACCOUNTING I" + Database spelling may vary. + :return: str description. + """ return self.data[0][DESCRIPTION_KEY] @property def section_type(self) -> str: + """ + Returns type of course section. + :return: str 'standard', 'hybrid' or 'online' + """ return self.school.type_codes.get(self.course_id[-1], STANDARD_TYPE) @property def days(self) -> ty.Set[str]: """ Gets set of days during which class/section meets. - :return: Set[str] + example result: {'T', 'Th', 'F'} :raises DataError if non-online entry has 'TBA' or other value in 'days' field. + :return: Set[str] """ days = set() for entry in self.data: @@ -484,24 +541,55 @@ def days(self) -> ty.Set[str]: return days def _unpack_entry_days(self, entry: SECTION_ENTRY_T) -> ty.Set[str]: + """ + Gets set of days that are listed in passed entry. + example result: {'T', 'Th', 'F'} + :param entry: SECTION_ENTRY_T (Dict[str, str]) + :raises DataError if non-online entry has 'TBA' or other value + in 'days' field. + :return: Set[str] + """ if entry[ROOM_KEY] == ONLINE: + # If entry is online, entry will not have a valid room. + # (usually 'TBA') + # Return empty set, since no in-persons meetings are in + # this entry. return set() s = entry[DAYS_KEY] if s == TBA: + # If days are 'TBA' and class is not online, raise an + # exception rather than trying to handle odd edge case. raise DataError(f'{self} Entry days have not yet been entered') + # Iterate over days in order of longest to shortest, + # removing day codes from temporary string as they are found. + # This is so that a day with a longer code, ex: 'Th' will be + # found and removed from the string before 'T' + # otherwise, the day code 'Th' would produce result: {'T'} and + # then be unable to parse the remaining 'h'. days = set() for day_code in DAYS_DECREASING_LEN: if day_code in s: s = s.replace(day_code, '') days.add(day_code) + if s: + raise DataError(f'Could not parse {entry[DAYS_KEY]}, ' + f'unknown day code: {s})') return days @property def start_date(self) -> maya.MayaDT: + """ + Gets start date of class. + :return: MayaDT + """ return maya.when(self.data[0][START_KEY]) @property def end_date(self) -> maya.MayaDT: + """ + Gets end date of class. + :return: MayaDT + """ return maya.when(self.data[0][END_KEY]) @property @@ -526,44 +614,86 @@ def durations(self) -> ty.List['ClassDuration']: @property def rooms(self) -> ty.Set['str']: + """ + Gets set of rooms (or occasionally other locations) in which + class meets. + :return: Set[str] + """ rooms = set() [rooms.add(duration.room) for duration in self.durations] return rooms @property def status(self) -> str: + """ + Gets status of course section. + :return: str 'OPEN', 'WAITLIST', 'FULL' + """ return self.data[0][STATUS_KEY] @property def campus(self) -> str: + """ + Returns campus code for campus. + (Not the same thing as 'school'). + :return: str (ex: 'FH') + """ return self.data[0][CAMPUS_KEY] @property def units(self) -> float: + """ + Gets number of units of course section. + :return: float + """ return float(self.data[0][UNITS_KEY]) @property def instructor_name(self) -> str: + """ + Gets name of instructor. + :return: str + """ return self.data[0][INSTRUCTOR_KEY] @property def open_seats_available(self) -> int: + """ + Gets number of open seats in course section. + :return: int + """ return int(self.data[0][SEATS_KEY]) @property def waitlist_seats_available(self) -> int: + """ + Gets number of open waitlist seats in course section. + :return: int + """ return int(self.data[0][WAIT_SEATS_KEY]) @property def waitlist_capacity(self) -> int: + """ + Gets overall capacity of waitlist, both open and filled. + :return: int + """ return int(self.data[0][WAIT_CAP_KEY]) @property def school_name(self) -> str: + """ + Gets name of school + :return: str, (ex: 'DA') + """ return self.quarter.school_name @property def school(self) -> SchoolView: + """ + Gets course section's school data view. + :return: SchoolView + """ return self.quarter.school @property @@ -580,14 +710,27 @@ def _offline_data(self) -> SECTION_ENTRY_T or None: @property def department(self) -> DepartmentQuarterView: + """ + Gets view of data for department that course section is under. + :return: DepartmentQuarterView + """ return self.course.department @property def quarter(self) -> QuarterView: + """ + Gets view of data for quarter that course section + occurs within. + :return: QuarterView + """ return self.department.quarter @property def model(self) -> DataModel: + """ + Gets data model. + :return: DataModel + """ return self.quarter.model def __repr__(self) -> str: From d7d64ded9aba3bee11ee3c2aa5ef1cccfe25fd4e Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sun, 3 Jun 2018 11:03:40 -0700 Subject: [PATCH 11/60] Fixed odd occasional test issue by using absolute import. --- tests/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_server.py b/tests/test_server.py index a68625d..700d191 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -8,7 +8,7 @@ # Try to get generated data. try: - from .test_db import data as test_data + from tests.test_db import data as test_data except ImportError as e: raise ImportError('Test Data could not be imported. If the data.py file ' 'does not exist, it can be generated using the ' From 652f550a5ca4826f9e520498f0feac1d40f3aec9 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 3 Jun 2018 11:16:25 -0700 Subject: [PATCH 12/60] Minor debug message improvements, and minor additions. --- owl/model.py | 46 +++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/owl/model.py b/owl/model.py index 717f58a..cc3b23e 100644 --- a/owl/model.py +++ b/owl/model.py @@ -340,6 +340,9 @@ def __getitem__(self, dept_name: str) -> 'DepartmentQuarterView': def db(self) -> tinydb.TinyDB: return self.quarter.db + def __repr__(self) -> str: + return f'{self.quarter}.Departments' + class DepartmentQuarterView: """ @@ -363,6 +366,10 @@ def model(self): def courses(self): return self.Courses(self) + def __repr__(self) -> str: + return f'DepartmentQuarterView[dept: {self.name}, ' \ + f'qtr: {self.quarter.name}]' + class Courses: """ Helper class for accessing courses within department. @@ -371,11 +378,18 @@ def __init__(self, department: 'DepartmentQuarterView'): self.department = department def __getitem__(self, course_name: str): - return CourseQuarterView( - self.department, - course_name, - self.department.data[course_name] - ) + try: + return CourseQuarterView( + self.department, + course_name, + self.department.data[course_name] + ) + except KeyError as e: + raise KeyError(f'No course found named: {course_name} in ' + f'{self}') from e + + def __repr__(self) -> str: + return f'{self.department}.Courses' class CourseQuarterView: @@ -526,6 +540,12 @@ def durations(self) -> ty.List['ClassDuration']: @property def rooms(self) -> ty.Set['str']: + """ + Gets set of rooms in which class meets. + Set will be empty if course is online. + :return: Set[str] + :raises DataError if unexpected values are found in data. + """ rooms = set() [rooms.add(duration.room) for duration in self.durations] return rooms @@ -546,6 +566,10 @@ def units(self) -> float: def instructor_name(self) -> str: return self.data[0][INSTRUCTOR_KEY] + @property + def instructor(self) -> InstructorView: + return InstructorView(self.model, self.instructor_name) + @property def open_seats_available(self) -> int: return int(self.data[0][SEATS_KEY]) @@ -566,18 +590,6 @@ def school_name(self) -> str: def school(self) -> SchoolView: return self.quarter.school - @property - def _online_data(self) -> SECTION_ENTRY_T or None: - for entry in self.data: - if entry[ROOM_KEY] == ONLINE: - return entry - - @property - def _offline_data(self) -> SECTION_ENTRY_T or None: - for entry in self.data: - if entry[ROOM_KEY] != ONLINE: - return entry - @property def department(self) -> DepartmentQuarterView: return self.course.department From ddbe32713aa7eab5e2def3002d7b16f73670e35d Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sun, 3 Jun 2018 12:18:12 -0700 Subject: [PATCH 13/60] Changed import order in model.py to be pep-8 compliant. --- owl/model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/owl/model.py b/owl/model.py index 38a0695..d342325 100644 --- a/owl/model.py +++ b/owl/model.py @@ -2,10 +2,11 @@ This module contains the data model used by server.py. """ import os -import tinydb import typing as ty import string import weakref + +import tinydb import maya DB_EXT = '.json' @@ -672,7 +673,7 @@ def instructor_name(self) -> str: return self.data[0][INSTRUCTOR_KEY] @property - def instructor(self) -> InstructorView: + def instructor(self) -> 'InstructorView': """ Gets data view of instructor for this course section. :return: InstanceView From f6885700e67d8923757b0039d60dc31980b7caaf Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 3 Jun 2018 18:04:41 -0700 Subject: [PATCH 14/60] Mending more minor model mistakes. --- owl/model.py | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/owl/model.py b/owl/model.py index d342325..3b7ee5f 100644 --- a/owl/model.py +++ b/owl/model.py @@ -5,6 +5,7 @@ import typing as ty import string import weakref +import re import tinydb import maya @@ -22,8 +23,7 @@ 3: 'winter', 4: 'spring' } -DAYS = 'M', 'T', 'W', 'Th', 'F', 'S', 'U' -DAYS_DECREASING_LEN = sorted(DAYS, key=lambda day: -len(day)) +DAYS_PATTERN = f"^{'(M|T|W|Th|F|S|U)?'*7}$" STANDARD_TYPE = 'standard' ONLINE_TYPE = 'online' @@ -253,8 +253,8 @@ def get_quarter(cls, model: DataModel, name: str) -> 'QuarterView': def db(self) -> tinydb.TinyDB: """ Database getter; - Database will only be created when this property is accessed, - and then stored for future uses. + Database will only be created when this property is first + accessed, and will then stored for future uses. :return: TinyDB """ if not self._db: @@ -550,10 +550,8 @@ def days(self) -> ty.Set[str]: in 'days' field. :return: Set[str] """ - days = set() - for entry in self.data: - days |= self._unpack_entry_days(entry) - return days + return {day for entry in self.data + for day in self._unpack_entry_days(entry)} def _unpack_entry_days(self, entry: SECTION_ENTRY_T) -> ty.Set[str]: """ @@ -575,20 +573,10 @@ def _unpack_entry_days(self, entry: SECTION_ENTRY_T) -> ty.Set[str]: # If days are 'TBA' and class is not online, raise an # exception rather than trying to handle odd edge case. raise DataError(f'{self} Entry days have not yet been entered') - # Iterate over days in order of longest to shortest, - # removing day codes from temporary string as they are found. - # This is so that a day with a longer code, ex: 'Th' will be - # found and removed from the string before 'T' - # otherwise, the day code 'Th' would produce result: {'T'} and - # then be unable to parse the remaining 'h'. - days = set() - for day_code in DAYS_DECREASING_LEN: - if day_code in s: - s = s.replace(day_code, '') - days.add(day_code) - if s: - raise DataError(f'Could not parse {entry[DAYS_KEY]}, ' - f'unknown day code: {s})') + matches = re.match(DAYS_PATTERN, entry[DAYS_KEY]) + if not matches: + raise DataError(f'Could not parse days string: {entry[DAYS_KEY]}') + days = set(matches.groups()) - {None} return days @property @@ -635,9 +623,7 @@ def rooms(self) -> ty.Set['str']: :return: Set[str] :raises DataError if unexpected values are found in data. """ - rooms = set() - [rooms.add(duration.room) for duration in self.durations] - return rooms + return {duration.room for duration in self.durations} @property def status(self) -> str: From d8c97ead3974e09ebdfa28a4627f0e8e322569e7 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 4 Jun 2018 14:13:14 -0700 Subject: [PATCH 15/60] Began implementing request.py; handles validation + feedback of requests. --- owl/request.py | 162 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 owl/request.py diff --git a/owl/request.py b/owl/request.py new file mode 100644 index 0000000..a907f1e --- /dev/null +++ b/owl/request.py @@ -0,0 +1,162 @@ +""" +This module contains request classes, which validate and facilitate +access to data passed by api users in get requests. + +This module also allows more readable results to api users, that list +any potential issues with their requests, before they are acted on, +and without requiring error code lookups. +""" + +import typing as ty + +# fields +DEPARTMENT_KEY = 'dept' +COURSE_KEY = 'course' +QUARTER_KEY = 'quarter' + +# Argument keywords +ANY = 'any' +ALL = 'all' +LATEST = 'latest' + + +class RequestException(Exception): + """ + Exception raised when a request has received bad data. + RequestException may be given a string to return to the caller + as part of a 400 error. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.caller_msg = kwargs.get('caller_msg', 'Bad Request') + + +def throw_issues_after_init(cls: ty.TypeVar): + return cls + + +class Request: + def __init__(self): + self.issues: ty.List['Request.Issue'] = [] + self.checked_fields: ty.Dict[str, 'Request.Field'] = {} + + def __setattr__(self, k: ty.Any, v: ty.Any): + if isinstance(v, self.Field): + v.name = k + self.checked_fields[k] = v + else: + try: + field = self.checked_fields[k] + except KeyError: + pass + else: + issues = field.validate(v) + self.issues += issues + super().__setattr__(k, v) + + def __repr__(self) -> str: + return f'GetOneRequest[dept: {self.department}, course: {self.course}]' + + class Field: + """ + Class assisting with validation of passed data, to allow better + feedback to api user. + """ + def __init__( + self, + field_type: ty.TypeVar=object, + valid_values: ty.Collection[ty.Any]=(), + default=None, + type_coercion=True, + modifier: ty.Callable[[ty.Any], ty.Any] = None + ): + self.name: str = '' # Set externally by Request + self.type = field_type + self.valid_values = valid_values + self.default = default + self.type_coercion = type_coercion + self.modifier = modifier + + def validate(self, value: ty.Any) -> ty.List['Request.Issue']: + issues = [] + + def validate_type(v: ty.Any) -> 'Request.Issue' or None: + if isinstance(value, self.type): + return None + + if self.type_coercion: + try: + coerced_value = self.type(value) + except (ValueError, TypeError): + pass + else: + if isinstance(coerced_value, self.type): + return None + return self.issue( + f'Cannot coerce passed value: {v} of type: ' + f'{type(v).__name__} to expected ' + f'type: {self.type.__name__}.' + ) + return self.issue( + f'Expected type: {self.type.__name__}, but passed value' + f'{v} has type: {type(v).__name__}.') + + def validate_value(v: ty.Any) -> 'Request.Issue' or None: + try: + modified_v = self.modify(v) + except (ValueError, TypeError): + modified_v = v + + if modified_v not in self.valid_values: + if modified_v == v: + return self.issue( + f'Passed value: {v} is not valid. List of valid' + f'Values: {self.valid_values}.') + else: + return self.issue( + f'Passed value: {v}, (modified to {modified_v})' + f'is not valid. List of valid values: ' + f'{self.valid_values}.' + ) + return None + + if self.type != object: + issue = validate_type(value) + if issue: + issues.append(issue) + + if self.valid_values: + issue = validate_value(value) + if issue: + issues.append(issue) + + return issues + + def modify(self, value: ty.Any) -> ty.Any: + """ + Modifies passed value so that it can be stored in field. + :param value: Any + :return: Any + """ + return self.modifier(value) + + def issue(self, msg: str) -> 'Request.Issue': + return Request.Issue(self.name, msg) + + def __repr__(self) -> str: + return f'Request.Field[{self.name}, type: {self.type.__name__}]' + + class Issue: + """ + Class detailing a specific issue found with a request. + """ + + def __init__(self, field_name: str, msg: str): + self.field_name = field_name + self.msg = msg + + def user_string(self) -> str: + return f'{self.field_name}: {self.msg}' + + def __repr__(self): + return f'Issue[field: {self.field_name}, msg: {self.msg}]' From b1954042150803c40af3f246921ff52c8c879e59 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Tue, 5 Jun 2018 22:48:27 -0700 Subject: [PATCH 16/60] Implemented Request class and associated classes and tests. --- owl/request.py | 143 ++++++++++++++++++++++++++++-------- tests/model/test_request.py | 41 +++++++++++ 2 files changed, 154 insertions(+), 30 deletions(-) create mode 100644 tests/model/test_request.py diff --git a/owl/request.py b/owl/request.py index a907f1e..445e30d 100644 --- a/owl/request.py +++ b/owl/request.py @@ -26,36 +26,82 @@ class RequestException(Exception): RequestException may be given a string to return to the caller as part of a 400 error. """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.caller_msg = kwargs.get('caller_msg', 'Bad Request') + def __init__(self, *args, user_msg='Bad Request'): + super().__init__(*args) + self.user_msg = user_msg -def throw_issues_after_init(cls: ty.TypeVar): - return cls +class Request: + _check_for_unused_args = True # Able to be changed in subclasses. + _raise_issues_after_unpacking = True # Able to be changed in subclasses. -class Request: - def __init__(self): + def __init__(self, *args, **kwargs): self.issues: ty.List['Request.Issue'] = [] self.checked_fields: ty.Dict[str, 'Request.Field'] = {} + req_args = RequestArguments(*args, **kwargs) + self.unpack(req_args) + + def check_for_unused_args(): + if not req_args: + raise TypeError( + 'Unused arguments can only be detected if request ' + 'arguments are wrapped with a RequestArguments instance. ' + 'This can be done with the wrap_request_arguments ' + 'decorator.') + for k, v in req_args.unused().items(): + self.issues.append(Request.Issue(k, 'Unused argument.')) + + if self._check_for_unused_args: + check_for_unused_args() + if self._raise_issues_after_unpacking and self.issues: + self.raise_issues() + + def unpack(self, request_args: 'RequestArguments') -> None: + """ + This method should be overridden by subclasses to define how + arguments should be unpacked by request. + :param request_args: RequestArguments + :return: None + """ + raise NotImplementedError( + 'unpack() should be overridden by subclasses.') + def __setattr__(self, k: ty.Any, v: ty.Any): - if isinstance(v, self.Field): - v.name = k - self.checked_fields[k] = v + """ + If attribute assigned to has same key as a Request.Field + instance owned by the class, the assigned value is first + validated by the Field, before being assigned, and any issues + are added to the collection maintained by Request. + :param k: str + :param v: Any + :return: None + """ + try: + field: 'Request.Field' = type(self).__dict__[k] + except KeyError: + pass else: - try: - field = self.checked_fields[k] - except KeyError: - pass - else: - issues = field.validate(v) - self.issues += issues - super().__setattr__(k, v) + issues = field.validate(v) + self.issues += issues + super().__setattr__(k, v) + + def raise_issues(self) -> None: + """ + Raises a RequestException which contains string representations + of all issues so far encountered by Request with passed data. + :return: None + """ + user_msg = '\n'.join(issue.user_string for issue in self.issues) + raise RequestException( + 'Issues were encountered during request initialization', + user_msg=f'The Following issues were encountered' + f'while creating request: {user_msg}' + ) def __repr__(self) -> str: - return f'GetOneRequest[dept: {self.department}, course: {self.course}]' + return f'Request[]' class Field: """ @@ -64,20 +110,26 @@ class Field: """ def __init__( self, - field_type: ty.TypeVar=object, - valid_values: ty.Collection[ty.Any]=(), + t: ty.TypeVar = object, + valid: ty.Container[ty.Any] = (), default=None, type_coercion=True, modifier: ty.Callable[[ty.Any], ty.Any] = None - ): + ) -> None: self.name: str = '' # Set externally by Request - self.type = field_type - self.valid_values = valid_values + self.type = t + self.valid_values = valid self.default = default self.type_coercion = type_coercion self.modifier = modifier def validate(self, value: ty.Any) -> ty.List['Request.Issue']: + """ + Validates passed value. + :param value: value that may be assigned to field in a + Request instance. + :return: List[Request.Issue] + """ issues = [] def validate_type(v: ty.Any) -> 'Request.Issue' or None: @@ -112,12 +164,11 @@ def validate_value(v: ty.Any) -> 'Request.Issue' or None: return self.issue( f'Passed value: {v} is not valid. List of valid' f'Values: {self.valid_values}.') - else: - return self.issue( - f'Passed value: {v}, (modified to {modified_v})' - f'is not valid. List of valid values: ' - f'{self.valid_values}.' - ) + return self.issue( + f'Passed value: {v}, (modified to {modified_v})' + f'is not valid. List of valid values: ' + f'{self.valid_values}.' + ) return None if self.type != object: @@ -141,6 +192,14 @@ def modify(self, value: ty.Any) -> ty.Any: return self.modifier(value) def issue(self, msg: str) -> 'Request.Issue': + """ + Creates a new issue using name of field as Issue + field param. + This is intended as a helper method for internal use but + may also be used externally. + :param msg: str + :return: Request.Issue + """ return Request.Issue(self.name, msg) def __repr__(self) -> str: @@ -155,8 +214,32 @@ def __init__(self, field_name: str, msg: str): self.field_name = field_name self.msg = msg + @property def user_string(self) -> str: return f'{self.field_name}: {self.msg}' def __repr__(self): return f'Issue[field: {self.field_name}, msg: {self.msg}]' + + +class RequestArguments(dict): + """ + Dictionary subclass that is intended to wrap arguments passed to + a Request. + RequestArguments will track which arguments have been used, in + order to help identify surplus or misnamed fields. + """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.retrieved = set() + + def __getitem__(self, k): + self.retrieved.add(k) + return super().__getitem__(k) + + def unused(self) -> ty.Dict[ty.Any, ty.Any]: + """ + Returns dictionary of unused keys and their associated values. + :return: dict + """ + return {k: v for k, v in self.items() if k not in self.retrieved} diff --git a/tests/model/test_request.py b/tests/model/test_request.py new file mode 100644 index 0000000..2e7e535 --- /dev/null +++ b/tests/model/test_request.py @@ -0,0 +1,41 @@ +from unittest import TestCase + +import owl.request + + +class TestRequest(TestCase): + + def test_request_can_be_constructed_using_correct_args(self): + self.ExampleRequest(a='foo', b='bar') + # no checks needed here. + + def test_request_throws_request_error_with_invalid_type(self): + self.assertRaises( + owl.request.RequestException, + lambda: self.ExampleRequest({'a': 'foo', 'b': 5}) + ) + + def test_request_throws_request_error_with_invalid_value(self): + self.assertRaises( + owl.request.RequestException, + lambda: self.ExampleRequest({'a': 'foo', 'b': 'blah'}) + ) + + def test_correct_values_are_stored_in_request(self): + request = self.ExampleRequest({'a': 'foo', 'b': 'bar'}) + self.assertEqual('foo', request.test_field_1) + self.assertEqual('bar', request.test_field_2) + + def test_extra_values_cause_request_error_to_be_raised(self): + self.assertRaises( + owl.request.RequestException, + lambda: self.ExampleRequest({'a': 'foo', 'b': 'bar', 'c': 'blah'}) + ) + + class ExampleRequest(owl.request.Request): + test_field_1 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) + test_field_2 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) + + def unpack(self, request_args: owl.request.RequestArguments): + self.test_field_1: str = request_args['a'] + self.test_field_2: str = request_args['b'] From f733a5d13e6d5853ef38e0826e0bb960745f3966 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Wed, 6 Jun 2018 13:29:20 -0700 Subject: [PATCH 17/60] Added more request tests + made additional bugfixes. --- owl/request.py | 22 +++++++++------------- tests/model/test_request.py | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/owl/request.py b/owl/request.py index 445e30d..4f2237a 100644 --- a/owl/request.py +++ b/owl/request.py @@ -9,16 +9,6 @@ import typing as ty -# fields -DEPARTMENT_KEY = 'dept' -COURSE_KEY = 'course' -QUARTER_KEY = 'quarter' - -# Argument keywords -ANY = 'any' -ALL = 'all' -LATEST = 'latest' - class RequestException(Exception): """ @@ -51,7 +41,7 @@ def check_for_unused_args(): 'This can be done with the wrap_request_arguments ' 'decorator.') for k, v in req_args.unused().items(): - self.issues.append(Request.Issue(k, 'Unused argument.')) + self.issues.append(Request.Issue(repr(k), 'Unused argument.')) if self._check_for_unused_args: check_for_unused_args() @@ -85,6 +75,7 @@ def __setattr__(self, k: ty.Any, v: ty.Any): else: issues = field.validate(v) self.issues += issues + v = field.modify(v) super().__setattr__(k, v) def raise_issues(self) -> None: @@ -95,7 +86,8 @@ def raise_issues(self) -> None: """ user_msg = '\n'.join(issue.user_string for issue in self.issues) raise RequestException( - 'Issues were encountered during request initialization', + 'Issues were encountered during request initialization: ' + f'{self.issues}', user_msg=f'The Following issues were encountered' f'while creating request: {user_msg}' ) @@ -189,7 +181,7 @@ def modify(self, value: ty.Any) -> ty.Any: :param value: Any :return: Any """ - return self.modifier(value) + return self.modifier(value) if self.modifier else value def issue(self, msg: str) -> 'Request.Issue': """ @@ -237,6 +229,10 @@ def __getitem__(self, k): self.retrieved.add(k) return super().__getitem__(k) + def get(self, k, d=None): + self.retrieved.add(k) + return super().get(k, d) + def unused(self) -> ty.Dict[ty.Any, ty.Any]: """ Returns dictionary of unused keys and their associated values. diff --git a/tests/model/test_request.py b/tests/model/test_request.py index 2e7e535..2a4207b 100644 --- a/tests/model/test_request.py +++ b/tests/model/test_request.py @@ -32,6 +32,17 @@ def test_extra_values_cause_request_error_to_be_raised(self): lambda: self.ExampleRequest({'a': 'foo', 'b': 'bar', 'c': 'blah'}) ) + def test_values_are_modified_as_expected(self): + class CapitalizingRequest(owl.request.Request): + test_field = owl.request.Request.Field( + t=str, modifier=lambda s: s.upper()) + + def unpack(self, request_args: 'owl.request.RequestArguments'): + self.test_field = request_args.get('a') + + request = CapitalizingRequest({'a': 'foo'}) + self.assertEqual('FOO', request.test_field) + class ExampleRequest(owl.request.Request): test_field_1 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) test_field_2 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) @@ -39,3 +50,17 @@ class ExampleRequest(owl.request.Request): def unpack(self, request_args: owl.request.RequestArguments): self.test_field_1: str = request_args['a'] self.test_field_2: str = request_args['b'] + + +class TestRequestField(TestCase): + def test_field_with_no_valid_field_arg_passed_accepts_all_values(self): + field = owl.request.Request.Field(t=int) + self.assertEqual([], field.validate(2)) # Check no issue is returned. + + def test_field_with_valid_field_arg_passed_accepts_valid_values(self): + field = owl.request.Request.Field(t=int, valid=(2,)) + self.assertEqual([], field.validate(2)) # Check no issue is returned. + + def test_field_returns_issue_when_invalid_value_passed(self): + field = owl.request.Request.Field(t=int, valid=(2,)) + self.assertEqual(1, len(field.validate(3))) From 33ab1870e451c05cc2b6f92d03c2eff230ee794e Mon Sep 17 00:00:00 2001 From: TryExcept Date: Wed, 6 Jun 2018 15:16:17 -0700 Subject: [PATCH 18/60] Added docstrings to Issue methods. --- owl/request.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/owl/request.py b/owl/request.py index 4f2237a..da9d5d9 100644 --- a/owl/request.py +++ b/owl/request.py @@ -208,9 +208,19 @@ def __init__(self, field_name: str, msg: str): @property def user_string(self) -> str: + """ + Issue representation as a string that may be presented to + the user of the api. + :return: str + """ return f'{self.field_name}: {self.msg}' def __repr__(self): + """ + Issue representation as a string intended for display to + a developer. + :return: str + """ return f'Issue[field: {self.field_name}, msg: {self.msg}]' From 42001fc6e6daf277df3204d3401e5446bbfb7aaa Mon Sep 17 00:00:00 2001 From: TryExcept Date: Wed, 6 Jun 2018 21:08:38 -0700 Subject: [PATCH 19/60] Added custom validators for Request.Fields --- owl/request.py | 7 ++++++- tests/model/test_request.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/owl/request.py b/owl/request.py index da9d5d9..8be5b26 100644 --- a/owl/request.py +++ b/owl/request.py @@ -106,7 +106,8 @@ def __init__( valid: ty.Container[ty.Any] = (), default=None, type_coercion=True, - modifier: ty.Callable[[ty.Any], ty.Any] = None + modifier: ty.Callable[[ty.Any], ty.Any] = None, + validator: ty.Callable[[ty.Any], ty.List[str]] = None ) -> None: self.name: str = '' # Set externally by Request self.type = t @@ -114,6 +115,7 @@ def __init__( self.default = default self.type_coercion = type_coercion self.modifier = modifier + self.validator = validator def validate(self, value: ty.Any) -> ty.List['Request.Issue']: """ @@ -173,6 +175,9 @@ def validate_value(v: ty.Any) -> 'Request.Issue' or None: if issue: issues.append(issue) + if self.validator: + issues += (self.issue(msg) for msg in self.validator(value)) + return issues def modify(self, value: ty.Any) -> ty.Any: diff --git a/tests/model/test_request.py b/tests/model/test_request.py index 2a4207b..7a1e006 100644 --- a/tests/model/test_request.py +++ b/tests/model/test_request.py @@ -64,3 +64,13 @@ def test_field_with_valid_field_arg_passed_accepts_valid_values(self): def test_field_returns_issue_when_invalid_value_passed(self): field = owl.request.Request.Field(t=int, valid=(2,)) self.assertEqual(1, len(field.validate(3))) + + def test_custom_validator_can_add_issue_successfully(self): + def must_exceed_5(x): + if x > 5: + return [] + else: + return ['Value was not greater than 5'] + + field = owl.request.Request.Field(t=int, validator=must_exceed_5) + self.assertEqual(1, len(field.validate(4))) From ace37308af38c9914f12db8393bfe9063316c4a6 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Thu, 7 Jun 2018 12:55:28 -0700 Subject: [PATCH 20/60] Minor request.py changes. --- owl/request.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/owl/request.py b/owl/request.py index 8be5b26..7e41681 100644 --- a/owl/request.py +++ b/owl/request.py @@ -10,7 +10,7 @@ import typing as ty -class RequestException(Exception): +class RequestException(ValueError): """ Exception raised when a request has received bad data. RequestException may be given a string to return to the caller @@ -58,7 +58,7 @@ def unpack(self, request_args: 'RequestArguments') -> None: raise NotImplementedError( 'unpack() should be overridden by subclasses.') - def __setattr__(self, k: ty.Any, v: ty.Any): + def __setattr__(self, k: ty.Any, v: ty.Any) -> None: """ If attribute assigned to has same key as a Request.Field instance owned by the class, the assigned value is first @@ -207,7 +207,7 @@ class Issue: Class detailing a specific issue found with a request. """ - def __init__(self, field_name: str, msg: str): + def __init__(self, field_name: str, msg: str) -> None: self.field_name = field_name self.msg = msg @@ -220,7 +220,7 @@ def user_string(self) -> str: """ return f'{self.field_name}: {self.msg}' - def __repr__(self): + def __repr__(self) -> str: """ Issue representation as a string intended for display to a developer. From 7a529138682ef8d42e2fafd068d229dbef2f45b1 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Thu, 7 Jun 2018 15:10:22 -0700 Subject: [PATCH 21/60] Requests now can store sub-requests. --- owl/request.py | 65 +++++++++++++++++++++++++------------ tests/model/test_request.py | 42 +++++++++++++++++++++++- 2 files changed, 85 insertions(+), 22 deletions(-) diff --git a/owl/request.py b/owl/request.py index 7e41681..207575d 100644 --- a/owl/request.py +++ b/owl/request.py @@ -16,9 +16,15 @@ class RequestException(ValueError): RequestException may be given a string to return to the caller as part of a 400 error. """ - def __init__(self, *args, user_msg='Bad Request'): + def __init__( + self, + *args, + user_msg='Bad Request', + issues: ty.List['Request.Issue'] = () + ) -> None: super().__init__(*args) self.user_msg = user_msg + self.issues = issues class Request: @@ -29,6 +35,7 @@ class Request: def __init__(self, *args, **kwargs): self.issues: ty.List['Request.Issue'] = [] self.checked_fields: ty.Dict[str, 'Request.Field'] = {} + self.has_parent: bool = kwargs.pop('__has_parent', False) req_args = RequestArguments(*args, **kwargs) self.unpack(req_args) @@ -45,7 +52,10 @@ def check_for_unused_args(): if self._check_for_unused_args: check_for_unused_args() - if self._raise_issues_after_unpacking and self.issues: + if all(( + self._raise_issues_after_unpacking, + self.issues, + not self.has_parent)): self.raise_issues() def unpack(self, request_args: 'RequestArguments') -> None: @@ -89,7 +99,8 @@ def raise_issues(self) -> None: 'Issues were encountered during request initialization: ' f'{self.issues}', user_msg=f'The Following issues were encountered' - f'while creating request: {user_msg}' + f'while creating request: {user_msg}', + issues=self.issues ) def __repr__(self) -> str: @@ -105,7 +116,7 @@ def __init__( t: ty.TypeVar = object, valid: ty.Container[ty.Any] = (), default=None, - type_coercion=True, + type_coercion=False, modifier: ty.Callable[[ty.Any], ty.Any] = None, validator: ty.Callable[[ty.Any], ty.List[str]] = None ) -> None: @@ -126,18 +137,24 @@ def validate(self, value: ty.Any) -> ty.List['Request.Issue']: """ issues = [] + try: + modified_v = self.modify(value) + except (ValueError, TypeError): + modified_v = value + + if self.type_coercion: + try: + coerced_value = self.type(modified_v) + except (ValueError, TypeError): + coerced_value = modified_v + else: + coerced_value = modified_v + def validate_type(v: ty.Any) -> 'Request.Issue' or None: - if isinstance(value, self.type): + if isinstance(coerced_value, self.type): return None if self.type_coercion: - try: - coerced_value = self.type(value) - except (ValueError, TypeError): - pass - else: - if isinstance(coerced_value, self.type): - return None return self.issue( f'Cannot coerce passed value: {v} of type: ' f'{type(v).__name__} to expected ' @@ -148,18 +165,13 @@ def validate_type(v: ty.Any) -> 'Request.Issue' or None: f'{v} has type: {type(v).__name__}.') def validate_value(v: ty.Any) -> 'Request.Issue' or None: - try: - modified_v = self.modify(v) - except (ValueError, TypeError): - modified_v = v - - if modified_v not in self.valid_values: - if modified_v == v: + if coerced_value not in self.valid_values: + if coerced_value == v: return self.issue( f'Passed value: {v} is not valid. List of valid' f'Values: {self.valid_values}.') return self.issue( - f'Passed value: {v}, (modified to {modified_v})' + f'Passed value: {v}, (modified to {coerced_value})' f'is not valid. List of valid values: ' f'{self.valid_values}.' ) @@ -176,7 +188,8 @@ def validate_value(v: ty.Any) -> 'Request.Issue' or None: issues.append(issue) if self.validator: - issues += (self.issue(msg) for msg in self.validator(value)) + issues += (self.issue(msg) for msg in + self.validator(coerced_value)) return issues @@ -202,6 +215,16 @@ def issue(self, msg: str) -> 'Request.Issue': def __repr__(self) -> str: return f'Request.Field[{self.name}, type: {self.type.__name__}]' + class SubRequestField(Field): + def __init__(self, t): + super().__init__( + t, + modifier=lambda args: t(args, __has_parent=True), + validator=lambda request: + [f'Sub-Field {repr(issue.field_name)}: {issue.msg}' + for issue in request.issues] + ) + class Issue: """ Class detailing a specific issue found with a request. diff --git a/tests/model/test_request.py b/tests/model/test_request.py index 7a1e006..40a053e 100644 --- a/tests/model/test_request.py +++ b/tests/model/test_request.py @@ -4,7 +4,6 @@ class TestRequest(TestCase): - def test_request_can_be_constructed_using_correct_args(self): self.ExampleRequest(a='foo', b='bar') # no checks needed here. @@ -43,6 +42,30 @@ def unpack(self, request_args: 'owl.request.RequestArguments'): request = CapitalizingRequest({'a': 'foo'}) self.assertEqual('FOO', request.test_field) + def test_sub_request_field_propagates_issues(self): + self.assertRaises( + owl.request.RequestException, + lambda: SuperRequest({'a': {'b': 5}}) + ) + + def test_that_raising_of_sub_request_issues_is_delayed(self): + class Super2Request(owl.request.Request): + sub_request_a = owl.request.Request.SubRequestField(SubRequest) + sub_request_b = owl.request.Request.SubRequestField(SubRequest) + + def unpack(self, request_args: 'owl.request.RequestArguments'): + self.sub_request_a = request_args.get('a') + self.sub_request_b = request_args.get('b') + + try: + Super2Request({'a': {'b': 6}, 'b': {'b': 5}}) + except owl.request.RequestException as e: + self.assertEqual(2, len(e.issues)) + + def test_sub_request_field_does_not_raise_issues_with_correct_args(self): + req = SuperRequest({'a': {'b': 'foo'}}) + self.assertEqual('foo', req.sub_request.test_field) + class ExampleRequest(owl.request.Request): test_field_1 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) test_field_2 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) @@ -74,3 +97,20 @@ def must_exceed_5(x): field = owl.request.Request.Field(t=int, validator=must_exceed_5) self.assertEqual(1, len(field.validate(4))) + + +######### + + +class SubRequest(owl.request.Request): + test_field = owl.request.Request.Field(t=str) + + def unpack(self, request_args: 'owl.request.RequestArguments'): + self.test_field = request_args.get('b') + + +class SuperRequest(owl.request.Request): + sub_request = owl.request.Request.SubRequestField(SubRequest) + + def unpack(self, request_args: 'owl.request.RequestArguments'): + self.sub_request = request_args.get('a') \ No newline at end of file From 46df6efbb563eb14afeb43aa74d769913a3bb4a9 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sat, 9 Jun 2018 22:32:43 -0700 Subject: [PATCH 22/60] Added some more annotations to help pycharm's type annotation detection. --- owl/model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/owl/model.py b/owl/model.py index 3b7ee5f..d243a5f 100644 --- a/owl/model.py +++ b/owl/model.py @@ -82,7 +82,7 @@ class DataError(Exception): class DataModel: - def __init__(self, db_dir: str): + def __init__(self, db_dir: str) -> None: """ Initializer for DataModel that uses the passed db_dir path to find database tables and store cached values. @@ -203,7 +203,7 @@ class QuarterView: # which during its __init__ method, registers itself with # the model. - def __init__(self, model: DataModel, name: str): + def __init__(self, model: DataModel, name: str) -> None: """ Instantiates a new QuarterView. When instantiated, QuarterView will attempt to register @@ -757,6 +757,6 @@ class InstructorView: """ Class handling access of instructor data. """ - def __init__(self, model: DataModel, name: str): + def __init__(self, model: DataModel, name: str) -> None: self.model: DataModel = model self.name: str = name From aa3ce740f0efce461f2ac1faeb0860e057fc3741 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Sun, 10 Jun 2018 09:48:21 -0700 Subject: [PATCH 23/60] Condensed generate_test_data; slightly more readable now? --- tests/generate_test_data.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/generate_test_data.py b/tests/generate_test_data.py index ab3b9a1..eded553 100644 --- a/tests/generate_test_data.py +++ b/tests/generate_test_data.py @@ -17,33 +17,25 @@ def generate_data_py(): with open(os.path.join(TEST_DATA_DIR, 'data.py'), 'w') as file: # test_sample_url_can_be_generated data = {'dept': 'test_dept', 'course': 'test_course'} - file.write(f"test_sample_url_can_be_generated_data = {data}\n") # test_get_one_dept data = {'dept': 'CS'} - dept = test_database.table(f"{data['dept']}").all() - file.write(f"test_get_one_dept_data = {dept}\n") # test_get_one_dept_and_course data = {'dept': 'CS', 'course': '2A'} - dept = test_database.table(f"{data['dept']}").all() - course = next((e[f"{data['course']}"] for e in dept if f"{data['course']}" in e)) - file.write(f"test_get_one_dept_and_course_data = {course}\n") # test_get_two_dept data = {'courses': [{'dept': 'CS'}, {'dept': 'MATH'}]} - depts = [] for i in data['courses']: depts.append(test_database.table(i['dept']).all()) - file.write(f"test_get_two_dept_data = {depts}\n") From 2a195a0b327ab5f1ea66967b5176ad132f970d2c Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 14:18:34 -0700 Subject: [PATCH 24/60] Modified ClassDuration to store maya interval. --- owl/model.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/owl/model.py b/owl/model.py index d243a5f..3965c5e 100644 --- a/owl/model.py +++ b/owl/model.py @@ -749,8 +749,15 @@ def __init__( ): self.day = day self.room = room - self.start = start - self.end = end + self.interval = maya.MayaInterval(start, end) + + @property + def start(self): + return self.interval.start + + @property + def end(self): + return self.interval.end class InstructorView: From b7347a1194e9ec6499380c68bc9f38d3bb693b44 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 14:22:03 -0700 Subject: [PATCH 25/60] Implemented input classes and schema. --- owl/input.py | 20 +++++++++++++ owl/schema.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 owl/input.py create mode 100644 owl/schema.py diff --git a/owl/input.py b/owl/input.py new file mode 100644 index 0000000..17d7e5c --- /dev/null +++ b/owl/input.py @@ -0,0 +1,20 @@ +""" +This module contains request classes, which validate and facilitate +access to data passed by api users in get requests. + +This module also allows more readable results to api users, that list +any potential issues with their requests, before they are acted on, +and without requiring error code lookups. +""" +import flask_inputs +from flask_inputs.validators import JsonSchema + +from owl.schema import get_definition + + +class GetOneInput(flask_inputs.Inputs): + json = [JsonSchema(schema=get_definition('get_one'))] + + +class GetManyInput(flask_inputs.Inputs): + json = [JsonSchema(schema=get_definition('get_many'))] diff --git a/owl/schema.py b/owl/schema.py new file mode 100644 index 0000000..504f6cf --- /dev/null +++ b/owl/schema.py @@ -0,0 +1,79 @@ + +definitions = { + 'status_filter': { + 'type': 'object', + 'properties': { + 'open': {'type': 'int'}, + 'waitlist': {'type': 'int'}, + 'full': {'type': 'int'}, + } + }, + 'type_filter': { + 'type': 'object', + 'properties': { + 'standard': {'type': 'int'}, + 'online': {'type': 'int'}, + 'hybrid': {'type': 'int'} + } + }, + 'day_filter': { + 'type': 'object', + 'properties': { + 'M': {'type': 'int'}, + 'T': {'type': 'int'}, + 'W': {'type': 'int'}, + 'Th': {'type': 'int'}, + 'F': {'type': 'int'}, + 'S': {'type': 'int'}, + 'U': {'type': 'int'}, + } + }, + 'time_filter': { + 'type': 'object', + 'properties': { + 'start': {'type': 'string'}, + 'end': {'type': 'string'}, + } + }, + 'filter': { + 'type': 'object', + 'properties': { + 'status': {'$ref': '#/definitions/status_filter'}, + 'type': {'$ref': '#/definitions/type_filter'}, + 'days': {'$ref': '#/definitions/day_filter'}, + 'time': {'$ref': '#/definitions/time_filter'}, + }, + }, + 'get_one': { + 'type': 'object', + 'properties': { + 'quarter': {'type': 'string'}, + 'department': {'type': 'string'}, + 'course': {'type': 'string'}, + 'filter': {'$ref': '#/definitions/filter'}, + }, + 'required': ['department'] + }, + 'get_many': { + 'type': 'object', + 'properties': { + 'courses': { + 'type': 'array', + 'items': { + 'type': 'object', + 'additionalProperties': { + '$ref': '#/definitions/get_one' + } + } + }, + 'filter': {'$ref': '#/definitions/filter'}, + }, + 'required': ['courses'] + } +} + + +def get_definition(definition: str): + d = definitions[definition] + d['definitions'] = definitions + return d From 3525416fd3bcec91cdf79b470a7f39aba1c78620 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 14:22:22 -0700 Subject: [PATCH 26/60] Implemented section filter. --- owl/filter.py | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 owl/filter.py diff --git a/owl/filter.py b/owl/filter.py new file mode 100644 index 0000000..c5753a0 --- /dev/null +++ b/owl/filter.py @@ -0,0 +1,89 @@ +""" +Contains classes and methods for filtering data. +""" + +import typing as ty +import maya + +import owl.model + +# Keys +MONDAY_KEY = 'M' +TUESDAY_KEY = 'T' +WEDNESDAY_KEY = 'W' +THURSDAY_KEY = 'Th' +FRIDAY_KEY = 'F' +SATURDAY_KEY = 'S' +SUNDAY_KEY = 'U' + +# Filter Argument keywords +ANY = 'any' +ALL = 'all' + + +class SectionFilter: + """ + Filters courses based on passed parameters. + """ + def __init__( + self, + status: ty.Dict[str, int] = None, + types: ty.Dict[str, int] = None, + days: ty.Dict[str, int] = None, + time: ty.Dict[str, str] = None + ): + self.status = status + self.types = types + self.days = days + self.time = time + + def check(self, section: owl.model.SectionQuarterView) -> bool: + """ + Determines whether passed course passes filter + :param section: owl.model.SectionQuarterView + :return: bool + """ + + # Nested functions filter courses by taking a course key and + # returning a boolean indicating whether they should be included + # or excluded. True if they are to be included, False if excluded. + + def status_filter() -> bool: + # {'open':0, 'waitlist':0, 'full':0} + if not self.status: + return True + # Create 'mask' of course statuses that are to be included. + status_mask = {k for (k, v) in self.status.items() if v} + # Return True only if course status is in mask. + return section.status in status_mask + + def type_filter() -> bool: + # {'standard':1, 'online':1, 'hybrid':0} + if not self.types: + return True + # Get course section + mask = {k for (k, v) in self.types.items() if v} + return section.section_type in mask + + def day_filter() -> bool: + # {'M':1, 'T':0, 'W':1, 'Th':0, 'F':0, 'S':0, 'U':0} + if not self.days: + return True + # create set of days that are allowed by passed filters + mask = {k for (k, v) in self.days.items() if v} + return section.days <= mask + + def time_filter() -> bool: + # {'start':'8:30 AM', 'end':'9:40 PM'} + if not self.time: + return True + filter_range = maya.MayaInterval( + start=maya.when(self.time['start']), + end=maya.when(self.time['end'])) + for duration in section.durations: + if not filter_range.contains(duration.interval): + return False + return True + + return all(( + status_filter(), type_filter(), day_filter(), time_filter())) From 679d49fc64ec52942fc9243ec42d1141c363bc7f Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 14:22:42 -0700 Subject: [PATCH 27/60] Implemented data model Accessor class. --- owl/access.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 owl/access.py diff --git a/owl/access.py b/owl/access.py new file mode 100644 index 0000000..47ce6eb --- /dev/null +++ b/owl/access.py @@ -0,0 +1,122 @@ +""" +Intermediary module facilitating access of model. + +Methods contained are distinct from those in owl.model because they +are intended primarily for the convenience of the user facing module +(server.py) and contain methods and classes that may result in user- +feedback (eg: AccessException's user_msg and similar). + +Methods and classes are separate from server.py because they deal +primarily with access and retrieval of data, and not the form of the +response to the api user's request. +""" +import typing as ty + +import owl.model +import owl.filter + + +# Access Argument keywords +ANY = 'any' +ALL = 'all' +LATEST = 'latest' + + +class AccessException(ValueError): + """ + Exception raised when requested data cannot be accessed, but not as + result of an internal error or bug, or badly a badly formed + request. This usually is because the requested data does not exist. + """ + def __init__(self, *args, user_msg='Bad Request'): + super().__init__(*args) + self.user_msg = user_msg + + +class ModelAccessor: + def __init__(self, model: owl.model.DataModel) -> None: + self.model = model + + # This method may be removed in the future if the get_one + # method path is removed. + def get_one( + self, + school: str, + department: str, + course: str = ALL, + quarter: str = LATEST, + section_filter: owl.filter.SectionFilter = None + ) -> ty.Union[owl.model.COURSE_DATA_T, owl.model.DEPT_DATA_T]: + if course == ALL: + return self.get_department_data( + school, quarter, department, section_filter) + else: + return self.get_course_data( + school, quarter, department, course, section_filter) + + def get_department_data( + self, + school: str, + department: str, + quarter: str = LATEST, + section_filter: owl.filter.SectionFilter = None + ) -> owl.model.DEPT_DATA_T: + # Find department view + department_view = self._get_department(school, department, quarter) + + # get data from department view + return { + course_view.name: { + section_view.crn: section_view.data for + section_view in course_view.sections if + section_filter.check(section_view) + } + for course_view in department_view.courses + } + + def get_course_data( + self, + school: str, + department: str, + course: str, + quarter: str = LATEST, + section_filter: owl.filter.SectionFilter = None + ) -> owl.model.COURSE_DATA_T: + # Find department view + department_view = self._get_department(school, department, quarter) + try: + course_view = department_view.courses[course] + except KeyError as e: + raise AccessException( + f'No course in {department} with name: {course}.') from e + + return { + section_view.crn: section_view.data for + section_view in course_view.sections if + section_filter.check(section_view) + } + + def _get_department( + self, + school: str, + department: str, + quarter: str = LATEST, + ) -> owl.model.DepartmentQuarterView: + try: + school_view: owl.model.SchoolView = self.model.schools[school] + except KeyError as e: + raise AccessException( + f'No school found with identifier: {school_view}.') from e + try: + quarter_view = school_view.quarters[quarter] + except KeyError as e: + raise AccessException( + f'No quarter in {school} with name: {quarter}.') from e + try: + department_view = quarter_view.departments[department] + except KeyError as e: + raise AccessException( + f'No department in {quarter} with name: {department}.') from e + return department_view + + From 63bf8d3d4c08f785b7face2bdf2b05b6b0d0f902 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 14:23:22 -0700 Subject: [PATCH 28/60] Outdated classes now skipped (should be changed/removed in the near future). --- tests/test_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_server.py b/tests/test_server.py index 700d191..396d8df 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,10 +1,10 @@ from os.path import join -from unittest import TestCase +from unittest import TestCase, SkipTest from tinydb import TinyDB import settings -from server import generate_url, get_one, get_many +from server import generate_url # Try to get generated data. try: @@ -25,6 +25,7 @@ def test_sample_url_can_be_generated(self): ) +@SkipTest class TestGetOne(TestCase): def test_get_one_dept(self): data = {'dept': 'CS'} # floof.li/single?dept=CS @@ -54,6 +55,7 @@ def test_get_one_dept_and_course(self): ) +@SkipTest class TestGetMany(TestCase): def test_get_many_dept(self): data = {'courses': [{'dept': 'CS'}, {'dept': 'MATH'}]} @@ -73,6 +75,8 @@ def test_get_many_dept_returns_n_courses(self): len(result[0]) ) + +@SkipTest class TestFilters(TestCase): def test_filters_status_returns_n_courses(self): data = {'courses': [{'dept':'CS', 'course':'1A'}], From 52057d71eb0645fad6ccedd06d2aff6acf7f78e8 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 18:23:54 -0700 Subject: [PATCH 29/60] Removed tests for functions no longer implemented in server.py --- tests/test_server.py | 119 ------------------------------------------- 1 file changed, 119 deletions(-) diff --git a/tests/test_server.py b/tests/test_server.py index 396d8df..22ac0e4 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -24,122 +24,3 @@ def test_sample_url_can_be_generated(self): generate_url('test_dept', 'test_course') ) - -@SkipTest -class TestGetOne(TestCase): - def test_get_one_dept(self): - data = {'dept': 'CS'} # floof.li/single?dept=CS - - result = get_one(db=test_database, data=data, filters=dict()) - self.assertEqual( - test_data.test_get_one_dept_data, - result - ) - - def test_get_one_dept_returns_n_courses(self): - data = {'dept': 'CS'} - - result = get_one(db=test_database, data=data, filters=dict()) - self.assertEqual( - len(test_data.test_get_one_dept_data[0].keys()), - len(result[0].keys()) - ) - - def test_get_one_dept_and_course(self): - data = {'dept': 'CS', 'course': '2A'} - - result = get_one(db=test_database, data=data, filters=dict()) - self.assertEqual( - test_data.test_get_one_dept_and_course_data, - result - ) - - -@SkipTest -class TestGetMany(TestCase): - def test_get_many_dept(self): - data = {'courses': [{'dept': 'CS'}, {'dept': 'MATH'}]} - - result = get_many(db=test_database, data=data['courses'], filters=dict()) - self.assertEqual( - test_data.test_get_two_dept_data, - result - ) - - def test_get_many_dept_returns_n_courses(self): - data = {'courses': [{'dept': 'CS'}, {'dept': 'MATH'}]} - - result = get_many(db=test_database, data=data['courses'], filters=dict()) - self.assertEqual( - len(test_data.test_get_two_dept_data[0]), - len(result[0]) - ) - - -@SkipTest -class TestFilters(TestCase): - def test_filters_status_returns_n_courses(self): - data = {'courses': [{'dept':'CS', 'course':'1A'}], - 'filters': {'status': {'open':0, 'waitlist':1, 'full':0}}} - - result = get_many(db=test_database, data=data['courses'], filters=data['filters']) - - self.assertEqual( - 1, - len(result[0].keys()) - ) - - def test_filters_type_returns_n_courses(self): - data = {'courses':[{'dept':'CS', 'course':'1A'}], - 'filters':{'types':{'standard':0, 'online':1, 'hybrid':0}}} - - result = get_many(db=test_database, data=data['courses'], filters=data['filters']) - - self.assertEqual( - 4, - len(result[0].keys()) - ) - - def test_filters_days_returns_n_courses(self): - data = {'courses':[{'dept':'DANC', 'course':'14'}], - 'filters':{'days':{'M':1, 'T':0, 'W':0, 'Th':0, 'F':0, 'S':0, 'U':0}}} - - result = get_many(db=test_database, data=data['courses'], filters=data['filters']) - - self.assertEqual( - 1, - len(result[0].keys()) - ) - - def test_filter_days_will_return_correct_n_courses_with_all_days_set(self): - data = { - 'courses': [{ - 'dept': 'DANC', - 'course': '14' - }], - 'filters': { - 'days': { - 'M': 1, - 'T': 1, - 'W': 1, - 'Th': 1, - 'F': 1, - 'S': 1, - 'U': 1 - } - } - } - - result = get_many(db=test_database, data=data['courses'], filters=data['filters']) - self.assertEqual(len(result[0].keys()), 3) - - def test_filters_time_returns_n_courses(self): - data = {'courses':[{'dept':'DANC', 'course':'14'}], - 'filters': {'time':{'start':'7:30 AM', 'end':'12:00 PM'}}} - - result = get_many(db=test_database, data=data['courses'], filters=data['filters']) - - self.assertEqual( - 1, - len(result[0].keys()) - ) From b299d18f835e9b06de5e71c19253cd1c9968c980 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 18:25:00 -0700 Subject: [PATCH 30/60] Added test_access.py, reimplementing some tests from test_server.py --- tests/model/test_access.py | 44 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/model/test_access.py diff --git a/tests/model/test_access.py b/tests/model/test_access.py new file mode 100644 index 0000000..777bd7f --- /dev/null +++ b/tests/model/test_access.py @@ -0,0 +1,44 @@ +from unittest import TestCase + +import os + +from settings import TEST_RESOURCES_DIR +from owl.model import DataModel +from owl.access import ModelAccessor + +try: + from tests.test_db import data as test_data +except ImportError as e: + raise ImportError('Test Data could not be imported. If the data.py file ' + 'does not exist, it can be generated using the ' + 'generate_test_data.py script') from e + + +class TestGetOne(TestCase): + def test_get_one_dept(self): + accessor = get_accessor('model_test_dir_a') + result = accessor.get_one('fh', department='CS') + self.assertEqual( + test_data.test_get_one_dept_data[0], + result + ) + + def test_get_one_dept_returns_n_courses(self): + accessor = get_accessor('model_test_dir_a') + result = accessor.get_one(school='fh', department='CS') + self.assertEqual( + len(test_data.test_get_one_dept_data[0].keys()), + len(result.keys()) + ) + + def test_get_one_dept_and_course(self): + accessor = get_accessor('model_test_dir_a') + result = accessor.get_one(school='fh', department='CS', course='2A') + self.assertEqual( + test_data.test_get_one_dept_and_course_data, + result + ) + + +def get_accessor(dir_name: str): + return ModelAccessor(DataModel(os.path.join(TEST_RESOURCES_DIR, dir_name))) From 0f784d976593c9372e8643fef2c076be54469deb Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 18:25:35 -0700 Subject: [PATCH 31/60] Update to model and model accessor. --- owl/access.py | 18 +++++++++++------- owl/model.py | 25 +++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/owl/access.py b/owl/access.py index 47ce6eb..5cb56f9 100644 --- a/owl/access.py +++ b/owl/access.py @@ -49,10 +49,10 @@ def get_one( ) -> ty.Union[owl.model.COURSE_DATA_T, owl.model.DEPT_DATA_T]: if course == ALL: return self.get_department_data( - school, quarter, department, section_filter) + school, department, quarter, section_filter) else: return self.get_course_data( - school, quarter, department, course, section_filter) + school, department, course, quarter, section_filter) def get_department_data( self, @@ -69,7 +69,7 @@ def get_department_data( course_view.name: { section_view.crn: section_view.data for section_view in course_view.sections if - section_filter.check(section_view) + not section_filter or section_filter.check(section_view) } for course_view in department_view.courses } @@ -93,7 +93,7 @@ def get_course_data( return { section_view.crn: section_view.data for section_view in course_view.sections if - section_filter.check(section_view) + not section_filter or section_filter.check(section_view) } def _get_department( @@ -103,12 +103,16 @@ def _get_department( quarter: str = LATEST, ) -> owl.model.DepartmentQuarterView: try: - school_view: owl.model.SchoolView = self.model.schools[school] + school_view: owl.model.SchoolView = \ + self.model.schools[school.upper()] except KeyError as e: raise AccessException( - f'No school found with identifier: {school_view}.') from e + f'No school found with identifier: {school}.') from e try: - quarter_view = school_view.quarters[quarter] + if quarter == LATEST: + quarter_view = school_view.latest_quarter + else: + quarter_view = school_view.quarters[quarter] except KeyError as e: raise AccessException( f'No quarter in {school} with name: {quarter}.') from e diff --git a/owl/model.py b/owl/model.py index 3965c5e..79096ea 100644 --- a/owl/model.py +++ b/owl/model.py @@ -166,8 +166,20 @@ def quarters(self) -> ty.Dict[str, 'QuarterView']: associated with school. :return: QuarterView """ - return {name: quarter for name, quarter in self.model.quarters if - quarter.school_name == self.name} + return {name: quarter for name, quarter in self.model.quarters.items() + if quarter.school_name == self.name} + + @property + def latest_quarter(self) -> 'QuarterView': + """ + Gets view of most recent quarter. + :return: QuarterView + """ + latest = None + for quarter in self.quarters.values(): + if latest is None or int(quarter.name[:5]) > int(latest.name[:5]): + latest = quarter + return latest @property def type_codes(self) -> ty.Dict[str, str]: @@ -411,6 +423,11 @@ def __getitem__(self, course_name: str): raise KeyError(f'No course found named: {course_name} in ' f'{self}') from e + def __iter__(self): + for course_name, course_data in self.department.data.items(): + yield CourseQuarterView( + self.department, course_name, course_data) + def __repr__(self) -> str: return f'{self.department}.Courses' @@ -465,6 +482,10 @@ def __getitem__(self, section_name: str) -> 'SectionQuarterView': + section_name return section + def __iter__(self): + for section_data in self.course.data.values(): + yield SectionQuarterView(self.course, section_data) + def __repr__(self): return f'{self.course}.Sections' From c5ea03bfae19058eabae8b55b80311767132e8f8 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Sun, 10 Jun 2018 21:13:12 -0700 Subject: [PATCH 32/60] Refactoring mostly done. (bugs probably exist at this time) --- owl/access.py | 41 ++++- owl/input.py | 4 + owl/request.py | 279 ------------------------------- owl/schema.py | 9 + server.py | 319 +++++++++++------------------------- tests/model/test_access.py | 2 +- tests/model/test_request.py | 116 ------------- 7 files changed, 148 insertions(+), 622 deletions(-) delete mode 100644 owl/request.py delete mode 100644 tests/model/test_request.py diff --git a/owl/access.py b/owl/access.py index 5cb56f9..f8da2f3 100644 --- a/owl/access.py +++ b/owl/access.py @@ -62,7 +62,7 @@ def get_department_data( section_filter: owl.filter.SectionFilter = None ) -> owl.model.DEPT_DATA_T: # Find department view - department_view = self._get_department(school, department, quarter) + department_view = self.get_department(school, department, quarter) # get data from department view return { @@ -83,7 +83,7 @@ def get_course_data( section_filter: owl.filter.SectionFilter = None ) -> owl.model.COURSE_DATA_T: # Find department view - department_view = self._get_department(school, department, quarter) + department_view = self.get_department(school, department, quarter) try: course_view = department_view.courses[course] except KeyError as e: @@ -96,12 +96,11 @@ def get_course_data( not section_filter or section_filter.check(section_view) } - def _get_department( + def get_quarter( self, school: str, - department: str, quarter: str = LATEST, - ) -> owl.model.DepartmentQuarterView: + ) -> 'owl.model.QuarterView': try: school_view: owl.model.SchoolView = \ self.model.schools[school.upper()] @@ -116,6 +115,15 @@ def _get_department( except KeyError as e: raise AccessException( f'No quarter in {school} with name: {quarter}.') from e + return quarter_view + + def get_department( + self, + school: str, + department: str, + quarter: str = LATEST, + ) -> owl.model.DepartmentQuarterView: + quarter_view = self.get_quarter(school, quarter) try: department_view = quarter_view.departments[department] except KeyError as e: @@ -123,4 +131,27 @@ def _get_department( f'No department in {quarter} with name: {department}.') from e return department_view + def get_urls( + self, school: str, quarter: str = LATEST + ) -> ty.Dict[str, ty.Dict[str, str]]: + quarter_view = self.get_quarter(school, quarter) + return { + department_view.name: { + course_view.name: { + 'course': course_view.name, + 'dept': course_view.department.name + } for course_view in department_view.courses + } for department_view in quarter_view.departments + } + +def generate_url(dept: str, course: str) -> ty.Dict[str, str]: + """ + This is a helper function that generates a url string from a passed + department and course for the /urls route. + :param dept: str identifier for department + :param course: str + + :return: dict[str, str] + """ + return {"dept": f"{dept}", "course": f"{course}"} \ No newline at end of file diff --git a/owl/input.py b/owl/input.py index 17d7e5c..0ecd797 100644 --- a/owl/input.py +++ b/owl/input.py @@ -18,3 +18,7 @@ class GetOneInput(flask_inputs.Inputs): class GetManyInput(flask_inputs.Inputs): json = [JsonSchema(schema=get_definition('get_many'))] + + +class GetListInput(flask_inputs.Inputs): + json = [JsonSchema(schema=get_definition('get_list'))] diff --git a/owl/request.py b/owl/request.py deleted file mode 100644 index 207575d..0000000 --- a/owl/request.py +++ /dev/null @@ -1,279 +0,0 @@ -""" -This module contains request classes, which validate and facilitate -access to data passed by api users in get requests. - -This module also allows more readable results to api users, that list -any potential issues with their requests, before they are acted on, -and without requiring error code lookups. -""" - -import typing as ty - - -class RequestException(ValueError): - """ - Exception raised when a request has received bad data. - RequestException may be given a string to return to the caller - as part of a 400 error. - """ - def __init__( - self, - *args, - user_msg='Bad Request', - issues: ty.List['Request.Issue'] = () - ) -> None: - super().__init__(*args) - self.user_msg = user_msg - self.issues = issues - - -class Request: - - _check_for_unused_args = True # Able to be changed in subclasses. - _raise_issues_after_unpacking = True # Able to be changed in subclasses. - - def __init__(self, *args, **kwargs): - self.issues: ty.List['Request.Issue'] = [] - self.checked_fields: ty.Dict[str, 'Request.Field'] = {} - self.has_parent: bool = kwargs.pop('__has_parent', False) - - req_args = RequestArguments(*args, **kwargs) - self.unpack(req_args) - - def check_for_unused_args(): - if not req_args: - raise TypeError( - 'Unused arguments can only be detected if request ' - 'arguments are wrapped with a RequestArguments instance. ' - 'This can be done with the wrap_request_arguments ' - 'decorator.') - for k, v in req_args.unused().items(): - self.issues.append(Request.Issue(repr(k), 'Unused argument.')) - - if self._check_for_unused_args: - check_for_unused_args() - if all(( - self._raise_issues_after_unpacking, - self.issues, - not self.has_parent)): - self.raise_issues() - - def unpack(self, request_args: 'RequestArguments') -> None: - """ - This method should be overridden by subclasses to define how - arguments should be unpacked by request. - :param request_args: RequestArguments - :return: None - """ - raise NotImplementedError( - 'unpack() should be overridden by subclasses.') - - def __setattr__(self, k: ty.Any, v: ty.Any) -> None: - """ - If attribute assigned to has same key as a Request.Field - instance owned by the class, the assigned value is first - validated by the Field, before being assigned, and any issues - are added to the collection maintained by Request. - :param k: str - :param v: Any - :return: None - """ - try: - field: 'Request.Field' = type(self).__dict__[k] - except KeyError: - pass - else: - issues = field.validate(v) - self.issues += issues - v = field.modify(v) - super().__setattr__(k, v) - - def raise_issues(self) -> None: - """ - Raises a RequestException which contains string representations - of all issues so far encountered by Request with passed data. - :return: None - """ - user_msg = '\n'.join(issue.user_string for issue in self.issues) - raise RequestException( - 'Issues were encountered during request initialization: ' - f'{self.issues}', - user_msg=f'The Following issues were encountered' - f'while creating request: {user_msg}', - issues=self.issues - ) - - def __repr__(self) -> str: - return f'Request[]' - - class Field: - """ - Class assisting with validation of passed data, to allow better - feedback to api user. - """ - def __init__( - self, - t: ty.TypeVar = object, - valid: ty.Container[ty.Any] = (), - default=None, - type_coercion=False, - modifier: ty.Callable[[ty.Any], ty.Any] = None, - validator: ty.Callable[[ty.Any], ty.List[str]] = None - ) -> None: - self.name: str = '' # Set externally by Request - self.type = t - self.valid_values = valid - self.default = default - self.type_coercion = type_coercion - self.modifier = modifier - self.validator = validator - - def validate(self, value: ty.Any) -> ty.List['Request.Issue']: - """ - Validates passed value. - :param value: value that may be assigned to field in a - Request instance. - :return: List[Request.Issue] - """ - issues = [] - - try: - modified_v = self.modify(value) - except (ValueError, TypeError): - modified_v = value - - if self.type_coercion: - try: - coerced_value = self.type(modified_v) - except (ValueError, TypeError): - coerced_value = modified_v - else: - coerced_value = modified_v - - def validate_type(v: ty.Any) -> 'Request.Issue' or None: - if isinstance(coerced_value, self.type): - return None - - if self.type_coercion: - return self.issue( - f'Cannot coerce passed value: {v} of type: ' - f'{type(v).__name__} to expected ' - f'type: {self.type.__name__}.' - ) - return self.issue( - f'Expected type: {self.type.__name__}, but passed value' - f'{v} has type: {type(v).__name__}.') - - def validate_value(v: ty.Any) -> 'Request.Issue' or None: - if coerced_value not in self.valid_values: - if coerced_value == v: - return self.issue( - f'Passed value: {v} is not valid. List of valid' - f'Values: {self.valid_values}.') - return self.issue( - f'Passed value: {v}, (modified to {coerced_value})' - f'is not valid. List of valid values: ' - f'{self.valid_values}.' - ) - return None - - if self.type != object: - issue = validate_type(value) - if issue: - issues.append(issue) - - if self.valid_values: - issue = validate_value(value) - if issue: - issues.append(issue) - - if self.validator: - issues += (self.issue(msg) for msg in - self.validator(coerced_value)) - - return issues - - def modify(self, value: ty.Any) -> ty.Any: - """ - Modifies passed value so that it can be stored in field. - :param value: Any - :return: Any - """ - return self.modifier(value) if self.modifier else value - - def issue(self, msg: str) -> 'Request.Issue': - """ - Creates a new issue using name of field as Issue - field param. - This is intended as a helper method for internal use but - may also be used externally. - :param msg: str - :return: Request.Issue - """ - return Request.Issue(self.name, msg) - - def __repr__(self) -> str: - return f'Request.Field[{self.name}, type: {self.type.__name__}]' - - class SubRequestField(Field): - def __init__(self, t): - super().__init__( - t, - modifier=lambda args: t(args, __has_parent=True), - validator=lambda request: - [f'Sub-Field {repr(issue.field_name)}: {issue.msg}' - for issue in request.issues] - ) - - class Issue: - """ - Class detailing a specific issue found with a request. - """ - - def __init__(self, field_name: str, msg: str) -> None: - self.field_name = field_name - self.msg = msg - - @property - def user_string(self) -> str: - """ - Issue representation as a string that may be presented to - the user of the api. - :return: str - """ - return f'{self.field_name}: {self.msg}' - - def __repr__(self) -> str: - """ - Issue representation as a string intended for display to - a developer. - :return: str - """ - return f'Issue[field: {self.field_name}, msg: {self.msg}]' - - -class RequestArguments(dict): - """ - Dictionary subclass that is intended to wrap arguments passed to - a Request. - RequestArguments will track which arguments have been used, in - order to help identify surplus or misnamed fields. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.retrieved = set() - - def __getitem__(self, k): - self.retrieved.add(k) - return super().__getitem__(k) - - def get(self, k, d=None): - self.retrieved.add(k) - return super().get(k, d) - - def unused(self) -> ty.Dict[ty.Any, ty.Any]: - """ - Returns dictionary of unused keys and their associated values. - :return: dict - """ - return {k: v for k, v in self.items() if k not in self.retrieved} diff --git a/owl/schema.py b/owl/schema.py index 504f6cf..4ee6ccb 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -69,6 +69,15 @@ 'filter': {'$ref': '#/definitions/filter'}, }, 'required': ['courses'] + }, + 'get_list': { + 'type': 'object', + 'properties': { + 'quarter': {'type': 'string'}, + 'department': {'type': 'string'}, + 'course': {'type': 'string'}, + }, + 'required': ['department'] } } diff --git a/server.py b/server.py index 3111de1..fdaa9b0 100644 --- a/server.py +++ b/server.py @@ -1,14 +1,22 @@ from os.path import join from collections import defaultdict -from re import match -import itertools as itr import typing as ty # 3rd party from flask import Flask, jsonify, request, render_template from tinydb import TinyDB -from maya import when, MayaInterval + +# Owl modules +# The reason for the relatively verbose imports is to avoid names +# like 'model', 'request' or 'filter' being in the module namespace, +# which may easily be accidentally overridden or mistaken for variables +# of other types. +import settings +import owl.model # Bring in all objects from model without an ambiguous name. +import owl.input +import owl.access +import owl.filter # Quart config @@ -34,22 +42,61 @@ def add_cors_headers(response): FH_TYPE_ALIAS = {'standard': None, 'online': 'W', 'hybrid': 'Y'} DA_TYPE_ALIAS = {'standard': None, 'online': 'Z', 'hybrid': 'Y'} +# fields +DEPARTMENT_KEY = 'dept' +COURSE_KEY = 'course' +QUARTER_KEY = 'quarter' +FILTER_KEY = 'filters' + +FILTER_STATUS_KEY = 'status' +FILTER_TYPES_KEY = 'types' +FILTER_DAYS_KEY = 'days' +FILTER_TIME_KEY = 'time' + +data_model = owl.model.DataModel(settings.DB_DIR) +accessor = owl.access.ModelAccessor(data_model) + @application.route('/') def idx(): return render_template('index.html') +def basic_checks(input_type = None): + """ + Wrapper for an api call that handles common exception cases. + :param input_type: + :return: Callable + """ + def decorator(f): + def api_method_wrapper(*args, **kwargs): + if input_type: + inputs = input_type(request) + if not inputs.validate(): + return jsonify(success=False, errors=inputs.errors) + try: + response = f(*args, **kwargs) + except owl.access.AccessException as e: + return e.user_msg, 404 # Data not found + else: + return response + + api_method_wrapper.__name__ = f.__name__ + '_wrapper' + return api_method_wrapper + + return decorator + + @application.route('//single', methods=['GET']) -def api_one(campus): +@basic_checks(input_type=owl.input.GetOneInput) +def api_one(campus: str): """ `/single` with [GET] handles a single request to get a whole department or a whole course listing from the database It expects a mandatory query parameter `dept` and an optional `course`. - Example: - {'dept': 'CS', 'course': '2C'} + Example: {'dept': 'CS', 'course': '2C'} If only `dept` is requested, it checked for its existence in the database and then returns it. @@ -60,22 +107,24 @@ def api_one(campus): :return: 200 - Found entry and returned data successfully to the user. - :return: 404 - Could not find entry - """ - if campus not in CAMPUS_LIST: - return 'Error! Could not find campus in database', 404 - - raw = request.args - qp = {k: v.upper() for k, v in raw.items()} - - db = TinyDB(join(DB_ROOT, f'{CAMPUS_LIST[campus]}_database.json')) - data = get_one(db, qp, filters=dict()) - json = jsonify(data) - return (json, 200) if data else ( - 'Error! Could not find given selectors in database', 404) + :return: 400 - Badly formatted request. + :return: 404 - Could not find entry. + """ + # 'campus' refers here to the school, not the physical location, + # and is not the same thing as the 'campus' field in section data. + # This is something that should probably be refactored if possible. + data = accessor.get_one( + school=campus.upper(), + quarter=request.args.get(QUARTER_KEY, owl.access.LATEST), + department=request.args[DEPARTMENT_KEY], # No default; required field. + course=request.args.get(COURSE_KEY, owl.access.ALL), + section_filter=_get_section_filter(request.args), # May return None. + ) + return jsonify(data), 200 @application.route('//batch', methods=['POST']) +@basic_checks(input_type=owl.input.GetManyInput) def api_many(campus): """ `/batch` with [POST] handles a batch request to get many @@ -109,184 +158,33 @@ def api_many(campus): to the user. :return: 404 - Could not find one or more entries. """ - if campus not in CAMPUS_LIST: - return 'Error! Could not find campus in database', 404 - - db = TinyDB(join(DB_ROOT, f'{CAMPUS_LIST[campus]}_database.json')) - raw = request.get_json() - - data = raw['courses'] - filters = raw['filters'] if ('filters' in raw) else dict() - - courses = get_many(db=db, data=data, filters=filters) - if not courses: # null case from get_one (invalid param or filter) - return 'Error! Could not find one or more course ' \ - 'selectors in database', 404 - - json = jsonify({'courses': courses}) - return json, 200 - - -def get_one(db: TinyDB, data: dict, filters: dict): - """ - This is a helper used by the `/get` route to extract course data. - It works for both [GET] and [POST] and fetches data from the database - - :param db: (TinyDB) Database to retrieve data from - :param data: (dict) The query param or the POST body dict - :param filters: (dict) A optional dictionary of filters to be - passed to filter_courses() - - :return: course: (dict) A singular course listing from the database - (if it passes filters) - """ - - course = dict() - data_dept = data['dept'] - if data_dept in db.tables(): - table = db.table(f'{data_dept}') - entries = table.all() - - if 'course' not in data: - return entries - - data_course = data['course'] + course_filter = _get_section_filter(request.args) + def get_sub_request_data(args): try: - course = next((e[f'{data_course}'] for e in entries - if f'{data_course}' in e)) - if filters: - filter_courses(filters, course) - - except StopIteration: - return dict() - - return course + return accessor.get_one( + school=campus, + department=args[DEPARTMENT_KEY], + course=args.get(COURSE_KEY, owl.access.ALL), + quarter=args.get(QUARTER_KEY, owl.access.LATEST), + section_filter=_get_section_filter(args) or course_filter + ) + except owl.access.AccessException: + return {} + data = list(map(get_sub_request_data, request.args['courses'])) -def get_many(db: TinyDB, data: dict(), filters: dict()): - ret = [] + if any(not sub_data for sub_data in data): + response_code = 404 + else: + response_code = 200 - for course in data: - d = get_one(db, course, filters=filters) - if not d: # null case from get_one (invalid param or filter) - continue - ret.append(d) - - return ret - - -def filter_courses(filters: ty.Dict[str, ty.Any], course): - """ - This is a helper called by get_one() that filters a set of classes - based on some filter conditionals - - Be careful with these as they can be limiting on the data, often - returning as 404 when one of the courses does not pass the filter. - Additionally, filters like status and types can be extremely - limiting on the data. Some courses won't even offer non-online - classes. Below is an example input with these filters. - - Example filters: { - 'status': {'open':1, 'waitlist':0, 'full':0}, - 'types': {'standard':1, 'online':1, 'hybrid':0}, - 'days': {'M':1, 'T':0, 'W':1, 'Th':0, 'F':0, 'S':0, 'U':0}, - 'time': {'start':'8:30 AM', 'end':'9:40 PM'} - } - - :param filters: `status` - filter by the availability of a course - (Open, Waitlist, Full) - `types` - filter by the format of the course - (In Person, Online, Hybrid) - `days` - filter by the days the course should be - limited to (M, T, W, Th, F, S, U) - `time` - filter by a specified time interval - (8:30 AM - 9:40 PM) - :param course: (dict) the mutable course listing - - :return: None - """ - # Nested functions filter courses by taking a course key and - # returning a boolean indicating whether they should be included - # or excluded. True if they are to be included, False if excluded. - - def status_filter(course_key) -> bool: - # {'open':0, 'waitlist':0, 'full':0} - if 'status' not in filters: - return True - # Create 'mask' of course statuses that are to be included. - status_mask = {k for (k, v) in filters['status'].items() if v} - # Return True only if course status is in mask. - return course[course_key][0]['status'].lower() in status_mask - - def type_filter(course_key) -> bool: - # {'standard':1, 'online':1, 'hybrid':0} - if 'types' not in filters: - return True - # Get course section - section = get_key(course[course_key][0]['course']) - mask = set() - for k, v in filters['types'].items(): - if not v: - continue - mask.add(FH_TYPE_ALIAS[k]) - mask.add(DA_TYPE_ALIAS[k]) - return section[1] in mask - - def day_filter(course_key) -> bool: - # {'M':1, 'T':0, 'W':1, 'Th':0, 'F':0, 'S':0, 'U':0} - if 'days' in filters: - # create set of days that are allowed by passed filters - mask = {k for (k, v) in filters['days'].items() if v} - for class_ in course[course_key]: - days_match = match(DAYS_PATTERN, class_['days']) - course_days = {x for x in days_match.groups() if x} if \ - days_match else {} - # Return False if course day is not in mask. - if not course_days <= mask: - return False - return True - - def time_filter(course_key) -> bool: - # {'start':'8:30 AM', 'end':'9:40 PM'} - if 'time' in filters: - f_range = MayaInterval( - start=when(filters['time']['start']), - end=when(filters['time']['end'])) - for class_ in course[course_key]: - if '-' in class_['time']: - data = class_['time'].split('-') - class_range = MayaInterval( - start=when(data[0]), end=when(data[1])) - if not f_range.contains(class_range): - return False - return True - - def filter_all(k) -> bool: - return all((status_filter(k), type_filter(k), - day_filter(k), time_filter(k))) - - # remove each key that is evaluated false by filter_all - for key in itr.filterfalse(filter_all, set(course.keys())): - del course[key] - - -def get_key(key): - """ - This is the key parser for the course names - - :param key: (str) The unparsed string containing the course name - :return match_obj.groups(): (list) the string for the regex match - """ - k = key.split(' ') - i = 1 if len(k) < 3 else 2 - section = k[i] - - match_obj = match(COURSE_PATTERN, section) - return match_obj.groups() + json = jsonify({'courses': data}) + return json, response_code @application.route('//list', methods=['GET']) +@basic_checks(input_type=owl.input.GetListInput) def api_list(campus): """ `/list` with [GET] handles a single request to list department or @@ -305,21 +203,15 @@ def api_list(campus): if campus not in CAMPUS_LIST: return 'Error! Could not find campus in database', 404 - db = TinyDB(join(DB_ROOT, f'{CAMPUS_LIST[campus]}_database.json')) - - raw = request.args - qp = {k: v.upper() for k, v in raw.items()} + data = accessor.get_one( + school=campus.upper(), + quarter=request.args.get(QUARTER_KEY, owl.access.LATEST), + department=request.args[DEPARTMENT_KEY], # No default; required field. + course=request.args.get(COURSE_KEY, owl.access.ALL), + section_filter=_get_section_filter(request.args), # May return None. + ).keys() - if 'dept' not in qp: - return jsonify(', '.join(db.tables())), 200 - - qp_dept = qp['dept'] - if qp_dept in db.tables(): - table = db.table(f'{qp_dept}') - keys = set().union(*(d.keys() for d in table.all())) - return jsonify(', '.join(keys)), 200 - - return 'Error! Could not list', 404 + return ('Error! Could not list', 404) if not data else (jsonify(data), 200) @application.route('//urls', methods=['GET']) @@ -335,28 +227,13 @@ def api_list_url(campus): if campus not in CAMPUS_LIST: return 'Error! Could not find campus in database', 404 - db = TinyDB(join(DB_ROOT, f'{CAMPUS_LIST[campus]}_database.json')) - - data = defaultdict(list) - - for dept in db.tables(): - table = db.table(dept) - keys = set().union(*(d.keys() for d in table.all())) - data[f'{dept}'].append({k: generate_url(dept, k) for k in keys}) + data = accessor.get_urls(request.get()) return jsonify(data), 200 -def generate_url(dept: str, course: str) -> ty.Dict[str, str]: - """ - This is a helper function that generates a url string from a passed - department and course for the /urls route. - :param dept: str identifier for department - :param course: str - - :return: dict[str, str] - """ - return {"dept": f"{dept}", "course": f"{course}"} +def _get_section_filter(args): + return owl.filter.SectionFilter(args) if args[FILTER_KEY] else None if __name__ == '__main__': diff --git a/tests/model/test_access.py b/tests/model/test_access.py index 777bd7f..4401f6c 100644 --- a/tests/model/test_access.py +++ b/tests/model/test_access.py @@ -14,7 +14,7 @@ 'generate_test_data.py script') from e -class TestGetOne(TestCase): +class TestAccessor(TestCase): def test_get_one_dept(self): accessor = get_accessor('model_test_dir_a') result = accessor.get_one('fh', department='CS') diff --git a/tests/model/test_request.py b/tests/model/test_request.py deleted file mode 100644 index 40a053e..0000000 --- a/tests/model/test_request.py +++ /dev/null @@ -1,116 +0,0 @@ -from unittest import TestCase - -import owl.request - - -class TestRequest(TestCase): - def test_request_can_be_constructed_using_correct_args(self): - self.ExampleRequest(a='foo', b='bar') - # no checks needed here. - - def test_request_throws_request_error_with_invalid_type(self): - self.assertRaises( - owl.request.RequestException, - lambda: self.ExampleRequest({'a': 'foo', 'b': 5}) - ) - - def test_request_throws_request_error_with_invalid_value(self): - self.assertRaises( - owl.request.RequestException, - lambda: self.ExampleRequest({'a': 'foo', 'b': 'blah'}) - ) - - def test_correct_values_are_stored_in_request(self): - request = self.ExampleRequest({'a': 'foo', 'b': 'bar'}) - self.assertEqual('foo', request.test_field_1) - self.assertEqual('bar', request.test_field_2) - - def test_extra_values_cause_request_error_to_be_raised(self): - self.assertRaises( - owl.request.RequestException, - lambda: self.ExampleRequest({'a': 'foo', 'b': 'bar', 'c': 'blah'}) - ) - - def test_values_are_modified_as_expected(self): - class CapitalizingRequest(owl.request.Request): - test_field = owl.request.Request.Field( - t=str, modifier=lambda s: s.upper()) - - def unpack(self, request_args: 'owl.request.RequestArguments'): - self.test_field = request_args.get('a') - - request = CapitalizingRequest({'a': 'foo'}) - self.assertEqual('FOO', request.test_field) - - def test_sub_request_field_propagates_issues(self): - self.assertRaises( - owl.request.RequestException, - lambda: SuperRequest({'a': {'b': 5}}) - ) - - def test_that_raising_of_sub_request_issues_is_delayed(self): - class Super2Request(owl.request.Request): - sub_request_a = owl.request.Request.SubRequestField(SubRequest) - sub_request_b = owl.request.Request.SubRequestField(SubRequest) - - def unpack(self, request_args: 'owl.request.RequestArguments'): - self.sub_request_a = request_args.get('a') - self.sub_request_b = request_args.get('b') - - try: - Super2Request({'a': {'b': 6}, 'b': {'b': 5}}) - except owl.request.RequestException as e: - self.assertEqual(2, len(e.issues)) - - def test_sub_request_field_does_not_raise_issues_with_correct_args(self): - req = SuperRequest({'a': {'b': 'foo'}}) - self.assertEqual('foo', req.sub_request.test_field) - - class ExampleRequest(owl.request.Request): - test_field_1 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) - test_field_2 = owl.request.Request.Field(t=str, valid=('foo', 'bar')) - - def unpack(self, request_args: owl.request.RequestArguments): - self.test_field_1: str = request_args['a'] - self.test_field_2: str = request_args['b'] - - -class TestRequestField(TestCase): - def test_field_with_no_valid_field_arg_passed_accepts_all_values(self): - field = owl.request.Request.Field(t=int) - self.assertEqual([], field.validate(2)) # Check no issue is returned. - - def test_field_with_valid_field_arg_passed_accepts_valid_values(self): - field = owl.request.Request.Field(t=int, valid=(2,)) - self.assertEqual([], field.validate(2)) # Check no issue is returned. - - def test_field_returns_issue_when_invalid_value_passed(self): - field = owl.request.Request.Field(t=int, valid=(2,)) - self.assertEqual(1, len(field.validate(3))) - - def test_custom_validator_can_add_issue_successfully(self): - def must_exceed_5(x): - if x > 5: - return [] - else: - return ['Value was not greater than 5'] - - field = owl.request.Request.Field(t=int, validator=must_exceed_5) - self.assertEqual(1, len(field.validate(4))) - - -######### - - -class SubRequest(owl.request.Request): - test_field = owl.request.Request.Field(t=str) - - def unpack(self, request_args: 'owl.request.RequestArguments'): - self.test_field = request_args.get('b') - - -class SuperRequest(owl.request.Request): - sub_request = owl.request.Request.SubRequestField(SubRequest) - - def unpack(self, request_args: 'owl.request.RequestArguments'): - self.sub_request = request_args.get('a') \ No newline at end of file From c2b2043ced323d70d7278dadebcebc2541348214 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 00:19:07 -0700 Subject: [PATCH 33/60] Added input tests, corrected various issues. --- owl/input.py | 29 ++++++++++++++++++++++++++--- owl/schema.py | 29 +++++++++++++++-------------- tests/model/test_input.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 17 deletions(-) create mode 100644 tests/model/test_input.py diff --git a/owl/input.py b/owl/input.py index 0ecd797..b90b138 100644 --- a/owl/input.py +++ b/owl/input.py @@ -8,17 +8,40 @@ """ import flask_inputs from flask_inputs.validators import JsonSchema +from itertools import chain from owl.schema import get_definition -class GetOneInput(flask_inputs.Inputs): +class Inputs(flask_inputs.Inputs): + def validate(self): + """Validate incoming request data. Returns True if all data is valid. + Adds each of the validator's error messages to Inputs.errors if not valid. + + :returns: Boolean + """ + success = True + + for attribute, form in self._forms.items(): + if '_input' in form._fields: + form.process(self._get_values(attribute, coerse=False)) + else: + form.process(self._get_values(attribute)) + + if not form.validate(): + success = False + self.errors += chain(*form.errors.values()) + + return success + + +class GetOneInput(Inputs): json = [JsonSchema(schema=get_definition('get_one'))] -class GetManyInput(flask_inputs.Inputs): +class GetManyInput(Inputs): json = [JsonSchema(schema=get_definition('get_many'))] -class GetListInput(flask_inputs.Inputs): +class GetListInput(Inputs): json = [JsonSchema(schema=get_definition('get_list'))] diff --git a/owl/schema.py b/owl/schema.py index 4ee6ccb..27e305c 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -3,29 +3,29 @@ 'status_filter': { 'type': 'object', 'properties': { - 'open': {'type': 'int'}, - 'waitlist': {'type': 'int'}, - 'full': {'type': 'int'}, + 'open': {'type': 'number'}, + 'waitlist': {'type': 'number'}, + 'full': {'type': 'number'}, } }, 'type_filter': { 'type': 'object', 'properties': { - 'standard': {'type': 'int'}, - 'online': {'type': 'int'}, - 'hybrid': {'type': 'int'} + 'standard': {'type': 'number'}, + 'online': {'type': 'number'}, + 'hybrid': {'type': 'number'} } }, 'day_filter': { 'type': 'object', 'properties': { - 'M': {'type': 'int'}, - 'T': {'type': 'int'}, - 'W': {'type': 'int'}, - 'Th': {'type': 'int'}, - 'F': {'type': 'int'}, - 'S': {'type': 'int'}, - 'U': {'type': 'int'}, + 'M': {'type': 'number'}, + 'T': {'type': 'number'}, + 'W': {'type': 'number'}, + 'Th': {'type': 'number'}, + 'F': {'type': 'number'}, + 'S': {'type': 'number'}, + 'U': {'type': 'number'}, } }, 'time_filter': { @@ -83,6 +83,7 @@ def get_definition(definition: str): - d = definitions[definition] + d = definitions[definition].copy() d['definitions'] = definitions + return d diff --git a/tests/model/test_input.py b/tests/model/test_input.py new file mode 100644 index 0000000..949f0fe --- /dev/null +++ b/tests/model/test_input.py @@ -0,0 +1,30 @@ +from unittest import TestCase, SkipTest + +import owl.input + + +class TestInput(TestCase): + def test_get_one_allows_proper_input(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + @SkipTest + def test_get_one_disallows_input_without_department(self): + raw = NotRequest({ + 'quarter': '000011', + 'course': '1A', + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertEqual(1, len(inputs.errors)) + + +class NotRequest: + def __init__(self, args): + self.json = args From 77214580b5d2fa06de39a8ce93c3663bfd860934 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 00:20:40 -0700 Subject: [PATCH 34/60] Removed unused generate_url method. --- owl/access.py | 12 ------------ tests/test_server.py | 10 ---------- 2 files changed, 22 deletions(-) diff --git a/owl/access.py b/owl/access.py index f8da2f3..a2110c8 100644 --- a/owl/access.py +++ b/owl/access.py @@ -143,15 +143,3 @@ def get_urls( } for course_view in department_view.courses } for department_view in quarter_view.departments } - - -def generate_url(dept: str, course: str) -> ty.Dict[str, str]: - """ - This is a helper function that generates a url string from a passed - department and course for the /urls route. - :param dept: str identifier for department - :param course: str - - :return: dict[str, str] - """ - return {"dept": f"{dept}", "course": f"{course}"} \ No newline at end of file diff --git a/tests/test_server.py b/tests/test_server.py index 22ac0e4..6a2d400 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -4,7 +4,6 @@ from tinydb import TinyDB import settings -from server import generate_url # Try to get generated data. try: @@ -15,12 +14,3 @@ 'generate_test_data.py script') from e test_database = TinyDB(join(settings.TEST_RESOURCES_DIR, 'test_database.json')) - - -class TestGenerateURL(TestCase): - def test_sample_url_can_be_generated(self): - self.assertEqual( - test_data.test_sample_url_can_be_generated_data, - generate_url('test_dept', 'test_course') - ) - From 09cee4f28fc25928f360086c30b0f1fea6068f9a Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 00:31:48 -0700 Subject: [PATCH 35/60] Cleanup of server.py imports and constants. --- server.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/server.py b/server.py index fdaa9b0..50f04de 100644 --- a/server.py +++ b/server.py @@ -1,11 +1,11 @@ -from os.path import join -from collections import defaultdict - -import typing as ty +""" +This module contains functions for receiving user input and formatting +returned data. +Retrieval logic of data is done in access.py. +""" # 3rd party from flask import Flask, jsonify, request, render_template -from tinydb import TinyDB # Owl modules # The reason for the relatively verbose imports is to avoid names @@ -32,15 +32,7 @@ def add_cors_headers(response): ) application.after_request(add_cors_headers) -DB_ROOT = 'db/' - -CAMPUS_LIST = {'fh': '201911', 'da': '201912', 'test': 'test'} - -COURSE_PATTERN = r'[FD]0*(\d*\w?)\.?\d*([YWZH])?' -DAYS_PATTERN = f"^{'(M|T|W|Th|F|S|U)?'*7}$" - -FH_TYPE_ALIAS = {'standard': None, 'online': 'W', 'hybrid': 'Y'} -DA_TYPE_ALIAS = {'standard': None, 'online': 'Z', 'hybrid': 'Y'} +CAMPUS_LIST = {'fh', 'da'} # fields DEPARTMENT_KEY = 'dept' @@ -48,11 +40,6 @@ def add_cors_headers(response): QUARTER_KEY = 'quarter' FILTER_KEY = 'filters' -FILTER_STATUS_KEY = 'status' -FILTER_TYPES_KEY = 'types' -FILTER_DAYS_KEY = 'days' -FILTER_TIME_KEY = 'time' - data_model = owl.model.DataModel(settings.DB_DIR) accessor = owl.access.ModelAccessor(data_model) From d5a20f30192027b6cc426d6ca80a7fc3b37ea0e7 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 00:52:36 -0700 Subject: [PATCH 36/60] Code cleanup --- owl/access.py | 5 ++--- owl/input.py | 3 ++- owl/schema.py | 2 +- server.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/owl/access.py b/owl/access.py index a2110c8..94aa065 100644 --- a/owl/access.py +++ b/owl/access.py @@ -50,9 +50,8 @@ def get_one( if course == ALL: return self.get_department_data( school, department, quarter, section_filter) - else: - return self.get_course_data( - school, department, course, quarter, section_filter) + return self.get_course_data( + school, department, course, quarter, section_filter) def get_department_data( self, diff --git a/owl/input.py b/owl/input.py index b90b138..4106a59 100644 --- a/owl/input.py +++ b/owl/input.py @@ -6,10 +6,11 @@ any potential issues with their requests, before they are acted on, and without requiring error code lookups. """ -import flask_inputs from flask_inputs.validators import JsonSchema from itertools import chain +import flask_inputs + from owl.schema import get_definition diff --git a/owl/schema.py b/owl/schema.py index 27e305c..9c4c43f 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -85,5 +85,5 @@ def get_definition(definition: str): d = definitions[definition].copy() d['definitions'] = definitions - + return d diff --git a/server.py b/server.py index 50f04de..1a321f0 100644 --- a/server.py +++ b/server.py @@ -49,7 +49,7 @@ def idx(): return render_template('index.html') -def basic_checks(input_type = None): +def basic_checks(input_type=None): """ Wrapper for an api call that handles common exception cases. :param input_type: From c94645f32f1cd2b43624b502e85528732f462b41 Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 00:53:30 -0700 Subject: [PATCH 37/60] added flask-inputs to pipfile --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 4977ced..4b9ad42 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ maya = "*" pytest = "*" flask = "*" pylint = "*" +flask-inputs = "*" [dev-packages] From 0d4d3a9da6fd98c1c03f48223654366f09cf0cdb Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 00:57:40 -0700 Subject: [PATCH 38/60] Added jsonschema to dependencies. --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 4b9ad42..805030b 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ pytest = "*" flask = "*" pylint = "*" flask-inputs = "*" +jsonschema = "*" [dev-packages] From 19c44f82c857d7f8566edeccebcf6d622349724f Mon Sep 17 00:00:00 2001 From: TryExcept Date: Mon, 11 Jun 2018 01:02:48 -0700 Subject: [PATCH 39/60] Removed unused test_server.py (may be restored in the future) --- tests/test_server.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 tests/test_server.py diff --git a/tests/test_server.py b/tests/test_server.py deleted file mode 100644 index 6a2d400..0000000 --- a/tests/test_server.py +++ /dev/null @@ -1,16 +0,0 @@ -from os.path import join -from unittest import TestCase, SkipTest - -from tinydb import TinyDB - -import settings - -# Try to get generated data. -try: - from tests.test_db import data as test_data -except ImportError as e: - raise ImportError('Test Data could not be imported. If the data.py file ' - 'does not exist, it can be generated using the ' - 'generate_test_data.py script') from e - -test_database = TinyDB(join(settings.TEST_RESOURCES_DIR, 'test_database.json')) From ecc724fa5aa847ac6df29bf0b1d4e0327e862c14 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 01:04:39 -0700 Subject: [PATCH 40/60] Re-ordered imports in input.py to be pep-8 compliant. --- owl/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owl/input.py b/owl/input.py index 4106a59..e9690e6 100644 --- a/owl/input.py +++ b/owl/input.py @@ -6,9 +6,9 @@ any potential issues with their requests, before they are acted on, and without requiring error code lookups. """ -from flask_inputs.validators import JsonSchema from itertools import chain +from flask_inputs.validators import JsonSchema import flask_inputs from owl.schema import get_definition From 598e1e62d41ec5be99077d94ca2a2f6108b61318 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 01:33:15 -0700 Subject: [PATCH 41/60] DataModel now accepts files with name form '123456_database.json' --- owl/model.py | 3 ++- owl/schema.py | 1 - tests/generate_test_data.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/owl/model.py b/owl/model.py index 79096ea..6bbb23e 100644 --- a/owl/model.py +++ b/owl/model.py @@ -112,9 +112,10 @@ def find_quarters() -> ty.Dict[str, 'QuarterView']: """ quarters: ty.Dict[str, 'QuarterView'] = {} for file_name in os.listdir(self.db_dir): - name, ext = os.path.splitext(file_name) + long_name, ext = os.path.splitext(file_name) if ext != DB_EXT: continue # Ignore non-database files + name = long_name[:6] if all(c in string.digits for c in name): quarters[name] = QuarterView.get_quarter(self, name) return quarters diff --git a/owl/schema.py b/owl/schema.py index 9c4c43f..343e89c 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -85,5 +85,4 @@ def get_definition(definition: str): d = definitions[definition].copy() d['definitions'] = definitions - return d diff --git a/tests/generate_test_data.py b/tests/generate_test_data.py index eded553..97ce2b8 100644 --- a/tests/generate_test_data.py +++ b/tests/generate_test_data.py @@ -42,9 +42,9 @@ def generate_data_py(): def generate_data_model_test_files(): copies = { 'model_test_dir_a': ( - '000011', - '000012', - '000021', + '000011_database', + '000012_database', + '000021_database', ) } # create directories From e00c23e7de6622275dbf17de1f0fbf8c0a19926a Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 01:47:57 -0700 Subject: [PATCH 42/60] Fixed quarter db file path generation. --- owl/model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/owl/model.py b/owl/model.py index 6bbb23e..f84d781 100644 --- a/owl/model.py +++ b/owl/model.py @@ -11,6 +11,7 @@ import maya DB_EXT = '.json' +DB_SUFFIX = '_database' FH = 'FH' DA = 'DA' SCHOOL_NAMES_BY_CODE = { @@ -113,7 +114,7 @@ def find_quarters() -> ty.Dict[str, 'QuarterView']: quarters: ty.Dict[str, 'QuarterView'] = {} for file_name in os.listdir(self.db_dir): long_name, ext = os.path.splitext(file_name) - if ext != DB_EXT: + if ext != DB_EXT or not long_name.endswith(DB_SUFFIX): continue # Ignore non-database files name = long_name[:6] if all(c in string.digits for c in name): @@ -332,7 +333,7 @@ def path(self) -> str: Gets path to database file which contains quarter data. :return: str path """ - return os.path.join(self.model.db_dir, self.name) + DB_EXT + return os.path.join(self.model.db_dir, self.name) + DB_SUFFIX + DB_EXT @property def departments(self) -> 'Departments': From 69a6c220818ef485d59ac2bea55b81cf82515930 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 02:05:27 -0700 Subject: [PATCH 43/60] Added some documentation. --- owl/access.py | 15 +++++++++++++++ owl/input.py | 8 ++++++-- owl/model.py | 15 +++++++++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/owl/access.py b/owl/access.py index 94aa065..453d2f2 100644 --- a/owl/access.py +++ b/owl/access.py @@ -34,6 +34,9 @@ def __init__(self, *args, user_msg='Bad Request'): class ModelAccessor: + """ + Handles access of data from a specific data model. + """ def __init__(self, model: owl.model.DataModel) -> None: self.model = model @@ -47,6 +50,18 @@ def get_one( quarter: str = LATEST, section_filter: owl.filter.SectionFilter = None ) -> ty.Union[owl.model.COURSE_DATA_T, owl.model.DEPT_DATA_T]: + """ + Retrieves data for a department or course, as identified by + the api user. + If no course is passed, all of the specified department's data + is returned. + :param school: str (ex: 'fh') + :param department: str (ex: 'CS') + :param course: str (ex: 1A) (optional) + :param quarter: str (ex: '201812') + :param section_filter: filter kwargs + :return: COURSE_DATA_T or DEPT_DATA_T + """ if course == ALL: return self.get_department_data( school, department, quarter, section_filter) diff --git a/owl/input.py b/owl/input.py index e9690e6..3ef71ac 100644 --- a/owl/input.py +++ b/owl/input.py @@ -1,6 +1,6 @@ """ -This module contains request classes, which validate and facilitate -access to data passed by api users in get requests. +This module contains input classes, which validate data passed by api +users in requests. This module also allows more readable results to api users, that list any potential issues with their requests, before they are acted on, @@ -15,6 +15,10 @@ class Inputs(flask_inputs.Inputs): + """ + Extends flask_inputs.Input in order to fix issues with validate method. + """ + def validate(self): """Validate incoming request data. Returns True if all data is valid. Adds each of the validator's error messages to Inputs.errors if not valid. diff --git a/owl/model.py b/owl/model.py index f84d781..48cd1eb 100644 --- a/owl/model.py +++ b/owl/model.py @@ -493,6 +493,9 @@ def __repr__(self): class SectionQuarterView: + """ + View onto course section data in a specific quarter. + """ # All these fields should be equal if multiple entries exist. EQUAL_FIELDS = ( COURSE_ID_KEY, CRN_KEY, DESCRIPTION_KEY, STATUS_KEY, UNITS_KEY, @@ -775,11 +778,19 @@ def __init__( self.interval = maya.MayaInterval(start, end) @property - def start(self): + def start(self) -> maya.MayaDT: + """ + Gets start time of class meeting. + :return: MayaDT + """ return self.interval.start @property - def end(self): + def end(self) -> maya.MayaDT: + """ + Gets end time of class meeting. + :return: MayaDT + """ return self.interval.end From 1932bdcb35b3f6ad5add0d1d5d8d07ba4a18c7c6 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 11:19:42 -0700 Subject: [PATCH 44/60] Added more documentation to access.py methods. --- owl/access.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/owl/access.py b/owl/access.py index 453d2f2..8753047 100644 --- a/owl/access.py +++ b/owl/access.py @@ -75,6 +75,14 @@ def get_department_data( quarter: str = LATEST, section_filter: owl.filter.SectionFilter = None ) -> owl.model.DEPT_DATA_T: + """ + Gets data for a specific department in a single quarter. + :param school: str (ex: 'fh') + :param department: str (ex: 'CS') + :param quarter: str (ex: '201812') + :param section_filter: filter kwargs + :return: DEPT_DATA_T + """ # Find department view department_view = self.get_department(school, department, quarter) @@ -96,6 +104,15 @@ def get_course_data( quarter: str = LATEST, section_filter: owl.filter.SectionFilter = None ) -> owl.model.COURSE_DATA_T: + """ + Gets data for a specific course in a specific quarter. + :param school: str (ex: 'fh') + :param department: str (ex: 'CS') + :param course: str (ex: '1A') + :param quarter: str (ex: '201812') + :param section_filter: filter kwargs + :return: COURSE_DATA_T + """ # Find department view department_view = self.get_department(school, department, quarter) try: @@ -115,6 +132,13 @@ def get_quarter( school: str, quarter: str = LATEST, ) -> 'owl.model.QuarterView': + """ + Gets QuarterView, throwing readable access errors in the event + that a passed value is not found. + :param school: str (ex: 'fh') + :param quarter: str (ex: '201812') + :return: QuarterView + """ try: school_view: owl.model.SchoolView = \ self.model.schools[school.upper()] @@ -137,6 +161,13 @@ def get_department( department: str, quarter: str = LATEST, ) -> owl.model.DepartmentQuarterView: + """ + Gets department view of department data for a specific quarter. + :param school: str (ex: 'fh') + :param department: str (ex: 'CS') + :param quarter: str (ex: '201812') + :return: DepartmentQuarterView + """ quarter_view = self.get_quarter(school, quarter) try: department_view = quarter_view.departments[department] @@ -148,6 +179,12 @@ def get_department( def get_urls( self, school: str, quarter: str = LATEST ) -> ty.Dict[str, ty.Dict[str, str]]: + """ + Helps list all courses, their names and departments. + :param school: str (ex: 'fh') + :param quarter: str (ex: '201812') + :return: Dict[str, Dict[str, str]] + """ quarter_view = self.get_quarter(school, quarter) return { department_view.name: { From fb2a5c8e6864cb8ba4d756ba192729a13c55dcc9 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 11:32:11 -0700 Subject: [PATCH 45/60] Minor code cleanup and doc changes. --- owl/filter.py | 2 +- owl/input.py | 9 ++++++--- owl/schema.py | 5 +++++ server.py | 8 +++++++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/owl/filter.py b/owl/filter.py index c5753a0..c4e81ae 100644 --- a/owl/filter.py +++ b/owl/filter.py @@ -41,7 +41,7 @@ def check(self, section: owl.model.SectionQuarterView) -> bool: """ Determines whether passed course passes filter :param section: owl.model.SectionQuarterView - :return: bool + :return: bool (True if section passes filters) """ # Nested functions filter courses by taking a course key and diff --git a/owl/input.py b/owl/input.py index 3ef71ac..c9cb2b0 100644 --- a/owl/input.py +++ b/owl/input.py @@ -20,14 +20,17 @@ class Inputs(flask_inputs.Inputs): """ def validate(self): - """Validate incoming request data. Returns True if all data is valid. - Adds each of the validator's error messages to Inputs.errors if not valid. + """ + Validate incoming request data. Returns True if all data + is valid. + Adds each of the validator's error messages to Inputs.errors if + not valid. :returns: Boolean """ success = True - for attribute, form in self._forms.items(): + for attribute, form in self._forms.items(): # Fixed iterator here. if '_input' in form._fields: form.process(self._get_values(attribute, coerse=False)) else: diff --git a/owl/schema.py b/owl/schema.py index 343e89c..d656f00 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -83,6 +83,11 @@ def get_definition(definition: str): + """ + Gets schema definition with the passed name. + :param definition: str name of definition + :return: dict representation of json schema. + """ d = definitions[definition].copy() d['definitions'] = definitions return d diff --git a/server.py b/server.py index 1a321f0..8297c25 100644 --- a/server.py +++ b/server.py @@ -220,7 +220,13 @@ def api_list_url(campus): def _get_section_filter(args): - return owl.filter.SectionFilter(args) if args[FILTER_KEY] else None + """ + Produces a SectionFilter from passed arguments + :param args: request args + :return: SectionFilter or None + """ + return owl.filter.SectionFilter(**args[FILTER_KEY]) \ + if args[FILTER_KEY] else None if __name__ == '__main__': From 8493396bf5b1aebee6053e06640909ff9efe302b Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Mon, 11 Jun 2018 23:11:30 -0700 Subject: [PATCH 46/60] Fixed awkward wording in DataError docstring. --- owl/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/owl/model.py b/owl/model.py index 48cd1eb..8ee70fa 100644 --- a/owl/model.py +++ b/owl/model.py @@ -78,7 +78,7 @@ class DataError(Exception): Raised when accessed data cannot be handled. Distinct from ValueError because ValueError indicates invalid data has been passed, while DataError indicates data that has been - retrieved from model. + retrieved from model contains errors. """ From 3d790b9bbd3a122104a484e22286287ee82cf629 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 11:29:34 -0700 Subject: [PATCH 47/60] Added instructor and conflict filters to SectionFilter --- owl/access.py | 39 ++++++++++++++++++++++++++++++++------- owl/filter.py | 24 ++++++++++++++++++++++-- owl/model.py | 27 +++++++++++++++++++++++++++ owl/schema.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- server.py | 25 +++++++++++++++++++++++-- 5 files changed, 150 insertions(+), 14 deletions(-) diff --git a/owl/access.py b/owl/access.py index 8753047..8e72020 100644 --- a/owl/access.py +++ b/owl/access.py @@ -114,13 +114,7 @@ def get_course_data( :return: COURSE_DATA_T """ # Find department view - department_view = self.get_department(school, department, quarter) - try: - course_view = department_view.courses[course] - except KeyError as e: - raise AccessException( - f'No course in {department} with name: {course}.') from e - + course_view = self.get_course(school, department, course, quarter) return { section_view.crn: section_view.data for section_view in course_view.sections if @@ -176,6 +170,37 @@ def get_department( f'No department in {quarter} with name: {department}.') from e return department_view + def get_course( + self, + school: str, + department: str, + course: str, + quarter: str = LATEST, + ) -> owl.model.CourseQuarterView: + department_view = self.get_department(school, department, quarter) + try: + course_view = department_view.courses[course] + except KeyError as e: + raise AccessException( + f'No course in {department} with name: {course}.') from e + return course_view + + def get_section( + self, + school: str, + department: str, + course: str, + section: str, + quarter: str = LATEST, + ) -> owl.model.SectionQuarterView: + course_view = self.get_course(school, department, course, quarter) + try: + course_view = course_view.courses[course] + except KeyError as e: + raise AccessException( + f'No section in {course} with name: {section}.') from e + return course_view + def get_urls( self, school: str, quarter: str = LATEST ) -> ty.Dict[str, ty.Dict[str, str]]: diff --git a/owl/filter.py b/owl/filter.py index c4e81ae..400324f 100644 --- a/owl/filter.py +++ b/owl/filter.py @@ -30,12 +30,16 @@ def __init__( status: ty.Dict[str, int] = None, types: ty.Dict[str, int] = None, days: ty.Dict[str, int] = None, - time: ty.Dict[str, str] = None + time: ty.Dict[str, str] = None, + instructors: ty.Dict[str, int] = None, + conflict_sections: ty.Set[owl.model.SectionQuarterView] = None ): self.status = status self.types = types self.days = days self.time = time + self.instructors = instructors + self.conflict_sections = conflict_sections def check(self, section: owl.model.SectionQuarterView) -> bool: """ @@ -85,5 +89,21 @@ def time_filter() -> bool: return False return True + def instructor_filter() -> bool: + if not self.instructors: + return True + return section.instructor_name in \ + {k for k, v in self.instructors.items() if v} + + def conflict_filter() -> bool: + if not self.conflict_sections: + return True + for conflict_section in self.conflict_sections: + if section.conflicts(conflict_section): + return False + return True + return all(( - status_filter(), type_filter(), day_filter(), time_filter())) + status_filter(), type_filter(), day_filter(), time_filter(), + instructor_filter(), conflict_filter() + )) diff --git a/owl/model.py b/owl/model.py index 8ee70fa..0886995 100644 --- a/owl/model.py +++ b/owl/model.py @@ -6,6 +6,7 @@ import string import weakref import re +import itertools as itr import tinydb import maya @@ -605,6 +606,22 @@ def _unpack_entry_days(self, entry: SECTION_ENTRY_T) -> ty.Set[str]: days = set(matches.groups()) - {None} return days + def conflicts(self, other: 'SectionQuarterView') -> bool: + """ + Checks whether this section has any overlapping class meetings + with another passed section. + :param other: SectionQuarterView + :return: bool + """ + # There may be a more computationally efficient way to do this, + # if this becomes a highly used function, optimization should + # be looked into. + for own_duration, other_duration in \ + itr.product(self.durations, other.durations): + if own_duration.intersects(other_duration): + return False + return True + @property def start_date(self) -> maya.MayaDT: """ @@ -777,6 +794,16 @@ def __init__( self.room = room self.interval = maya.MayaInterval(start, end) + def intersects(self, other: 'ClassDuration') -> bool: + """ + Checks whether the ClassDuration overlaps with another passed + ClassDuration. + :param other: ClassDuration + :return: bool + """ + return self.day == other.day and \ + self.interval.intersects(other.interval) + @property def start(self) -> maya.MayaDT: """ diff --git a/owl/schema.py b/owl/schema.py index d656f00..79928bd 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -6,7 +6,8 @@ 'open': {'type': 'number'}, 'waitlist': {'type': 'number'}, 'full': {'type': 'number'}, - } + }, + "additionalProperties": False }, 'type_filter': { 'type': 'object', @@ -14,7 +15,8 @@ 'standard': {'type': 'number'}, 'online': {'type': 'number'}, 'hybrid': {'type': 'number'} - } + }, + "additionalProperties": False }, 'day_filter': { 'type': 'object', @@ -26,13 +28,40 @@ 'F': {'type': 'number'}, 'S': {'type': 'number'}, 'U': {'type': 'number'}, - } + }, + "additionalProperties": False }, 'time_filter': { 'type': 'object', 'properties': { 'start': {'type': 'string'}, 'end': {'type': 'string'}, + }, + "additionalProperties": False + }, + 'instructor_filter': { + 'type': 'object', + 'properties': {}, + "additionalProperties": { + "type": "object", + "required": [ + "age", + "gender" + ], + "properties": { + "age": { + "type": "string" + }, + "gender": { + "type": "string" + } + } + } + }, + 'conflict_filter': { + 'type': 'array', + 'items': { + '$ref': '#/definitions/get_section' } }, 'filter': { @@ -42,7 +71,21 @@ 'type': {'$ref': '#/definitions/type_filter'}, 'days': {'$ref': '#/definitions/day_filter'}, 'time': {'$ref': '#/definitions/time_filter'}, + 'instructor': {'$ref': '#/definitions/instructor_filter'}, + 'conflict_sections': {'$ref': '#/definitions/conflict_filter'} + }, + "additionalProperties": False + }, + 'get_section': { + 'type': 'object', + 'properties': { + 'quarter': {'type': 'string'}, + 'department': {'type': 'string'}, + 'course': {'type': 'string'}, + 'section': {'type': 'string'}, + 'filter': {'$ref': '#/definitions/filter'}, }, + 'required': ['department', 'course', 'section'] }, 'get_one': { 'type': 'object', diff --git a/server.py b/server.py index 8297c25..0512fa0 100644 --- a/server.py +++ b/server.py @@ -35,10 +35,13 @@ def add_cors_headers(response): CAMPUS_LIST = {'fh', 'da'} # fields +SCHOOL_KEY = 'school' DEPARTMENT_KEY = 'dept' COURSE_KEY = 'course' QUARTER_KEY = 'quarter' +SECTION_KEY = 'section' FILTER_KEY = 'filters' +FILTER_CONFLICTS_KEY = 'conflict_sections' data_model = owl.model.DataModel(settings.DB_DIR) accessor = owl.access.ModelAccessor(data_model) @@ -225,8 +228,26 @@ def _get_section_filter(args): :param args: request args :return: SectionFilter or None """ - return owl.filter.SectionFilter(**args[FILTER_KEY]) \ - if args[FILTER_KEY] else None + if not args[FILTER_KEY]: + return None + filter_kwargs = args.copy() + try: + conflict_sections = filter_kwargs[FILTER_CONFLICTS_KEY] + except KeyError: + pass + else: + # replace filter conflicts identifiers with set of + # section views. + filter_kwargs[FILTER_CONFLICTS_KEY] = { + accessor.get_section( + school=section_identifiers[SCHOOL_KEY], + quarter=section_identifiers[QUARTER_KEY], + department=section_identifiers[DEPARTMENT_KEY], + course=section_identifiers[COURSE_KEY], + section=section_identifiers[SECTION_KEY] + ) for section_identifiers in conflict_sections + } + return owl.filter.SectionFilter(**filter_kwargs) if __name__ == '__main__': From d1917c82502d6e5e911726457ff13a2580ccf237 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 12:10:24 -0700 Subject: [PATCH 48/60] Added input tests and fixed schema + removed placeholders. --- owl/schema.py | 29 ++----- tests/model/test_input.py | 174 +++++++++++++++++++++++++++++++++++++- 2 files changed, 181 insertions(+), 22 deletions(-) diff --git a/owl/schema.py b/owl/schema.py index 79928bd..59401a6 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -43,19 +43,7 @@ 'type': 'object', 'properties': {}, "additionalProperties": { - "type": "object", - "required": [ - "age", - "gender" - ], - "properties": { - "age": { - "type": "string" - }, - "gender": { - "type": "string" - } - } + "type": 'number' } }, 'conflict_filter': { @@ -83,7 +71,6 @@ 'department': {'type': 'string'}, 'course': {'type': 'string'}, 'section': {'type': 'string'}, - 'filter': {'$ref': '#/definitions/filter'}, }, 'required': ['department', 'course', 'section'] }, @@ -95,7 +82,8 @@ 'course': {'type': 'string'}, 'filter': {'$ref': '#/definitions/filter'}, }, - 'required': ['department'] + 'required': ['department'], + "additionalProperties": False }, 'get_many': { 'type': 'object', @@ -103,15 +91,13 @@ 'courses': { 'type': 'array', 'items': { - 'type': 'object', - 'additionalProperties': { - '$ref': '#/definitions/get_one' - } + '$ref': '#/definitions/get_one' } }, 'filter': {'$ref': '#/definitions/filter'}, }, - 'required': ['courses'] + 'required': ['courses'], + "additionalProperties": False }, 'get_list': { 'type': 'object', @@ -120,7 +106,8 @@ 'department': {'type': 'string'}, 'course': {'type': 'string'}, }, - 'required': ['department'] + 'required': ['department'], + "additionalProperties": False } } diff --git a/tests/model/test_input.py b/tests/model/test_input.py index 949f0fe..2cab9c2 100644 --- a/tests/model/test_input.py +++ b/tests/model/test_input.py @@ -14,7 +14,17 @@ def test_get_one_allows_proper_input(self): inputs.validate() self.assertFalse(inputs.errors) - @SkipTest + def test_get_one_raises_issues_when_extra_arguments_are_received(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'blah': 5, + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertTrue(inputs.errors) + def test_get_one_disallows_input_without_department(self): raw = NotRequest({ 'quarter': '000011', @@ -24,6 +34,168 @@ def test_get_one_disallows_input_without_department(self): inputs.validate() self.assertEqual(1, len(inputs.errors)) + def test_filter_can_accept_status_argument(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'filter': { + 'status': {'open': 1, 'waitlist': 1}, + } + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_filter_can_accept_type_argument(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'filter': { + 'type': {'hybrid': 1, 'online': 1}, + } + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_filter_can_accept_days_argument(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'filter': { + 'days': {'M': 1, 'T': 1, 'Th': 1}, + } + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_filter_can_accept_instructor_argument(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'filter': { + 'instructor': {'PlaceholderName': 1}, + } + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_filter_can_accept_all_arguments(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'filter': { + 'status': {'open': 1, 'waitlist': 1}, + 'type': {'hybrid': 1, 'online': 1}, + 'days': {'M': 1, 'T': 1, 'Th': 1}, + 'time': {'start': '8:30 AM', 'end': '4:30 PM'}, + 'instructor': {'PlaceholderName': 1}, + 'conflict_sections': [ + { + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'section': '123456', + }, + { + 'quarter': '000011', + 'department': 'CS', + 'course': '2A', + 'section': '654321', + } + ] + } + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_filter_does_not_accept_extra_arguments(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + 'filter': { + 'foo': {'Placeholder': 1}, + } + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertTrue(inputs.errors) + + def test_get_many_allows_proper_input(self): + request = NotRequest({ + 'courses': [ + { + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + }, + { + 'quarter': '000012', + 'department': 'CS', + }, + { + 'quarter': '000011', + 'department': 'MATH', + } + ] + }) + inputs = owl.input.GetManyInput(request) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_get_many_raises_issues_when_extra_arguments_are_received(self): + request = NotRequest({ + 'courses': [ + { + 'quarter': '000011', + 'department': 'CS', + 'course': '1A', + }, + { + 'quarter': '000012', + 'department': 'CS', + }, + { + 'quarter': '000011', + 'department': 'MATH', + } + ], + 'unneeded_arg': 2 + }) + inputs = owl.input.GetManyInput(request) + inputs.validate() + self.assertTrue(inputs.errors) + + def test_get_list_accepts_expected_arguments(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'MATH', + 'course': '1A', + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_get_list_does_not_accept_unexpected_arguments(self): + raw = NotRequest({ + 'quarter': '000011', + 'department': 'MATH', + 'course': '1A', + 'unexpected': 4 + }) + inputs = owl.input.GetOneInput(raw) + inputs.validate() + self.assertTrue(inputs.errors) + class NotRequest: def __init__(self, args): From dcfea37e06e5196113432cec25e3faaddf64309e Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 12:45:06 -0700 Subject: [PATCH 49/60] completed unfinished get_urls method + supporting methods. --- owl/input.py | 4 ++++ owl/model.py | 12 +++++++++++- owl/schema.py | 7 +++++++ server.py | 10 +++++++--- tests/model/test_access.py | 5 +++++ tests/model/test_input.py | 17 +++++++++++++++++ 6 files changed, 51 insertions(+), 4 deletions(-) diff --git a/owl/input.py b/owl/input.py index c9cb2b0..afab045 100644 --- a/owl/input.py +++ b/owl/input.py @@ -53,3 +53,7 @@ class GetManyInput(Inputs): class GetListInput(Inputs): json = [JsonSchema(schema=get_definition('get_list'))] + + +class GetUrlsInput(Inputs): + json = [JsonSchema(schema=get_definition('get_urls'))] diff --git a/owl/model.py b/owl/model.py index 8ee70fa..8505cab 100644 --- a/owl/model.py +++ b/owl/model.py @@ -357,7 +357,7 @@ def __init__(self, quarter: 'QuarterView'): def __getitem__(self, dept_name: str) -> 'DepartmentQuarterView': # screen department names that might otherwise access # internal tables, defaults, etc. - if any(char not in string.ascii_letters for char in dept_name): + if not self._valid_name(dept_name): raise ValueError( f'Invalid department name passed: {dept_name}') if dept_name not in self.db.tables(): @@ -368,6 +368,16 @@ def __getitem__(self, dept_name: str) -> 'DepartmentQuarterView': dept_data: DEPT_DATA_T = self.db.table(dept_name).all()[0] return DepartmentQuarterView(self.quarter, dept_name, dept_data) + def __iter__(self) -> ty.Iterable['DepartmentQuarterView']: + for dept_name in self.db.tables(): + if self._valid_name(dept_name): + data: DEPT_DATA_T = self.db.table(dept_name).all()[0] + yield DepartmentQuarterView(self.quarter, dept_name, data) + + @staticmethod + def _valid_name(dept_name: str) -> bool: + return all(char in string.ascii_letters for char in dept_name) + @property def db(self) -> tinydb.TinyDB: return self.quarter.db diff --git a/owl/schema.py b/owl/schema.py index d656f00..401a076 100644 --- a/owl/schema.py +++ b/owl/schema.py @@ -78,6 +78,13 @@ 'course': {'type': 'string'}, }, 'required': ['department'] + }, + 'get_urls': { + 'type': 'object', + 'properties': { + 'quarter': {'type': 'string'} + }, + 'additionalProperties': False } } diff --git a/server.py b/server.py index 8297c25..79c8836 100644 --- a/server.py +++ b/server.py @@ -202,10 +202,11 @@ def api_list(campus): @application.route('//urls', methods=['GET']) +@basic_checks(input_type=owl.input.GetUrlsInput) def api_list_url(campus): """ - `/urls` with [GET] returns a tree of all departments, their - courses, and the courses' endpoints to hit. + `/urls` with [GET] returns a tree of all departments in a quarter, + their courses, and the courses' endpoints to hit. :param campus: (str) The campus to retrieve data from @@ -214,7 +215,10 @@ def api_list_url(campus): if campus not in CAMPUS_LIST: return 'Error! Could not find campus in database', 404 - data = accessor.get_urls(request.get()) + data = accessor.get_urls( + school=campus, + quarter=request.args.get(QUARTER_KEY, owl.access.LATEST) + ) return jsonify(data), 200 diff --git a/tests/model/test_access.py b/tests/model/test_access.py index 4401f6c..4b31145 100644 --- a/tests/model/test_access.py +++ b/tests/model/test_access.py @@ -39,6 +39,11 @@ def test_get_one_dept_and_course(self): result ) + def test_get_urls_gets_all_departments(self): + accessor = get_accessor('model_test_dir_a') + result = accessor.get_urls(school='fh', quarter='000011') + self.assertEqual(74, len(result)) + def get_accessor(dir_name: str): return ModelAccessor(DataModel(os.path.join(TEST_RESOURCES_DIR, dir_name))) diff --git a/tests/model/test_input.py b/tests/model/test_input.py index 949f0fe..42dbdc2 100644 --- a/tests/model/test_input.py +++ b/tests/model/test_input.py @@ -24,6 +24,23 @@ def test_get_one_disallows_input_without_department(self): inputs.validate() self.assertEqual(1, len(inputs.errors)) + def test_get_urls_accepts_valid_input(self): + raw = NotRequest({ + 'quarter': '000011', + }) + inputs = owl.input.GetUrlsInput(raw) + inputs.validate() + self.assertFalse(inputs.errors) + + def test_get_urls_does_not_accept_extra_arguments(self): + raw = NotRequest({ + 'quarter': '000011', + 'Unneeded': 'CS', + }) + inputs = owl.input.GetUrlsInput(raw) + inputs.validate() + self.assertTrue(inputs.errors) + class NotRequest: def __init__(self, args): From aa373d1e5447c973f29831636a1d934329137ab8 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 15:30:26 -0700 Subject: [PATCH 50/60] Refactored model tests to use data in temporary directories. --- tests/model/test_model.py | 441 +++++++++++++++++++++----------------- 1 file changed, 240 insertions(+), 201 deletions(-) diff --git a/tests/model/test_model.py b/tests/model/test_model.py index 51097eb..d0d7329 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -1,5 +1,7 @@ import os +import tempfile +from distutils.dir_util import copy_tree from unittest import TestCase import maya @@ -12,264 +14,301 @@ class TestDataModel(TestCase): def test_model_finds_all_tables(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - self.assertEqual(3, len([quarter for quarter in data.quarters])) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + self.assertEqual(3, len([quarter for quarter in data.quarters])) def test_model_contains_correct_schools(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - self.assertEqual(2, len(data.schools)) - self.assertIn('FH', data.schools) - self.assertIn('DA', data.schools) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + self.assertEqual(2, len(data.schools)) + self.assertIn('FH', data.schools) + self.assertIn('DA', data.schools) def test_department_view_can_be_retrieved_from_quarter(self): # This test doesn't help much, but at least confirms that no # errors are thrown. - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - self.assertEqual('ACTG', dept.name) - self.assertIsInstance(dept, DepartmentQuarterView) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + self.assertEqual('ACTG', dept.name) + self.assertIsInstance(dept, DepartmentQuarterView) def test_courses_are_accessible_from_department(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - self.assertIsInstance(course, CourseQuarterView) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + self.assertIsInstance(course, CourseQuarterView) def test_sections_are_accessible_from_course(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40065'] - self.assertIsInstance(section, SectionQuarterView) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertIsInstance(section, SectionQuarterView) def test_section_course_id_returns_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40065'] - self.assertEqual('ACTG F001A02Y', section.course_id) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual('ACTG F001A02Y', section.course_id) def test_section_crn_returns_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40065'] - self.assertEqual('40065', section.crn) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual('40065', section.crn) def test_section_description_returns_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40065'] - self.assertEqual('FINANCIAL ACCOUNTING I', section.description) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual('FINANCIAL ACCOUNTING I', section.description) def test_section_status_returns_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['41130'] - self.assertEqual(OPEN, section.status) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['41130'] + self.assertEqual(OPEN, section.status) def test_section_days_returns_correctly_in_semi_online_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40065'] - self.assertEqual({'T', 'Th'}, section.days) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual({'T', 'Th'}, section.days) def test_section_days_property_raises_exception_in_online_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40018'] - self.assertEqual(set(), section.days) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40018'] + self.assertEqual(set(), section.days) def test_section_days_returns_correctly_in_offline_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ENGL'] - course = dept.courses['1A'] - section = course.sections['40140'] - self.assertEqual({'M', 'W'}, section.days) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ENGL'] + course = dept.courses['1A'] + section = course.sections['40140'] + self.assertEqual({'M', 'W'}, section.days) def test_section_days_returns_correctly_in_single_day_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['76'] - section = course.sections['41440'] - self.assertEqual({'W'}, section.days) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['76'] + section = course.sections['41440'] + self.assertEqual({'W'}, section.days) def test_section_days_returns_correctly_in_class_with_labs_a(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['PHYS'] - course = dept.courses['2B'] - section = course.sections['40582'] - self.assertEqual({'T', 'Th'}, section.days) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['2B'] + section = course.sections['40582'] + self.assertEqual({'T', 'Th'}, section.days) def test_section_days_returns_correctly_in_class_with_labs_b(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['PHYS'] - course = dept.courses['4D'] - section = course.sections['40208'] - self.assertEqual({'T', 'Th', 'F'}, section.days) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + self.assertEqual({'T', 'Th', 'F'}, section.days) def test_section_durations_return_correct_maya_date_times(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['PHYS'] - course = dept.courses['4D'] - section = course.sections['40208'] - durations = section.durations - self.assertEqual(4, len(durations)) - self.assertIsInstance(durations[0], ClassDuration) - self.assertIsInstance(durations[1], ClassDuration) - self.assertIsInstance(durations[2], ClassDuration) - self.assertIsInstance(durations[3], ClassDuration) - - # Check that each duration has expected times - for duration in durations: - if duration.day == 'Th': - self.assertEqual(maya.when('10:00AM'), duration.start) - self.assertEqual(maya.when('11:50AM'), duration.end) - elif duration.day == 'F': - self.assertEqual(maya.when('11:00 AM'), duration.start) - self.assertEqual(maya.when('11:50AM'), duration.end) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + durations = section.durations + self.assertEqual(4, len(durations)) + self.assertIsInstance(durations[0], ClassDuration) + self.assertIsInstance(durations[1], ClassDuration) + self.assertIsInstance(durations[2], ClassDuration) + self.assertIsInstance(durations[3], ClassDuration) + + # Check that each duration has expected times + for duration in durations: + if duration.day == 'Th': + self.assertEqual(maya.when('10:00AM'), duration.start) + self.assertEqual(maya.when('11:50AM'), duration.end) + elif duration.day == 'F': + self.assertEqual(maya.when('11:00 AM'), duration.start) + self.assertEqual(maya.when('11:50AM'), duration.end) def test_section_rooms_are_found_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['PHYS'] - course = dept.courses['4D'] - section = course.sections['40208'] - durations = section.durations - self.assertIsInstance(durations[0], ClassDuration) - self.assertIsInstance(durations[1], ClassDuration) - self.assertIsInstance(durations[2], ClassDuration) - self.assertIsInstance(durations[3], ClassDuration) - - # Check that each duration has expected times - for duration in durations: - if duration.day == 'Th': - self.assertEqual('4501', duration.room) - elif duration.day == 'F': - self.assertEqual('4501', duration.room) - elif duration.day == 'T' and \ - duration.start == maya.when('12:00PM'): - self.assertEqual('4718', duration.room) + + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + durations = section.durations + self.assertIsInstance(durations[0], ClassDuration) + self.assertIsInstance(durations[1], ClassDuration) + self.assertIsInstance(durations[2], ClassDuration) + self.assertIsInstance(durations[3], ClassDuration) + + # Check that each duration has expected times + for duration in durations: + if duration.day == 'Th': + self.assertEqual('4501', duration.room) + elif duration.day == 'F': + self.assertEqual('4501', duration.room) + elif duration.day == 'T' and \ + duration.start == maya.when('12:00PM'): + self.assertEqual('4718', duration.room) def test_section_rooms_are_found_correctly_for_multi_room_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['PHYS'] - course = dept.courses['4D'] - section = course.sections['40208'] - self.assertEqual({'4718', '4501'}, section.rooms) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['PHYS'] + course = dept.courses['4D'] + section = course.sections['40208'] + self.assertEqual({'4718', '4501'}, section.rooms) def test_section_rooms_are_found_correctly_for_single_room_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual({'3201'}, section.rooms) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual({'3201'}, section.rooms) def test_section_rooms_are_found_correctly_for_online_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1C'] - section = course.sections['40022'] - self.assertEqual(set(), section.rooms) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1C'] + section = course.sections['40022'] + self.assertEqual(set(), section.rooms) def test_section_end_returns_correct_maya_date(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ENGL'] - course = dept.courses['1A'] - section = course.sections['40140'] - end_date = section.end_date - self.assertIsInstance(end_date, maya.MayaDT) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ENGL'] + course = dept.courses['1A'] + section = course.sections['40140'] + end_date = section.end_date + self.assertIsInstance(end_date, maya.MayaDT) def test_section_type_is_determined_correctly_for_offline_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ENGL'] - course = dept.courses['1A'] - section = course.sections['40140'] - self.assertEqual(STANDARD_TYPE, section.section_type) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ENGL'] + course = dept.courses['1A'] + section = course.sections['40140'] + self.assertEqual(STANDARD_TYPE, section.section_type) def test_section_type_is_determined_correctly_for_online_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40018'] - self.assertEqual(ONLINE_TYPE, section.section_type) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40018'] + self.assertEqual(ONLINE_TYPE, section.section_type) def test_section_type_is_determined_correctly_for_hybrid_class(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1A'] - section = course.sections['40065'] - self.assertEqual(HYBRID_TYPE, section.section_type) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1A'] + section = course.sections['40065'] + self.assertEqual(HYBRID_TYPE, section.section_type) def test_section_campus_is_found_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual('FH', section.campus) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual('FH', section.campus) def test_section_units_are_found_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual(5, section.units) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(5, section.units) def test_section_instructor_name_returns_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual('Drake', section.instructor_name) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual('Drake', section.instructor_name) def test_section_seats_return_correctly_when_on_waitlist_status(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual(0, section.open_seats_available) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(0, section.open_seats_available) def test_section_wait_seats_return_correctly_when_on_waitlist_status(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual(7, section.waitlist_seats_available) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(7, section.waitlist_seats_available) def test_section_waitlist_capacity_returns_correctly(self): - data = DataModel(os.path.join(TEST_RESOURCES_DIR, 'model_test_dir_a')) - quarter = data.quarters['000011'] - dept = quarter.departments['ACTG'] - course = dept.courses['1B'] - section = course.sections['40067'] - self.assertEqual(15, section.waitlist_capacity) + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + dept = quarter.departments['ACTG'] + course = dept.courses['1B'] + section = course.sections['40067'] + self.assertEqual(15, section.waitlist_capacity) + + +def get_test_data_dir(dir_name: str) -> tempfile.TemporaryDirectory: + temp_dir = tempfile.TemporaryDirectory() + copy_tree(os.path.join(TEST_RESOURCES_DIR, dir_name), temp_dir.name) + return temp_dir From 3748667d076c46cde53247d7d42ee20e7a871e46 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 15:40:43 -0700 Subject: [PATCH 51/60] Refactored Accessor tests to use temporary data directories. --- tests/model/test_access.py | 51 ++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/model/test_access.py b/tests/model/test_access.py index 4b31145..dbc7b32 100644 --- a/tests/model/test_access.py +++ b/tests/model/test_access.py @@ -1,8 +1,7 @@ from unittest import TestCase -import os +from .test_model import get_test_data_dir -from settings import TEST_RESOURCES_DIR from owl.model import DataModel from owl.access import ModelAccessor @@ -16,34 +15,38 @@ class TestAccessor(TestCase): def test_get_one_dept(self): - accessor = get_accessor('model_test_dir_a') - result = accessor.get_one('fh', department='CS') - self.assertEqual( - test_data.test_get_one_dept_data[0], - result - ) + with get_test_data_dir('model_test_dir_a') as data_dir: + accessor = get_accessor(data_dir) + result = accessor.get_one('fh', department='CS') + self.assertEqual( + test_data.test_get_one_dept_data[0], + result + ) def test_get_one_dept_returns_n_courses(self): - accessor = get_accessor('model_test_dir_a') - result = accessor.get_one(school='fh', department='CS') - self.assertEqual( - len(test_data.test_get_one_dept_data[0].keys()), - len(result.keys()) - ) + with get_test_data_dir('model_test_dir_a') as data_dir: + accessor = get_accessor(data_dir) + result = accessor.get_one(school='fh', department='CS') + self.assertEqual( + len(test_data.test_get_one_dept_data[0].keys()), + len(result.keys()) + ) def test_get_one_dept_and_course(self): - accessor = get_accessor('model_test_dir_a') - result = accessor.get_one(school='fh', department='CS', course='2A') - self.assertEqual( - test_data.test_get_one_dept_and_course_data, - result - ) + with get_test_data_dir('model_test_dir_a') as data_dir: + accessor = get_accessor(data_dir) + result = accessor.get_one(school='fh', department='CS', course='2A') + self.assertEqual( + test_data.test_get_one_dept_and_course_data, + result + ) def test_get_urls_gets_all_departments(self): - accessor = get_accessor('model_test_dir_a') - result = accessor.get_urls(school='fh', quarter='000011') - self.assertEqual(74, len(result)) + with get_test_data_dir('model_test_dir_a') as data_dir: + accessor = get_accessor(data_dir) + result = accessor.get_urls(school='fh', quarter='000011') + self.assertEqual(74, len(result)) def get_accessor(dir_name: str): - return ModelAccessor(DataModel(os.path.join(TEST_RESOURCES_DIR, dir_name))) + return ModelAccessor(DataModel(dir_name)) From 8ba767dd11d709a0c7f2547e8cdb10ad63e1e27f Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 21:01:48 -0700 Subject: [PATCH 52/60] Began implementing quarter method caching. --- owl/access.py | 9 +----- owl/model.py | 66 ++++++++++++++++++++++++++++++++++++++ tests/model/test_access.py | 3 +- tests/model/test_model.py | 20 ++++++++++++ 4 files changed, 89 insertions(+), 9 deletions(-) diff --git a/owl/access.py b/owl/access.py index 8e72020..657ec9d 100644 --- a/owl/access.py +++ b/owl/access.py @@ -211,11 +211,4 @@ def get_urls( :return: Dict[str, Dict[str, str]] """ quarter_view = self.get_quarter(school, quarter) - return { - department_view.name: { - course_view.name: { - 'course': course_view.name, - 'dept': course_view.department.name - } for course_view in department_view.courses - } for department_view in quarter_view.departments - } + return quarter_view.urls diff --git a/owl/model.py b/owl/model.py index 3e4ca42..46c3666 100644 --- a/owl/model.py +++ b/owl/model.py @@ -13,6 +13,7 @@ DB_EXT = '.json' DB_SUFFIX = '_database' +CACHE_TABLE_NAME = 'cache_' FH = 'FH' DA = 'DA' SCHOOL_NAMES_BY_CODE = { @@ -83,6 +84,32 @@ class DataError(Exception): """ +def quarter_cache(f): + def wrapper(self: 'QuarterView', *args, **kwargs): + cache_data = self.db.table(CACHE_TABLE_NAME).get(doc_id=1) or {} + arg_hash = str(_hash_args(args, kwargs)) + # Get method cache dictionary + try: + method_cache = cache_data[f.__name__] + except KeyError: + method_cache = cache_data[f.__name__] = dict() + # Get result cache dictionary + try: + cached_result = method_cache[arg_hash] + except KeyError: + cached_result = method_cache[arg_hash] = \ + f(self, *args, **kwargs) + try: + self.db.table(CACHE_TABLE_NAME).update(cache_data, doc_ids=[1]) + except KeyError: + self.db.table(CACHE_TABLE_NAME).insert(cache_data) + # Update stored table + return cached_result + + wrapper.__name__ = f.__name__ + '_cache_wrapper' + return wrapper + + class DataModel: def __init__(self, db_dir: str) -> None: """ @@ -122,6 +149,7 @@ def find_quarters() -> ty.Dict[str, 'QuarterView']: quarters[name] = QuarterView.get_quarter(self, name) return quarters + self.clear_cache() # Clears caches, since data may now be changed. self.quarters = find_quarters() self.schools = {quarter.school_name: quarter.school for quarter in self.quarters.values()} @@ -142,6 +170,10 @@ def register_quarter(self, quarter: 'QuarterView'): f'an existing quarter in {self}.') self.quarter_instances[quarter.name] = quarter + def clear_cache(self): + for quarter in self.quarters.values(): + quarter.clear_cache() + def __repr__(self) -> str: return f'DataModel[{self.db_dir}]' @@ -264,6 +296,34 @@ def get_quarter(cls, model: DataModel, name: str) -> 'QuarterView': except KeyError: return cls(model, name) + def clear_cache(self): + self.db.purge_table(CACHE_TABLE_NAME) + + @property + @quarter_cache + def urls(self): + """ + Gets urls dictionary for quarter. + Returns in format: + { + CS: { + 1A: { + 'course': '1A' + 'dept': 'CS' + } + } + } + :return: Dict[str, Dict[str, Dict[str, str]]] + """ + return { + department_view.name: { + course_view.name: { + 'course': course_view.name, + 'dept': course_view.department.name + } for course_view in department_view.courses + } for department_view in self.departments + } + @property def db(self) -> tinydb.TinyDB: """ @@ -838,3 +898,9 @@ class InstructorView: def __init__(self, model: DataModel, name: str) -> None: self.model: DataModel = model self.name: str = name + + +def _hash_args(args, kwargs): + arg_hash: int = hash(args) + kwargs_hash: int = hash(repr(sorted(kwargs.items()))) + return hash((arg_hash, kwargs_hash)) diff --git a/tests/model/test_access.py b/tests/model/test_access.py index dbc7b32..2b1c870 100644 --- a/tests/model/test_access.py +++ b/tests/model/test_access.py @@ -35,7 +35,8 @@ def test_get_one_dept_returns_n_courses(self): def test_get_one_dept_and_course(self): with get_test_data_dir('model_test_dir_a') as data_dir: accessor = get_accessor(data_dir) - result = accessor.get_one(school='fh', department='CS', course='2A') + result = accessor.get_one( + school='fh', department='CS', course='2A') self.assertEqual( test_data.test_get_one_dept_and_course_data, result diff --git a/tests/model/test_model.py b/tests/model/test_model.py index d0d7329..6ff53a7 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -1,5 +1,6 @@ import os import tempfile +import time from distutils.dir_util import copy_tree from unittest import TestCase @@ -307,6 +308,25 @@ def test_section_waitlist_capacity_returns_correctly(self): section = course.sections['40067'] self.assertEqual(15, section.waitlist_capacity) + def test_quarter_cache_increases_url_access_speed_after_first_access(self): + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + + def time_urls(): + t0 = time.time() + urls = quarter.urls + tf = time.time() + return urls, tf - t0 + + urls1, elapsed1 = time_urls() + urls2, elapsed2 = time_urls() + urls3, elapsed3 = time_urls() + self.assertLess(elapsed2, elapsed1 / 4) + self.assertLess(elapsed3, elapsed1 / 4) + self.assertEqual(urls1, urls2) + self.assertEqual(urls1, urls3) + def get_test_data_dir(dir_name: str) -> tempfile.TemporaryDirectory: temp_dir = tempfile.TemporaryDirectory() From 9de217b37ac5292e602fd257b80ec38081957fe7 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 21:12:33 -0700 Subject: [PATCH 53/60] Added documentation + minor fixes. --- owl/access.py | 21 +++++++++++++++++++-- owl/model.py | 11 +++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/owl/access.py b/owl/access.py index 657ec9d..6606ccd 100644 --- a/owl/access.py +++ b/owl/access.py @@ -177,6 +177,14 @@ def get_course( course: str, quarter: str = LATEST, ) -> owl.model.CourseQuarterView: + """ + Gets view of data for a specific course in a specific quarter. + :param school: str (ex: 'fh') + :param department: str (ex: 'CS') + :param course: str (ex: '1A') + :param quarter: str (ex: '201812') + :return: CourseQuarterView + """ department_view = self.get_department(school, department, quarter) try: course_view = department_view.courses[course] @@ -193,13 +201,22 @@ def get_section( section: str, quarter: str = LATEST, ) -> owl.model.SectionQuarterView: + """ + Gets view of data for a specific section in a specific quarter. + :param school: str (ex: 'fh') + :param department: str (ex: 'CS') + :param course: str (ex: '1A') + :param quarter: str (ex: '201812') + :param section: str (ex: '12345') + :return: SectionQuarterView + """ course_view = self.get_course(school, department, course, quarter) try: - course_view = course_view.courses[course] + section_view = course_view.sections[section] except KeyError as e: raise AccessException( f'No section in {course} with name: {section}.') from e - return course_view + return section_view def get_urls( self, school: str, quarter: str = LATEST diff --git a/owl/model.py b/owl/model.py index 46c3666..5dceec9 100644 --- a/owl/model.py +++ b/owl/model.py @@ -683,7 +683,7 @@ def conflicts(self, other: 'SectionQuarterView') -> bool: :param other: SectionQuarterView :return: bool """ - # There may be a more computationally efficient way to do this, + # There may be a more computationally efficient way to do this; # if this becomes a highly used function, optimization should # be looked into. for own_duration, other_duration in \ @@ -900,7 +900,14 @@ def __init__(self, model: DataModel, name: str) -> None: self.name: str = name -def _hash_args(args, kwargs): +def _hash_args(args: ty.Tuple[ty.Any], kwargs: ty.Dict[str, ty.Any]) -> int: + """ + Produces a hash int from passed tuple of args and dictionary + of kwargs. + :param args: Tuple[Any] + :param kwargs: Dict[str, Any] + :return: int + """ arg_hash: int = hash(args) kwargs_hash: int = hash(repr(sorted(kwargs.items()))) return hash((arg_hash, kwargs_hash)) From 6ecc9c6ea4f9b1f796f87ccf86dffa0daaba8aaf Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 21:24:18 -0700 Subject: [PATCH 54/60] Renamed NotRequest to PseudoRequest --- tests/model/test_input.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/model/test_input.py b/tests/model/test_input.py index 644b1fe..045ce8f 100644 --- a/tests/model/test_input.py +++ b/tests/model/test_input.py @@ -5,7 +5,7 @@ class TestInput(TestCase): def test_get_one_allows_proper_input(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -15,7 +15,7 @@ def test_get_one_allows_proper_input(self): self.assertFalse(inputs.errors) def test_get_one_raises_issues_when_extra_arguments_are_received(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -26,7 +26,7 @@ def test_get_one_raises_issues_when_extra_arguments_are_received(self): self.assertTrue(inputs.errors) def test_get_one_disallows_input_without_department(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'course': '1A', }) @@ -35,7 +35,7 @@ def test_get_one_disallows_input_without_department(self): self.assertEqual(1, len(inputs.errors)) def test_filter_can_accept_status_argument(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -48,7 +48,7 @@ def test_filter_can_accept_status_argument(self): self.assertFalse(inputs.errors) def test_filter_can_accept_type_argument(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -61,7 +61,7 @@ def test_filter_can_accept_type_argument(self): self.assertFalse(inputs.errors) def test_filter_can_accept_days_argument(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -74,7 +74,7 @@ def test_filter_can_accept_days_argument(self): self.assertFalse(inputs.errors) def test_filter_can_accept_instructor_argument(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -87,7 +87,7 @@ def test_filter_can_accept_instructor_argument(self): self.assertFalse(inputs.errors) def test_filter_can_accept_all_arguments(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -118,7 +118,7 @@ def test_filter_can_accept_all_arguments(self): self.assertFalse(inputs.errors) def test_filter_does_not_accept_extra_arguments(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'CS', 'course': '1A', @@ -131,7 +131,7 @@ def test_filter_does_not_accept_extra_arguments(self): self.assertTrue(inputs.errors) def test_get_many_allows_proper_input(self): - request = NotRequest({ + request = PseudoRequest({ 'courses': [ { 'quarter': '000011', @@ -153,7 +153,7 @@ def test_get_many_allows_proper_input(self): self.assertFalse(inputs.errors) def test_get_many_raises_issues_when_extra_arguments_are_received(self): - request = NotRequest({ + request = PseudoRequest({ 'courses': [ { 'quarter': '000011', @@ -176,7 +176,7 @@ def test_get_many_raises_issues_when_extra_arguments_are_received(self): self.assertTrue(inputs.errors) def test_get_list_accepts_expected_arguments(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'MATH', 'course': '1A', @@ -186,7 +186,7 @@ def test_get_list_accepts_expected_arguments(self): self.assertFalse(inputs.errors) def test_get_list_does_not_accept_unexpected_arguments(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'department': 'MATH', 'course': '1A', @@ -197,7 +197,7 @@ def test_get_list_does_not_accept_unexpected_arguments(self): self.assertTrue(inputs.errors) def test_get_urls_accepts_valid_input(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', }) inputs = owl.input.GetUrlsInput(raw) @@ -205,7 +205,7 @@ def test_get_urls_accepts_valid_input(self): self.assertFalse(inputs.errors) def test_get_urls_does_not_accept_extra_arguments(self): - raw = NotRequest({ + raw = PseudoRequest({ 'quarter': '000011', 'Unneeded': 'CS', }) @@ -214,6 +214,6 @@ def test_get_urls_does_not_accept_extra_arguments(self): self.assertTrue(inputs.errors) -class NotRequest: +class PseudoRequest: def __init__(self, args): self.json = args From f3225034925171f48033ae162dcdb288b17f684a Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Wed, 13 Jun 2018 21:45:27 -0700 Subject: [PATCH 55/60] Added intersection tests for ClassDuration --- owl/model.py | 2 +- tests/model/test_input.py | 2 +- tests/model/test_model.py | 24 +++++++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/owl/model.py b/owl/model.py index 5dceec9..405f2ec 100644 --- a/owl/model.py +++ b/owl/model.py @@ -900,7 +900,7 @@ def __init__(self, model: DataModel, name: str) -> None: self.name: str = name -def _hash_args(args: ty.Tuple[ty.Any], kwargs: ty.Dict[str, ty.Any]) -> int: +def _hash_args(args: ty.Tuple, kwargs: ty.Dict[str, ty.Any]) -> int: """ Produces a hash int from passed tuple of args and dictionary of kwargs. diff --git a/tests/model/test_input.py b/tests/model/test_input.py index 045ce8f..19d29f9 100644 --- a/tests/model/test_input.py +++ b/tests/model/test_input.py @@ -1,4 +1,4 @@ -from unittest import TestCase, SkipTest +from unittest import TestCase import owl.input diff --git a/tests/model/test_model.py b/tests/model/test_model.py index 6ff53a7..bf3b057 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -167,7 +167,6 @@ def test_section_durations_return_correct_maya_date_times(self): self.assertEqual(maya.when('11:50AM'), duration.end) def test_section_rooms_are_found_correctly(self): - with get_test_data_dir('model_test_dir_a') as data_dir: data = DataModel(data_dir) quarter = data.quarters['000011'] @@ -328,6 +327,29 @@ def time_urls(): self.assertEqual(urls1, urls3) +class TestCourseDuration(TestCase): + def test_durations_intersect_returns_true_with_intersection(self): + a = ClassDuration( + 'M', '1234', maya.when('8:30 AM'), maya.when('10:30 AM')) + b = ClassDuration( + 'M', '5678', maya.when('9:00 AM'), maya.when('11:00 AM')) + self.assertTrue(a.intersects(b)) + + def test_durations_intersect_returns_false_with_differing_days(self): + a = ClassDuration( + 'M', '1234', maya.when('8:30 AM'), maya.when('10:30 AM')) + b = ClassDuration( + 'T', '5678', maya.when('9:00 AM'), maya.when('11:00 AM')) + self.assertFalse(a.intersects(b)) + + def test_durations_intersect_returns_false_with_differing_times(self): + a = ClassDuration( + 'M', '1234', maya.when('8:30 AM'), maya.when('10:30 AM')) + b = ClassDuration( + 'M', '5678', maya.when('2:00 PM'), maya.when('4:00 PM')) + self.assertFalse(a.intersects(b)) + + def get_test_data_dir(dir_name: str) -> tempfile.TemporaryDirectory: temp_dir = tempfile.TemporaryDirectory() copy_tree(os.path.join(TEST_RESOURCES_DIR, dir_name), temp_dir.name) From 9e8a6c27c7d777ba940773538f0c3e24fc6e9b4b Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Thu, 14 Jun 2018 13:55:46 -0700 Subject: [PATCH 56/60] Added serialization module and quarter duration property. --- owl/model.py | 114 +++++++++++++++++++++++++++++++++----- owl/serial.py | 54 ++++++++++++++++++ tests/model/test_model.py | 14 ++++- 3 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 owl/serial.py diff --git a/owl/model.py b/owl/model.py index 405f2ec..810b220 100644 --- a/owl/model.py +++ b/owl/model.py @@ -7,10 +7,13 @@ import weakref import re import itertools as itr +import json import tinydb import maya +import owl.serial + DB_EXT = '.json' DB_SUFFIX = '_database' CACHE_TABLE_NAME = 'cache_' @@ -95,16 +98,17 @@ def wrapper(self: 'QuarterView', *args, **kwargs): method_cache = cache_data[f.__name__] = dict() # Get result cache dictionary try: - cached_result = method_cache[arg_hash] + result = json.loads(method_cache[arg_hash], + object_hook=owl.serial.hook) except KeyError: - cached_result = method_cache[arg_hash] = \ - f(self, *args, **kwargs) + result = f(self, *args, **kwargs) + method_cache[arg_hash] = json.dumps(result, cls=owl.serial.Encoder) try: self.db.table(CACHE_TABLE_NAME).update(cache_data, doc_ids=[1]) except KeyError: self.db.table(CACHE_TABLE_NAME).insert(cache_data) # Update stored table - return cached_result + return result wrapper.__name__ = f.__name__ + '_cache_wrapper' return wrapper @@ -388,6 +392,42 @@ def school_name(self) -> str: def school(self) -> SchoolView: return SchoolView(self.model, self.school_name) + # Since the vast majority of classes are of the primary duration, + # it is not necessary to check all sections + # Assuming 75% of classes are of primary duration, sampling + # 4*2*3 (24) sections has only a (1-0.75)^24 (3.5e-15) chance of + # returning an incorrect value, assuming random sampling. + # + # Reducing the samples taken reduces response time by ~80 times, + # And keeps response times comfortably under 1s. + DEPARTMENTS_SAMPLED = 4 + COURSE_SAMPLES = 2 + SECTION_SAMPLES = 3 + + @property + @quarter_cache + def primary_duration(self): + """ + Gets duration of quarter for full-length classes. + :return: CalendarDuration + """ + greatest_calendar_duration = None + greatest_delta: float = 0 + for i, department in enumerate(self.departments): + for j, course in enumerate(department.courses): + for k, section in enumerate(course.sections): + duration: 'CalendarDuration' = section.calendar_duration + if duration.interval.duration > greatest_delta: + greatest_delta = duration.interval.duration + greatest_calendar_duration = duration + if k == self.SECTION_SAMPLES: + break + if j == self.COURSE_SAMPLES: + break + if i == self.DEPARTMENTS_SAMPLED: + break + return greatest_calendar_duration + @property def path(self) -> str: """ @@ -570,7 +610,7 @@ class SectionQuarterView: # All these fields should be equal if multiple entries exist. EQUAL_FIELDS = ( COURSE_ID_KEY, CRN_KEY, DESCRIPTION_KEY, STATUS_KEY, UNITS_KEY, - INSTRUCTOR_KEY, SEATS_KEY, WAIT_SEATS_KEY, WAIT_CAP_KEY + SEATS_KEY, WAIT_SEATS_KEY, WAIT_CAP_KEY ) def __init__( @@ -586,7 +626,9 @@ def __init__( if not all(entry[field_name] == self.data[0][field_name] for entry in self.data): raise ValueError( - f'{self}: {field_name} fields do not match in data') + f'{self}: {field_name} fields do not match in data: ' + f'{[entry[field_name] for entry in self.data]}' + ) @property def course_id(self) -> str: @@ -728,6 +770,14 @@ def durations(self) -> ty.List['ClassDuration']: durations.append(ClassDuration(day, room, start, end)) return durations + @property + def calendar_duration(self) -> 'CalendarDuration': + """ + Gets calendar duration of section. + :return: CalendarDuration + """ + return CalendarDuration(self.start_date, self.end_date) + @property def rooms(self) -> ty.Set['str']: """ @@ -764,20 +814,24 @@ def units(self) -> float: return float(self.data[0][UNITS_KEY]) @property - def instructor_name(self) -> str: + def instructor_names(self) -> ty.Set[str]: """ - Gets name of instructor. - :return: str + Gets names of instructors for this section. + This will usually be a set containing a single name, but + some classes will have different class sessions taught by + different instructors. + :return: Set[str] """ - return self.data[0][INSTRUCTOR_KEY] + return {entry[INSTRUCTOR_KEY] for entry in self.data} @property - def instructor(self) -> 'InstructorView': + def instructors(self) -> ty.Set['InstructorView']: """ Gets data view of instructor for this course section. :return: InstanceView """ - return InstructorView(self.model, self.instructor_name) + return {InstructorView(self.model, name) for + name in self.instructor_names} @property def open_seats_available(self) -> int: @@ -845,7 +899,41 @@ def model(self) -> DataModel: return self.quarter.model def __repr__(self) -> str: - return f'SectionQuarterView[cid: {self.course_id}, crn: {self.crn}]' + return f'SectionQuarterView[dept: {self.department.name}, cid: ' \ + f'{self.course_id}, crn: {self.crn}]' + + +@owl.serial.serializable +class CalendarDuration: + """ + Class storing data about a class calendar duration. + """ + def __init__(self, start: maya.MayaDT, end: maya.MayaDT): + self.interval = maya.MayaInterval(start, end) + + @classmethod + def from_serializable(cls, d: ty.Dict[str, str]) -> 'CalendarDuration': + return cls(start=maya.when(d['start']), end=maya.when(d['end'])) + + @property + def as_serializable(self) -> ty.Dict[str, str]: + return {'start': self.start.iso8601(), 'end': self.end.iso8601()} + + @property + def start(self) -> maya.MayaDT: + """ + Gets start date. + :return: MayaDT + """ + return self.interval.start + + @property + def end(self) -> maya.MayaDT: + """ + Gets end date. + :return: MayaDT + """ + return self.interval.end class ClassDuration: diff --git a/owl/serial.py b/owl/serial.py new file mode 100644 index 0000000..5f61afa --- /dev/null +++ b/owl/serial.py @@ -0,0 +1,54 @@ +""" +Contains functions and classes for serializing and de-serializing +data. +""" +import typing as ty + +from json import JSONEncoder + +AS_JSON_PROPERTY = 'as_json' +FROM_JSON_PROPERTY = 'from_json' +TYPE_KEY = '_type_key' + +serializable_types = {} + + +class Encoder(JSONEncoder): + """ + Encodes object as json + """ + def default(self, o): + k = _get_type_key(type(o)) + if k in serializable_types: + d = o.as_serializable + d[TYPE_KEY] = k + return d + return super().default(o) + + +def hook(o: ty.Any) -> ty.Any: + try: + k = o[TYPE_KEY] + except KeyError: + return o + else: + object_type = serializable_types[k] + return object_type.from_serializable(o) + + +def serializable(clz): + """ + Decorator for a class marking it as serializable. + :param clz: Type + :return: Type + """ + k = _get_type_key(clz) + if k in serializable_types: + raise ValueError(f'Passed type: {clz} name conflicts with previously' + f'registered type: {serializable_types[k]}') + serializable_types[k] = clz + return clz + + +def _get_type_key(clz: ty.Type): + return clz.__name__ diff --git a/tests/model/test_model.py b/tests/model/test_model.py index bf3b057..713be4f 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -271,14 +271,14 @@ def test_section_units_are_found_correctly(self): section = course.sections['40067'] self.assertEqual(5, section.units) - def test_section_instructor_name_returns_correctly(self): + def test_section_instructor_names_return_correctly(self): with get_test_data_dir('model_test_dir_a') as data_dir: data = DataModel(data_dir) quarter = data.quarters['000011'] dept = quarter.departments['ACTG'] course = dept.courses['1B'] section = course.sections['40067'] - self.assertEqual('Drake', section.instructor_name) + self.assertIn('Drake', section.instructor_names) def test_section_seats_return_correctly_when_on_waitlist_status(self): with get_test_data_dir('model_test_dir_a') as data_dir: @@ -326,6 +326,16 @@ def time_urls(): self.assertEqual(urls1, urls2) self.assertEqual(urls1, urls3) + def test_quarter_primary_start_and_end_dates_are_accurate(self): + # start: "04/09/2018", "end": "06/29/2018" + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + + duration = quarter.primary_duration + self.assertEqual(maya.when('04/09/2018'), duration.start) + self.assertEqual(maya.when('06/29/2018'), duration.end) + class TestCourseDuration(TestCase): def test_durations_intersect_returns_true_with_intersection(self): From 386de1725ddc15efa790da982974467b60e6c1a7 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Thu, 14 Jun 2018 14:05:15 -0700 Subject: [PATCH 57/60] Improved deserialization hook. --- owl/serial.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/owl/serial.py b/owl/serial.py index 5f61afa..cc6c491 100644 --- a/owl/serial.py +++ b/owl/serial.py @@ -27,9 +27,16 @@ def default(self, o): def hook(o: ty.Any) -> ty.Any: + """ + Hook taking the default-deserialized object (usually a dictionary) + and, if the object is recognized as an encoded serializable of + another type, re-creates the type instance. + :param o: Any type deserializable by default. + :return: Any + """ try: - k = o[TYPE_KEY] - except KeyError: + k = o.pop(TYPE_KEY) + except (KeyError, AttributeError): return o else: object_type = serializable_types[k] From 38a46fe0a8aa4e8fd68a322fa3f0e9d0aed11520 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Thu, 14 Jun 2018 14:12:15 -0700 Subject: [PATCH 58/60] Added documentation for encoder default method. --- owl/serial.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/owl/serial.py b/owl/serial.py index cc6c491..7ed37a1 100644 --- a/owl/serial.py +++ b/owl/serial.py @@ -17,7 +17,16 @@ class Encoder(JSONEncoder): """ Encodes object as json """ - def default(self, o): + def default(self, o: ty.Any) -> ty.Any: + """ + Handles the default serialization case, wherein an object is + not by default serializable. + If the object is recognized as a custom serializable object, + it is converted into a serializable object. + Otherwise the superclass JSONEncoder default method is used. + :param o: Any + :return: Any serializable + """ k = _get_type_key(type(o)) if k in serializable_types: d = o.as_serializable From 9cb81a007396b42d96ebb101008d8a7cf6289541 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Thu, 14 Jun 2018 14:23:27 -0700 Subject: [PATCH 59/60] Test speed of and ability to cache custom serializable objects. --- tests/model/test_model.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/model/test_model.py b/tests/model/test_model.py index 713be4f..cf01704 100644 --- a/tests/model/test_model.py +++ b/tests/model/test_model.py @@ -336,6 +336,24 @@ def test_quarter_primary_start_and_end_dates_are_accurate(self): self.assertEqual(maya.when('04/09/2018'), duration.start) self.assertEqual(maya.when('06/29/2018'), duration.end) + def test_caching_improves_quarter_primary_start_and_end_date_access(self): + # start: "04/09/2018", "end": "06/29/2018" + with get_test_data_dir('model_test_dir_a') as data_dir: + data = DataModel(data_dir) + quarter = data.quarters['000011'] + + def time_calendar_date(): + t0 = time.time() + duration = quarter.primary_duration + tf = time.time() + return duration, tf - t0 + + duration1, elapsed1 = time_calendar_date() + duration2, elapsed2 = time_calendar_date() + duration3, elapsed3 = time_calendar_date() + self.assertLess(elapsed2, elapsed1 / 4) + self.assertLess(elapsed3, elapsed1 / 4) + class TestCourseDuration(TestCase): def test_durations_intersect_returns_true_with_intersection(self): @@ -360,6 +378,9 @@ def test_durations_intersect_returns_false_with_differing_times(self): self.assertFalse(a.intersects(b)) +# todo: more rigorously test _hash_args + + def get_test_data_dir(dir_name: str) -> tempfile.TemporaryDirectory: temp_dir = tempfile.TemporaryDirectory() copy_tree(os.path.join(TEST_RESOURCES_DIR, dir_name), temp_dir.name) From 0602e48b7b6b954331f0a618b95d77cffd80a234 Mon Sep 17 00:00:00 2001 From: tryexceptelse Date: Thu, 14 Jun 2018 14:27:20 -0700 Subject: [PATCH 60/60] Added documentation for _get_type_key in serial.py --- owl/serial.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/owl/serial.py b/owl/serial.py index 7ed37a1..d04aebd 100644 --- a/owl/serial.py +++ b/owl/serial.py @@ -66,5 +66,12 @@ def serializable(clz): return clz -def _get_type_key(clz: ty.Type): +def _get_type_key(clz: ty.Type) -> ty.Any: + """ + Gets key used for storing passed type in dictionary of + serializable types. Should return a unique value for each passed + type. + :param clz: Type + :return: ty.Any + """ return clz.__name__