From 6b9f80a7c6173ca23d4c878724332bbdc0f77626 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Fri, 3 Oct 2025 11:43:06 +0200 Subject: [PATCH 1/2] Place all the fixture related test support code in its own file. --- tests/support/__init__.py | 141 +------------------- tests/support/fixturesupp.py | 175 +++++++++++++++++++++++++ tests/test_mig_shared_accountreq.py | 8 +- tests/test_mig_shared_configuration.py | 7 +- tests/test_mig_shared_install.py | 3 +- tests/test_mig_shared_localfile.py | 3 +- 6 files changed, 194 insertions(+), 143 deletions(-) create mode 100644 tests/support/fixturesupp.py diff --git a/tests/support/__init__.py b/tests/support/__init__.py index 68c14043a..fa86c20f7 100644 --- a/tests/support/__init__.py +++ b/tests/support/__init__.py @@ -42,7 +42,8 @@ from unittest import TestCase, main as testmain from tests.support.configsupp import FakeConfiguration -from tests.support.suppconst import MIG_BASE, TEST_BASE, TEST_FIXTURE_DIR, \ +from tests.support.fixturesupp import _PreparedFixture +from tests.support.suppconst import MIG_BASE, TEST_BASE, \ TEST_DATA_DIR, TEST_OUTPUT_DIR, ENVHELP_OUTPUT_DIR from tests.support._env import MIG_ENV, PY2 @@ -350,61 +351,6 @@ def pretty_display_path(absolute_path): assert not relative_path.startswith('..') return relative_path - def prepareFixtureAssert(self, fixture_relpath, fixture_format=None): - """Prepare to assert a value against a fixture.""" - - fixture_data, fixture_path = fixturefile( - fixture_relpath, fixture_format) - return SimpleNamespace( - assertAgainstFixture=lambda val: MigTestCase._assertAgainstFixture( - self, - fixture_format, - fixture_data, - fixture_path, - value=val - ), - copy_as_temp=lambda prefix: self._fixture_copy_as_temp( - self, - fixture_format, - fixture_data, - fixture_path, - prefix=prefix - ) - ) - - @staticmethod - def _assertAgainstFixture(testcase, fixture_format, fixture_data, fixture_path, value=None): - """Compare a value against fixture data ensuring that in the case of - failure the location of the fixture is prepended to the diff.""" - - assert value is not None - originalMaxDiff = testcase.maxDiff - testcase.maxDiff = None - - raised_exception = None - try: - testcase.assertEqual(value, fixture_data) - except AssertionError as diffexc: - raised_exception = diffexc - finally: - testcase.maxDiff = originalMaxDiff - if raised_exception: - message = "value differed from fixture stored at %s\n\n%s" % ( - _to_display_path(fixture_path), raised_exception) - raise AssertionError(message) - - @staticmethod - def _fixture_copy_as_temp(testcase, fixture_format, fixture_data, fixture_path, prefix=None): - """Copy a fixture to temporary file at the given path prefix.""" - - assert prefix is not None - fixture_basename = os.path.basename(fixture_path) - fixture_name = fixture_basename[0:-len(fixture_format) - 1] - normalised_path = fixturefile_normname(fixture_name, prefix=prefix) - copied_fixture_file = testcase.temppath(normalised_path) - shutil.copyfile(fixture_path, copied_fixture_file) - return copied_fixture_file - @staticmethod def _provision_test_user(self, distinguished_name): """Provide a means to fabricate a useable test user on demand. @@ -418,7 +364,10 @@ def _provision_test_user(self, distinguished_name): # ensure a user db that includes our test user conf_user_db_home = ensure_dirs_exist(self.configuration.user_db_home) - prepared_fixture = self.prepareFixtureAssert( + # note: this is a non-standard direct use of fixture preparation due + # to this being bootstrap code and should not be used elsewhere + prepared_fixture = _PreparedFixture.from_relpath( + self, 'MiG-users.db--example', fixture_format='pickle', ) @@ -462,84 +411,6 @@ def ensure_dirs_exist(absolute_dir): return absolute_dir -def fixturefile(relative_path, fixture_format=None): - """Support function for loading fixtures from their serialised format. - - Doing so is a little more involved than it may seem because serialisation - formats may not capture various nuances of the python data they represent. - For this reason each supported format defers to a format specific function - which can then, for example, load hints about deserialization. - """ - - assert fixture_format is not None, "fixture format must be specified" - assert not os.path.isabs( - relative_path), "fixture is not relative to fixture folder" - relative_path_with_ext = "%s.%s" % (relative_path, fixture_format) - tmp_path = os.path.join(TEST_FIXTURE_DIR, relative_path_with_ext) - assert os.path.isfile(tmp_path), \ - "fixture file for format is not present: %s" % \ - (relative_path_with_ext,) - #_, extension = os.path.splitext(os.path.basename(tmp_path)) - #assert fixture_format == extension, "fixture file does not match format" - - data = None - - if fixture_format == 'binary' or fixture_format == 'pickle': - with open(tmp_path, 'rb') as binfile: - data = binfile.read() - elif fixture_format == 'json': - data = _fixturefile_json(tmp_path) - else: - raise AssertionError( - "unsupported fixture format: %s" % (fixture_format,)) - - if fixture_format == 'pickle': - data = pickle.loads(data) - - return data, tmp_path - - -def fixturefile_normname(relative_path, prefix=''): - """Grab normname from relative_path and optionally add a path prefix""" - normname, _ = relative_path.split('--') - if prefix: - return os.path.join(prefix, normname) - return normname - - -_FIXTUREFILE_HINTAPPLIERS = { - 'array_of_tuples': lambda value: [tuple(x) for x in value] -} - - -def _fixturefile_json(json_path): - hints = ConfigParser() - - # let's see if there are loading hints - try: - hints_path = "%s.ini" % (json_path,) - with open(hints_path) as hints_file: - hints.read_file(hints_file) - except FileNotFoundError: - pass - - with io.open(json_path) as json_file: - json_object = json.load(json_file) - - for item_name, item_hint in hints['DEFAULT'].items(): - loaded_value = json_object[item_name] - value_from_loaded_value = _FIXTUREFILE_HINTAPPLIERS[item_hint] - json_object[item_name] = value_from_loaded_value(loaded_value) - - return json_object - - -def fixturepath(relative_path): - """Get absolute fixture path for relative_path""" - tmp_path = os.path.join(TEST_FIXTURE_DIR, relative_path) - return tmp_path - - def temppath(relative_path, test_case, ensure_dir=False): """Register relative_path as a temp path and schedule automatic clean up after unit tests unless skip_clean is set. Anchors the temp path in diff --git a/tests/support/fixturesupp.py b/tests/support/fixturesupp.py new file mode 100644 index 000000000..056334936 --- /dev/null +++ b/tests/support/fixturesupp.py @@ -0,0 +1,175 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# configsupp - configuration helpers for unit tests +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Fixture related details within the test support library.""" + +from configparser import ConfigParser +import json +import os +import pickle +import shutil +from types import SimpleNamespace + +from tests.support.suppconst import TEST_FIXTURE_DIR + + +def _fixturefile_loadrelative(relative_path, fixture_format=None): + """Support function for loading fixtures from their serialised format. + + Doing so is a little more involved than it may seem because serialisation + formats may not capture various nuances of the python data they represent. + For this reason each supported format defers to a format specific function + which can then, for example, load hints about deserialization. + """ + + assert fixture_format is not None, "fixture format must be specified" + assert not os.path.isabs( + relative_path), "fixture is not relative to fixture folder" + relative_path_with_ext = "%s.%s" % (relative_path, fixture_format) + tmp_path = os.path.join(TEST_FIXTURE_DIR, relative_path_with_ext) + assert os.path.isfile(tmp_path), \ + "fixture file for format is not present: %s" % \ + (relative_path_with_ext,) + #_, extension = os.path.splitext(os.path.basename(tmp_path)) + #assert fixture_format == extension, "fixture file does not match format" + + data = None + + if fixture_format == 'binary' or fixture_format == 'pickle': + with open(tmp_path, 'rb') as binfile: + data = binfile.read() + elif fixture_format == 'json': + data = _fixturefile_json(tmp_path) + else: + raise AssertionError( + "unsupported fixture format: %s" % (fixture_format,)) + + if fixture_format == 'pickle': + data = pickle.loads(data) + + return data, tmp_path + + +def _fixturefile_normname(relative_path, prefix=''): + """Grab normname from relative_path and optionally add a path prefix""" + normname, _ = relative_path.split('--') + if prefix: + return os.path.join(prefix, normname) + return normname + + +_FIXTUREFILE_HINTAPPLIERS = { + 'array_of_tuples': lambda value: [tuple(x) for x in value] +} + + +def _fixturefile_json(json_path): + hints = ConfigParser() + + # let's see if there are loading hints + try: + hints_path = "%s.ini" % (json_path,) + with open(hints_path) as hints_file: + hints.read_file(hints_file) + except FileNotFoundError: + pass + + with open(json_path) as json_file: + json_object = json.load(json_file) + + for item_name, item_hint in hints['DEFAULT'].items(): + loaded_value = json_object[item_name] + value_from_loaded_value = _FIXTUREFILE_HINTAPPLIERS[item_hint] + json_object[item_name] = value_from_loaded_value(loaded_value) + + return json_object + + +def fixturepath(relative_path): + """Get absolute fixture path for relative_path""" + tmp_path = os.path.join(TEST_FIXTURE_DIR, relative_path) + return tmp_path + + +class _PreparedFixture: + def __init__(self, testcase, + fixture_format, + fixture_data, + fixture_path): + self.testcase = testcase + self.fixture_format = fixture_format + self.fixture_data = fixture_data + self.fixture_path = fixture_path + + def assertAgainstFixture(self, value): + """Compare a value against fixture data ensuring that in the case of + failure the location of the fixture is prepended to the diff.""" + + assert value is not None + testcase = self.testcase + originalMaxDiff = testcase.maxDiff + testcase.maxDiff = None + + raised_exception = None + try: + testcase.assertEqual(value, self.fixture_data) + except AssertionError as diffexc: + raised_exception = diffexc + finally: + testcase.maxDiff = originalMaxDiff + if raised_exception: + message = "value differed from fixture stored at %s\n\n%s" % ( + _to_display_path(self.fixture_path), raised_exception) + raise AssertionError(message) + + def copy_as_temp(self, prefix=None): + """Copy a fixture to temporary file at the given path prefix.""" + + assert prefix is not None + fixture_basename = os.path.basename(self.fixture_path) + fixture_name = fixture_basename[0:-len(self.fixture_format) - 1] + normalised_path = _fixturefile_normname(fixture_name, prefix=prefix) + copied_fixture_file = self.testcase.temppath(normalised_path) + shutil.copyfile(self.fixture_path, copied_fixture_file) + return copied_fixture_file + + @staticmethod + def from_relpath(testcase, fixture_relpath, fixture_format): + """ + Instantiate a fixture hint object from a supplied relative path to + the on-disk fixture file. + """ + + fixture_data, fixture_path = _fixturefile_loadrelative( + fixture_relpath, fixture_format) + return _PreparedFixture(testcase, fixture_format, fixture_data, fixture_path) + + +class FixtureAssertMixin: + def prepareFixtureAssert(self, fixture_relpath, fixture_format=None): + """Prepare to assert a value against a fixture.""" + return _PreparedFixture.from_relpath(self, fixture_relpath, fixture_format) diff --git a/tests/test_mig_shared_accountreq.py b/tests/test_mig_shared_accountreq.py index 08de1011a..c847e34bc 100644 --- a/tests/test_mig_shared_accountreq.py +++ b/tests/test_mig_shared_accountreq.py @@ -33,7 +33,8 @@ import sys import unittest -from tests.support import MigTestCase, testmain, fixturefile, ensure_dirs_exist +from tests.support import MigTestCase, testmain, ensure_dirs_exist +from tests.support.fixturesupp import FixtureAssertMixin import mig.shared.accountreq as accountreq from mig.shared.base import canonical_user, distinguished_name_to_user, \ @@ -41,7 +42,7 @@ from mig.shared.defaults import keyword_auto -class MigSharedAccountreq__peers(MigTestCase): +class MigSharedAccountreq__peers(MigTestCase, FixtureAssertMixin): """Unit tests for peers related functions within the accountreq module""" TEST_PEER_DN = '/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=peer@example.com' @@ -68,7 +69,8 @@ def _string_if_bytes(value): return {_string_if_bytes(x): _string_if_bytes(y) for x, y in value.items()} def _peer_dict_from_fixture(self): - fixture_data, _ = fixturefile("peer_user_dict", fixture_format="json") + prepared_fixture = self.prepareFixtureAssert("peer_user_dict", fixture_format="json") + fixture_data = prepared_fixture.fixture_data assert fixture_data["distinguished_name"] == self.TEST_PEER_DN return fixture_data diff --git a/tests/test_mig_shared_configuration.py b/tests/test_mig_shared_configuration.py index 7c9aef06e..bda302ed5 100644 --- a/tests/test_mig_shared_configuration.py +++ b/tests/test_mig_shared_configuration.py @@ -31,8 +31,9 @@ import os import unittest -from tests.support import MigTestCase, TEST_DATA_DIR, PY2, testmain, \ - fixturefile +from tests.support import MigTestCase, TEST_DATA_DIR, PY2, testmain +from tests.support.fixturesupp import FixtureAssertMixin + from mig.shared.configuration import Configuration @@ -45,7 +46,7 @@ def _to_dict(obj): if not (k.startswith('__') or _is_method(v))} -class MigSharedConfiguration(MigTestCase): +class MigSharedConfiguration(MigTestCase, FixtureAssertMixin): """Wrap unit tests for the corresponding module""" def test_argument_storage_protocols(self): diff --git a/tests/test_mig_shared_install.py b/tests/test_mig_shared_install.py index 5c7e567a1..b06f3c928 100644 --- a/tests/test_mig_shared_install.py +++ b/tests/test_mig_shared_install.py @@ -37,7 +37,8 @@ import sys from tests.support import MIG_BASE, TEST_OUTPUT_DIR, MigTestCase, \ - testmain, temppath, cleanpath, fixturepath, is_path_within + testmain, temppath, cleanpath, is_path_within +from tests.support.fixturesupp import fixturepath from mig.shared.defaults import keyword_auto from mig.shared.install import determine_timezone, generate_confs diff --git a/tests/test_mig_shared_localfile.py b/tests/test_mig_shared_localfile.py index cc856d0e9..62a5112aa 100644 --- a/tests/test_mig_shared_localfile.py +++ b/tests/test_mig_shared_localfile.py @@ -36,7 +36,8 @@ sys.path.append(os.path.realpath( os.path.join(os.path.dirname(__file__), ".."))) -from tests.support import MigTestCase, fixturepath, temppath, testmain +from tests.support import MigTestCase, temppath, testmain + from mig.shared.serverfile import LOCK_EX from mig.shared.localfile import LocalFile From f9fab4cba8e4d1f888876415102a204e1cb44ebd Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Mon, 6 Oct 2025 13:19:48 +0200 Subject: [PATCH 2/2] Double down on human readable fixtures and support via extended hints. One of the key aspects of using fixtures is maintaining the ability to know their contents with ease. This was slightly undermined by allowing pickled fixture data. Use the opportunity presented by having to handle the inclusion of timestamps at now points in the future, something that is something of a necessity for use of fixtures going forward, both to add support for doing so but push it further so such values are specified within a file readable format. There were pre-existing provisions for loading JSON as well as being able to specify additional transformations of properties to perform as a series of hints. Extend and make use of this mechanism for a new fixture output path - this allows fixture data to be written out in a forat different than it is read in (which can include transformation). Make use of this new support to rework the test user provisioning. The use is now defined in JSON and, rather than copy a pre-existing file representing the requisite user database, write this database based on the JSON data. Do the work necessary to make sure the data in this file is output with strings transformed into bytes as is the format of the user database. --- tests/fixture/MiG-users.db--example.json | 24 ++ tests/fixture/MiG-users.db--example.json.ini | 2 + tests/fixture/MiG-users.db--example.pickle | 187 ---------------- .../mig_shared_configuration--new.json.ini | 2 +- tests/fixture/peer_user_dict.json | 7 +- tests/support/__init__.py | 18 +- tests/support/fixturesupp.py | 211 +++++++++++++++--- 7 files changed, 223 insertions(+), 228 deletions(-) create mode 100644 tests/fixture/MiG-users.db--example.json create mode 100644 tests/fixture/MiG-users.db--example.json.ini delete mode 100644 tests/fixture/MiG-users.db--example.pickle diff --git a/tests/fixture/MiG-users.db--example.json b/tests/fixture/MiG-users.db--example.json new file mode 100644 index 000000000..5f2707c12 --- /dev/null +++ b/tests/fixture/MiG-users.db--example.json @@ -0,0 +1,24 @@ +{ + "/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@example.com": { + "full_name": "Test User", + "organization": "Test Org", + "state": "NA", + "country": "DK", + "email": "test@example.com", + "comment": "This is the create comment", + "password": "", + "password_hash": "PBKDF2$sha256$10000$XMZGaar/pU4PvWDr$w0dYjezF6JGtSiYPexyZMt3lM2134uix", + "distinguished_name": "/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@example.com", + "locality": "", + "organizational_unit": "", + "expire": { + "_FixtureHint": true, + "hint": "today_relative", + "modifier": "days|+5", + "value": 0 + }, + "created": 1759921716.5731237, + "unique_id": "8txp5j0lxVL5xiWp0N4pxzeswIVWlkEuqubucyubWiSaSGZeH0bZDbHQr46FQoBT", + "openid_names": [] + } +} diff --git a/tests/fixture/MiG-users.db--example.json.ini b/tests/fixture/MiG-users.db--example.json.ini new file mode 100644 index 000000000..70f5a7de6 --- /dev/null +++ b/tests/fixture/MiG-users.db--example.json.ini @@ -0,0 +1,2 @@ +[ONWRITE] +convert_dict_bytes_to_strings_kv = True diff --git a/tests/fixture/MiG-users.db--example.pickle b/tests/fixture/MiG-users.db--example.pickle deleted file mode 100644 index 69374a1ae..000000000 --- a/tests/fixture/MiG-users.db--example.pickle +++ /dev/null @@ -1,187 +0,0 @@ -(dp0 -V/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=dummy-user -p1 -(dp2 -Vfull_name -p3 -VTest User -p4 -sVorganization -p5 -VTest Org -p6 -sVstate -p7 -VNA -p8 -sVcountry -p9 -VDK -p10 -sVemail -p11 -Vdummy-user -p12 -sVcomment -p13 -VThis is the create comment -p14 -sVpassword -p15 -V -p16 -sVpassword_hash -p17 -VPBKDF2$sha256$10000$t0JM/JjkQ347th0Q$QupJt53hA5KhESEeqDhTQTCPOrCBvZ6H -p18 -sVdistinguished_name -p19 -V/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=dummy-user -p20 -sVlocality -p21 -g16 -sVorganizational_unit -p22 -g16 -sVexpire -p23 -I9999999999 -sVcreated -p24 -F1726233828.2676349 -sVunique_id -p25 -VktyCKIRg9HvsVzXMQ22EaKS67t9atchv9JKTiJqrtBiGN3qksKrbTTYIH8mitY2K -p26 -sVopenid_names -p27 -(lp28 -sVold_password_hash -p29 -VPBKDF2$sha256$10000$GL7Qq92iLe/hZXBo$ZwB/5IZqgU7onP+ZqZk9zcHVZOx7jmWz -p30 -sVrenewed -p31 -F1726389298.7801197 -ssV/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=dummy-user -p32 -(dp33 -Vfull_name -p34 -VTest User -p35 -sVorganization -p36 -VTest Org -p37 -sVstate -p38 -VNA -p39 -sVcountry -p40 -VDK -p41 -sVemail -p42 -Vdummy-user -p43 -sVcomment -p44 -VThis is the create comment -p45 -sVpassword -p46 -g16 -sVpassword_hash -p47 -VPBKDF2$sha256$10000$kZ8WgLNH+wg3X11d$t1d08MV4g215WYW7S7EbkjHqDF+MCjMa -p48 -sVdistinguished_name -p49 -V/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=dummy-user -p50 -sVlocality -p51 -g16 -sVorganizational_unit -p52 -g16 -sVexpire -p53 -I9999999999 -sVcreated -p54 -F1726602273.7987707 -sVunique_id -p55 -VKdYHJ21t37jAoHUmBq6t8Xnsnih6JWR5i0QepHoVXfDpQxz9fQGnEmegoDNrPzbe -p56 -sVopenid_names -p57 -(lp58 -sVold_password_hash -p59 -VPBKDF2$sha256$10000$yObizsUepZvvJ0/r$uKIt7n6Lf/7WXD6pKDGyvT30L2uowBnV -p60 -sVrenewed -p61 -F1727707333.0969944 -ssV/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@example.com -p62 -(dp63 -Vfull_name -p64 -VTest User -p65 -sVorganization -p66 -VTest Org -p67 -sVstate -p68 -VNA -p69 -sVcountry -p70 -VDK -p71 -sVemail -p72 -Vtest@example.com -p73 -sVcomment -p74 -VThis is the create comment -p75 -sVpassword -p76 -g16 -sVpassword_hash -p77 -VPBKDF2$sha256$10000$/TkhLk4yMGf6XhaY$7HUeQ9iwCkE4YMQAaCd+ZdrN+y8EzkJH -p78 -sVdistinguished_name -p79 -g62 -sVlocality -p80 -g16 -sVorganizational_unit -p81 -g16 -sVexpire -p82 -I9999999999 -sVcreated -p83 -F1727434813.0792377 -sVunique_id -p84 -VaTza92klrnN2wfylm6HnphCy9C3PReGpQ6jklJ7zF3xjeaUDw36tW95Avx43vtba -p85 -sVopenid_names -p86 -(lp87 -ss. diff --git a/tests/fixture/mig_shared_configuration--new.json.ini b/tests/fixture/mig_shared_configuration--new.json.ini index b90e780f0..970efccaa 100644 --- a/tests/fixture/mig_shared_configuration--new.json.ini +++ b/tests/fixture/mig_shared_configuration--new.json.ini @@ -1,4 +1,4 @@ -[DEFAULT] +[ATTRIBUTES] auto_add_user_permit = array_of_tuples auto_add_user_with_peer = array_of_tuples site_cloud_access = array_of_tuples diff --git a/tests/fixture/peer_user_dict.json b/tests/fixture/peer_user_dict.json index 790a65017..1a30ae371 100644 --- a/tests/fixture/peer_user_dict.json +++ b/tests/fixture/peer_user_dict.json @@ -7,7 +7,12 @@ "comment": "test@example.com", "locality": "", "organizational_unit": "", - "expire": 1758970812, + "expire": { + "_FixtureHint": true, + "hint": "today_relative", + "modifier": "days|+5", + "value": 0 + }, "created": 1727434813.0792377, "openid_names": [], "distinguished_name": "/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=peer@example.com" diff --git a/tests/support/__init__.py b/tests/support/__init__.py index fa86c20f7..62d352a2b 100644 --- a/tests/support/__init__.py +++ b/tests/support/__init__.py @@ -352,10 +352,12 @@ def pretty_display_path(absolute_path): return relative_path @staticmethod - def _provision_test_user(self, distinguished_name): + def _provision_test_user(testcase, distinguished_name): """Provide a means to fabricate a useable test user on demand. """ + self = testcase + # ensure a user home directory for our test user conf_user_home = os.path.normpath(self.configuration.user_home) from mig.shared.base import client_id_dir @@ -369,10 +371,8 @@ def _provision_test_user(self, distinguished_name): prepared_fixture = _PreparedFixture.from_relpath( self, 'MiG-users.db--example', - fixture_format='pickle', - ) - - test_db_file = prepared_fixture.copy_as_temp(prefix=conf_user_db_home) + fixture_format='json') + prepared_fixture.write_to_dir(conf_user_db_home, output_format='pickle') # create the test user home directory ensure_dirs_exist(test_user_dir) @@ -383,14 +383,6 @@ def _provision_test_user(self, distinguished_name): return test_user_dir -def _to_display_path(value): - """Convert a relative path to one to be shown as part of test output.""" - display_path = os.path.relpath(value, MIG_BASE) - if not display_path.startswith('.'): - return "./" + display_path - return display_path - - def is_path_within(path, start=None, _msg=None): """Check if path is within start directory""" try: diff --git a/tests/support/fixturesupp.py b/tests/support/fixturesupp.py index 056334936..032e9e67e 100644 --- a/tests/support/fixturesupp.py +++ b/tests/support/fixturesupp.py @@ -28,13 +28,15 @@ """Fixture related details within the test support library.""" from configparser import ConfigParser +from datetime import date, timedelta import json import os import pickle import shutil +from time import mktime from types import SimpleNamespace -from tests.support.suppconst import TEST_FIXTURE_DIR +from tests.support.suppconst import MIG_BASE, TEST_FIXTURE_DIR def _fixturefile_loadrelative(relative_path, fixture_format=None): @@ -59,18 +61,17 @@ def _fixturefile_loadrelative(relative_path, fixture_format=None): data = None - if fixture_format == 'binary' or fixture_format == 'pickle': + if fixture_format == 'binary': with open(tmp_path, 'rb') as binfile: data = binfile.read() elif fixture_format == 'json': - data = _fixturefile_json(tmp_path) + with open(tmp_path) as jsonfile: + data = json.load(jsonfile, object_hook=_FixtureHint.object_hook) + _hints_apply_if_present(tmp_path, data) else: raise AssertionError( "unsupported fixture format: %s" % (fixture_format,)) - if fixture_format == 'pickle': - data = pickle.loads(data) - return data, tmp_path @@ -82,31 +83,147 @@ def _fixturefile_normname(relative_path, prefix=''): return normname -_FIXTUREFILE_HINTAPPLIERS = { - 'array_of_tuples': lambda value: [tuple(x) for x in value] +def _hints_apply_array_of_tuples(value, modifier): + """Generate values for array_of_tuples hint.""" + assert modifier is None + return [tuple(x) for x in value] + + +def _hints_apply_today_relative(value, modifier): + """Generate values for today_relative hint.""" + + kind, delta = modifier.split('|') + if kind == "days": + time_delta = timedelta(days=int(delta)) + adjusted_datetime = date.today() + time_delta + return int(mktime(adjusted_datetime.timetuple())) + else: + raise NotImplementedError("unspported today_relative modifier") + + +def _hints_apply_dict_bytes_to_strings_kv(input_dict): + output_dict = {} + + for k, v in input_dict.items(): + key_to_use = k + if isinstance(k, str): + key_to_use = bytes(k, 'utf8') + + if isinstance(v, dict): + output_dict[key_to_use] = _hints_apply_dict_bytes_to_strings_kv(v) + continue + + val_to_use = v + if isinstance(v, str): + val_to_use = bytes(v, 'utf8') + + output_dict[key_to_use] = val_to_use + + return output_dict + + +_FIXTUREFILE_APPLIERS_ATTRIBUTES = { + 'array_of_tuples': _hints_apply_array_of_tuples, + 'today_relative': _hints_apply_today_relative, +} + + +_FIXTUREFILE_APPLIERS_ONWRITE = { + 'convert_dict_bytes_to_strings_kv': _hints_apply_dict_bytes_to_strings_kv, } -def _fixturefile_json(json_path): +def _hints_apply_if_present(fixture_path, json_object): + """Apply hints to the supplied data in-place if relevant.""" + + _hints_apply_from_instances_if_present(json_object) + _hints_apply_from_ini_if_present(fixture_path, json_object) + + +def _hints_apply_from_instances_if_present(json_object): + """Recursively aply hints to any hint instances in the supplied data.""" + + for k, v in json_object.items(): + if isinstance(v, dict): + _hints_apply_from_instances_if_present(v) + continue + + if isinstance(v, _FixtureHint): + json_object[k] = _FixtureHint.decode_hint(v) + pass + + +def _hints_for_fixture(fixture_path): + """Load any hints that may be specified for a given fixture.""" + hints = ConfigParser() # let's see if there are loading hints try: - hints_path = "%s.ini" % (json_path,) + hints_path = "%s.ini" % (fixture_path,) with open(hints_path) as hints_file: hints.read_file(hints_file) except FileNotFoundError: pass - with open(json_path) as json_file: - json_object = json.load(json_file) + # ensure empty required fixture to avoid extra conditionals later + for required_section in ['ATTRIBUTES']: + if not hints.has_section(required_section): + hints.add_section(required_section) + + return hints + + +def _hints_apply_from_ini_if_present(fixture_path, json_object): + """Amend the supplied object in place with any applicable hints.""" + + hints = _hints_for_fixture(fixture_path) + + # apply any attriutes hints ahead of specified conversions such that any + # key can be specified matching what is visible within the loaded fixture + for item_name, item_hint_unparsed in hints['ATTRIBUTES'].items(): + loaded_value = json_object[item_name] + + item_hint_and_maybe_modifier = item_hint_unparsed.split('--') + item_hint = item_hint_and_maybe_modifier[0] + if len(item_hint_and_maybe_modifier) == 2: + modifier = item_hint_and_maybe_modifier[1] + elif len(item_hint_and_maybe_modifier) == 1: + modifier = None + else: + raise NotImplementedError("failed to parse hint and modifier") + value_from_loaded_value = _FIXTUREFILE_APPLIERS_ATTRIBUTES[item_hint] + + json_object[item_name] = value_from_loaded_value(loaded_value, modifier) + + +class _FixtureHint: + """Named type allowing idenfication of fixture hints.""" + + def __init__(self, hint=None, modifier=None, value=None): + self.hint = hint + self.modifier = modifier + self.value = value + + @staticmethod + def decode_hint(hint_obj): + """Produce a value based on the properties of a hint instance.""" + assert isinstance(hint_obj, _FixtureHint) + value_from_loaded_value = _FIXTUREFILE_APPLIERS_ATTRIBUTES[hint_obj.hint] + return value_from_loaded_value(hint_obj.value, hint_obj.modifier) + + @staticmethod + def object_hook(decoded_object): + """ + Function for use as JSON loading hook which will transform + the serialised representation of a hint into an instance. + """ - for item_name, item_hint in hints['DEFAULT'].items(): - loaded_value = json_object[item_name] - value_from_loaded_value = _FIXTUREFILE_HINTAPPLIERS[item_hint] - json_object[item_name] = value_from_loaded_value(loaded_value) + if "_FixtureHint" in decoded_object: + fixture_hint = _FixtureHint(decoded_object["hint"], decoded_object["modifier"]) + return _FixtureHint.decode_hint(fixture_hint) - return json_object + return decoded_object def fixturepath(relative_path): @@ -115,7 +232,19 @@ def fixturepath(relative_path): return tmp_path +def _to_display_path(value): + """Convert an absolute path to one to be shown as part of test output.""" + display_path = os.path.relpath(value, MIG_BASE) + if not display_path.startswith('.'): + return "./" + display_path + return display_path + + class _PreparedFixture: + """ + Object representing a loaded fixture prepared for use within a test case. + """ + def __init__(self, testcase, fixture_format, fixture_data, @@ -146,22 +275,52 @@ def assertAgainstFixture(self, value): _to_display_path(self.fixture_path), raised_exception) raise AssertionError(message) - def copy_as_temp(self, prefix=None): - """Copy a fixture to temporary file at the given path prefix.""" + def write_to_dir(self, target_dir, output_format=None): + """ + Write loaded fixture data to temporary file to the specified target + directory applying any onwrite hints that may be specified. + """ - assert prefix is not None + assert os.path.isabs(target_dir) fixture_basename = os.path.basename(self.fixture_path) fixture_name = fixture_basename[0:-len(self.fixture_format) - 1] - normalised_path = _fixturefile_normname(fixture_name, prefix=prefix) - copied_fixture_file = self.testcase.temppath(normalised_path) - shutil.copyfile(self.fixture_path, copied_fixture_file) - return copied_fixture_file + normalised_path = _fixturefile_normname(fixture_name, prefix=target_dir) + fixture_file_target = self.testcase.temppath(normalised_path) + + output_data = self.fixture_data + + # now apply any onwrite conversions + hints = _hints_for_fixture(self.fixture_path) + for item_name in hints['ONWRITE']: + if item_name not in _FIXTUREFILE_APPLIERS_ONWRITE: + raise AssertionError( + "unsupported fixture conversion: %s" % (item_name,)) + + enabled = hints.getboolean('ONWRITE', item_name) + if not enabled: + continue + + apply_conversion = _FIXTUREFILE_APPLIERS_ONWRITE[item_name] + output_data = apply_conversion(output_data) + + if output_format == 'binary': + with open(fixture_file_target, 'wb') as fixture_outputfile: + fixture_outputfile.write(output_data) + elif output_format == 'json': + with open(fixture_file_target, 'w') as fixture_outputfile: + json.dump(output_data, fixture_outputfile) + elif output_format == 'pickle': + with open(fixture_file_target, 'wb') as fixture_outputfile: + pickle.dump(output_data, fixture_outputfile) + else: + raise AssertionError( + "unsupported fixture format: %s" % (output_format,)) @staticmethod def from_relpath(testcase, fixture_relpath, fixture_format): """ - Instantiate a fixture hint object from a supplied relative path to - the on-disk fixture file. + Obtain a prepared fixture given a relative path to the on-disk file + containing its data. """ fixture_data, fixture_path = _fixturefile_loadrelative(