Skip to content

Commit

Permalink
[Test] Add makefile_test.py, to verify make in all build-modes.
Browse files Browse the repository at this point in the history
Previous commit that introduced support for building with LOC-ELF
encoding scheme (i.e, SHA f7e7b10) is only exercising building
with the LOC-ELF encoding scheme for a very simple unit-test,
unit/single_file_elf_src/single_file_prog_elf_test.c

We need a better way to trigger either the default LOC-encoding
or the LOC-ELF encoding technique. The approach chosen, to be implemented
in subsequent commits, is to flag-off of an env-var, LOC_ENABLED={1,2}.
Default mode, with this env-var unset is to use default LOC-encoding.

This commit adds a collection of `make` build-mode pytest cases to exercise
`make` with diff env-variables, such as:

    # Default build mode, using Python generator.
 1. $ make

    # Default LOC-encoding, using Python generator.
 2. $ LOC_ENABLED=1 make ...

    # LOC-ELF encoding, w/o using Python generator.
 3. $ LOC_ENABLED=2 make ...

The test cases carefully verify that:

 a) The required files are generated in the expected sub-dirs for (1) and (2)

 b) The Python generator is _not_ invoked for (3), i.e., no generated files
    are found after the `make` completes. The sources compile because the
    provided src/loc.c and include/loc.h are used to build all the sample /
    unit-test programs.

Some of these test cases are failing so are currently skipped with
this commit. Future changes will be applied to fix the `make` build
rules issues, enabling failing tests.
  • Loading branch information
gapisback committed Apr 11, 2024
1 parent edc2d47 commit 393854f
Show file tree
Hide file tree
Showing 2 changed files with 312 additions and 1 deletion.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -348,8 +348,8 @@ endif

.PHONY: install

# Run the unit-test binary and run individual sample example program binaries.
run-tests: run-unit-tests run-test-code
./test.sh

run-unit-tests: all-tests
@echo
Expand Down
311 changes: 311 additions & 0 deletions tests/makefile_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,311 @@
# #############################################################################
# makefile_test.py
#
"""
Collection of basic test cases to exercise different combinations of `make`
commands. Cross-check that the generation step _does_ happen only when needed
and Makefile-build rules are not indiscriminately triggering the Python
generator when LOC_ENABLED=2 (non-default build mode) is run.
Each test-case hammers out `make` target invocation(s) with different
environment variable settings for LOC_ENABLED, so that we exercise all
possible useful combinations that users could invoke.
"""

# #############################################################################
import os
import subprocess as sp
import pytest

# #############################################################################
# Setup some variables pointing to diff dir/sub-dir full-paths.
# Dir-tree:
# /<....>/LineOfCode/
# loc/<generator-scripts>
# test-code/{ < collection of test program source dirs > }
# tests/<this-file>
# Full dir-path where this tests/ dir lives
LocTestsDir = os.path.realpath(os.path.dirname(__file__))
LocDirRoot = os.path.realpath(LocTestsDir + '/..')

LOC_EXPLICITLY_UNSET = "0"
LOC_DEFAULT = "1"
LOC_ELF_ENCODING = "2"

# #############################################################################
# To see output from test-cases run:
# $ pytest --capture=tee-sys tests/makefile_test.py -k test_make_help
# #############################################################################
def test_make_help():
"""Test `make help`"""

exec_make(['make', 'help'])

# #############################################################################
def test_make_all():
"""Test `make all`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++" })
assert make_rv is True

# #############################################################################
def test_make_all_loc_eq_0():
"""Test `LOC_ENABLED=0 make all`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_EXPLICITLY_UNSET})
assert make_rv is True

# #############################################################################
def test_make_all_tests():
"""Test `make all-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all-tests'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++" })
assert make_rv is True

# #############################################################################
def test_make_all_tests_loc_eq_0():
"""Test `LOC_ENABLED=0 make all-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all-tests'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_EXPLICITLY_UNSET})
assert make_rv is True

# #############################################################################
def test_make_all_run_tests():
"""Test `make all` followed by `make run-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++" })
make_rv = exec_make(['make', 'run-tests'])
assert make_rv is True

# #############################################################################
def test_make_all_run_tests_loc_eq_0():
"""Test `make all` followed by `LOC_ENABLED=0 make run-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_EXPLICITLY_UNSET})
make_rv = exec_make(['make', 'run-tests'])
assert make_rv is True

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=1 build mode")
def test_make_all_run_tests_loc_eq_1():
"""Test `make all` followed by `LOC_ENABLED=1 make run-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_DEFAULT})
make_rv = exec_make(['make', 'run-tests'])
assert make_rv is True
verify_unit_test_gen_files()

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=2 build mode")
def test_make_all_run_tests_loc_elf():
"""Test `make all` followed by `LOC_ENABLED=2 make run-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_ELF_ENCODING})
make_rv = exec_make(['make', 'run-tests'])
assert make_rv is True
verify_unit_test_gen_files(loc_generate = False)

# #############################################################################
def test_make_run_unit_tests():
"""Test `make run-unit-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-unit-tests'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++" })
assert make_rv is True
verify_unit_test_gen_files()

# #############################################################################
def test_make_run_unit_tests_loc_eq_0():
"""Test `LOC_ENABLED=1 make run-unit-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-unit-tests'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_EXPLICITLY_UNSET})
assert make_rv is True
verify_unit_test_gen_files()

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=1 build mode")
def test_make_run_unit_tests_loc_eq_1():
"""Test `LOC_ENABLED=1 make run-unit-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-unit-tests'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_DEFAULT})
assert make_rv is True
verify_unit_test_gen_files()

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=2 build mode")
def test_make_run_unit_tests_loc_elf():
"""Test `LOC_ENABLED=2 make run-unit-tests`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-unit-tests'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_ELF_ENCODING})
assert make_rv is True
verify_unit_test_gen_files(loc_generate = False)

# #############################################################################
@pytest.mark.skip(reason="make all-test-code fails with default build mode")
def test_make_all_test_code():
"""Test `make all-test-code`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all-test-code'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++" })
assert make_rv is True

# #############################################################################
@pytest.mark.skip(reason="make all-test-code fails with default build mode")
def test_make_all_test_code_loc_eq_0():
"""Test `LOC_ENABLED=0 make all-test-code`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'all-test-code'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_EXPLICITLY_UNSET})
assert make_rv is True

# #############################################################################
@pytest.mark.skip(reason="make run-test-code fails with default build mode")
def test_make_run_test_code():
"""Test `make run-test-code`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-test-code'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++" })
assert make_rv is True
verify_test_code_gen_files()

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=0 build mode")
def test_make_run_test_code_loc_eq_0():
"""Test `LOC_ENABLED=1 make run-test-code`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-test-code'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_EXPLICITLY_UNSET})
assert make_rv is True
verify_test_code_gen_files()

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=1 build mode")
def test_make_run_test_code_loc_eq_1():
"""Test `LOC_ENABLED=1 make run-test-code`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-test-code'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_DEFAULT})
assert make_rv is True
verify_test_code_gen_files()

# #############################################################################
@pytest.mark.skip(reason="make fails with LOC_ENABLED=2 build mode")
def test_make_run_test_code_loc_elf():
"""Test `LOC_ENABLED=2 make run-test-code`"""

make_rv = exec_make(['make', 'clean'])
make_rv = exec_make(['make', 'run-test-code'],
{ "BUILD_VERBOSE": "1", "CC": "gcc", "LD": "g++",
"LOC_ENABLED": LOC_ELF_ENCODING})
assert make_rv is True
verify_test_code_gen_files(loc_generate = False)

# #############################################################################
# Helper test methods
# #############################################################################

def verify_unit_test_gen_files(loc_generate:bool = True) -> bool:
"""
Verify the state of generated files for unit-test sources.
For unit-tests we will generate LOC-files in the unit-tests' dir.
For ELF-based unit-test, we should never generate any files.
"""
dirname = LocTestsDir + '/unit/'
for genfile in [ 'loc.h', 'loc_tokens.h', 'loc_filenames.c' ]:
make_rv = verify_file_exists(dirname + 'single_file_src/', genfile)
if loc_generate is True:
assert make_rv is True
else:
assert make_rv is False

make_rv = verify_file_exists(dirname + 'single_file_elf_src/', genfile)
assert make_rv is False


# #############################################################################
def verify_test_code_gen_files(loc_generate:bool = True) -> bool:
"""
Verify the state of generated files for sample test-code sources.
For sample tests we will generate LOC-files in the sample-tests' dir.
While building sample-tests using ELF-based encoding, we should never
generate any files.
"""
dirname = LocDirRoot + '/test-code/'
for test_code_dir in [ 'single-file-cpp-program'
]:
for genfile in [ 'loc.h', 'loc_tokens.h', 'loc_filenames.c' ]:
make_rv = verify_file_exists(dirname + test_code_dir, genfile)
if loc_generate is True:
print(f'{dirname=}, {test_code_dir=}, {genfile=}')
assert make_rv is True
else:
assert make_rv is False

# #############################################################################
def verify_file_exists(dirname:str, filename:str) -> bool:
"""
Check that the expected [generated] file exists.
"""
return os.path.exists(dirname + '/' + filename)

# #############################################################################
def exec_make(cmdargs:list, extra_env:dict = None) -> bool:
"""Execute `make` command with specified arguments."""
if extra_env is None:
extra_env = {}
try:
# Need to run `make` from the dir where Makefile lives.
# Add-in user-specified env-vars to existing env of process.
result = sp.run(cmdargs, text=True, check=True, capture_output=True,
cwd = LocDirRoot,
env={**os.environ, **extra_env}
)
except sp.CalledProcessError as exc:
print("sp.run() Status: FAIL, rc =", exc.returncode,
"\nargs =", exc.args,
"\nstdout =", exc.stdout,
"\nstderr =", exc.stderr)
return False
else:
for line in str(result).split('\\n'):
print(line)
return True

0 comments on commit 393854f

Please sign in to comment.