diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..8164ee47 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +TEST_FILE := tests/run_all.py +ALL_TESTS = $(patsubst %.py,%,$(wildcard tests/test_*.py)) # e.g. tests/test_content tests/... + +# Run the test of a single file +test: export PYTHONWARNINGS=ignore +test: $(TEST_FILE) + -python $^ + +# Run submake for testing each test file +test_parallel: $(ALL_TESTS) + +$(ALL_TESTS):: + @$(MAKE) test TEST_FILE="$@.py" diff --git a/docs/environment.yml b/docs/environment.yml index d9b18dba..1c9f3539 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -3,3 +3,5 @@ dependencies: - python=3.5.1=0 - sphinx>1.4.0 - sphinx_rtd_theme>=0.1.9 +- pip: + - recommonmark>=0.4.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index 31cc87f5..7bd6be8b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -299,6 +299,15 @@ add_module_names = False +from recommonmark.transform import AutoStructify +def setup(app): + github_doc_root = 'https://github.com/datacamp/pythonwhat/blob/master/docs/source/' + app.add_config_value('recommonmark_config', { + 'url_resolver': lambda url: github_doc_root + url, + 'auto_toc_tree_section': 'Contents', + }, True) + app.add_transform(AutoStructify) + on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme diff --git a/docs/source/Processes.md b/docs/source/expression_tests.md similarity index 100% rename from docs/source/Processes.md rename to docs/source/expression_tests.md diff --git a/docs/source/index.rst b/docs/source/index.rst index a4d07b2c..d30a9c36 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,13 +8,21 @@ Contents: :maxdepth: 2 Home - quickstart_guide.md - simple_tests.md - parts_cheatsheet.rst - Processes - logic + quickstart_guide + simple_tests/index.rst + part_checks + expression_tests + logic_tests/index.rst spec2_summary - test_functions + + +Pythonwhat V1 +------------- + +.. toctree:: + + pythonwhat.wiki/index.rst + Indices and tables ================== diff --git a/docs/source/logic.md b/docs/source/logic.md deleted file mode 100644 index f1f19436..00000000 --- a/docs/source/logic.md +++ /dev/null @@ -1,135 +0,0 @@ -Logic -============== - -[TODO: this is a previous version, will update with spec2 functions] - -Several functions, such as `test_correct()`, `test_or()`, `test_function_definition()`, among others, also have arguments that expect another set of tests. This article will explain the different ways of specifying these 'sub-tests'. - -Let's take the example of `test_correct()`; this function takes two sets of tests. The first set is executed, and if it's fine, the second set is left alone. If the first set of tests results in an error, the second set is executed and the feedback is logged. - -### Example 1 - -As an example, suppose you want the student to calculate the mean of a Numpy array `arr` and store it in `res`. A possible solution could be: - - *** =solution - ```{python} - # Import numpy and create array - import numpy as np - arr = np.array([1, 2, 3, 4, 5, 6]) - - # Calculate result - result = np.mean(arr) - ``` - -The first part of the tests here would be to check `result`. If `result` is not correct, you want to check whether `np.mean()` has been called. - -The most concise way to do this, is with lambda functions; you specify two sets of tests, that in this case consist of one test each: - - *** =sct - ```{python} - test_correct(lambda: test_object('result'), - lambda: test_function('numpy.mean')) - success_msg("You own numpy!") - ``` - -Another way to do this, is by specifying two separate function definitions, e.g. `check` and `diagnose`, and pass these function definitions to `test_correct()`, as shown below. Notice that you have to pass the actual function definition, so `check` and `diagnose` without parentheses: - - *** =sct - ```{python} - def check(): - test_object('result') - - def diagnose(): - test_function('numpy.mean') - - test_correct(check, diagnose) - success_msg("You own numpy!") - ``` - -Of course, you can also use a lambda function for one test set, and a function definition for the other: - - *** =sct - ```{python} - def check(): - test_object('result') - - test_correct(check, - lambda: test_function('numpy.mean')) - success_msg("You own numpy!") - ``` - - -### Example 2 - -When writing SCTs for more complicated exercises, you'll probably want to pass along several tests to an argument. - -Suppose that you expect the student to create an object `result` once more, but this time it's the sum of calling the `np.mean()` function twice; once on `arr1`, once on `arr2`: - - *** =solution - ```{python} - # Import numpy and create array - import numpy as np - arr1 = np.array([1, 2, 3, 4, 5, 6]) - arr2 = np.array([6, 5, 4, 3, 2, 1]) - - # Calculate result - result = np.mean(arr) + np.mean(arr) - ``` - -Now, in the 'digging deeper' part of `test_correct()`, you want to check the `np.mean()` function twice. To do this, you'll want to use a function definition; lambda functions are not practical anymore: - - *** =sct - ```{python} - def diagnose(): - test_function('numpy.mean', index = 1) - test_function('numpy.mean', index = 2) - - test_correct(lambda: test_object('result'), - diagnose) - success_msg("You own numpy!") - ``` - -### Example 3: custom feedback - -To pass custom feedback feedback messages to these sub-SCTs, you should define the custom message inside the functions themselves, inside the function definition, or define it beforehand and pass it along as a default argument. Let's revisit the first example: - - *** =solution - ```{python} - # Import numpy and create array - import numpy as np - arr = np.array([1, 2, 3, 4, 5, 6]) - - # Calculate result - result = np.mean(arr) - ``` - -Here are the three different SCT options, this time with custom feedback: - - *** =sct - ```{python} - # OPTION 1 - msg1 = "Are you sure `result` is correct?" - msg2 = "Your call of `np.mean()` isn't right." - test_correct(lambda msg=msg1: test_object('result', incorrect_msg = msg), - lambda msg=msg2: test_function('numpy.mean', incorrect_msg = msg)) - success_msg("You own numpy!") - - # OPTION 2 - def check(): - test_object('result', incorrect_msg = "Are you sure `result` is correct?") - - def diagnose(): - test_function('numpy.mean', incorrect_msg = "Your call of `np.mean()` isn't right.") - - test_correct(check, diagnose) - success_msg("You own numpy!") - - - # OPTION 3 - def check(): - test_object('result', incorrect_msg = "Are you sure `result` is correct?") - - test_correct(check, - lambda: test_function('numpy.mean', incorrect_msg = "Your call of `np.mean()` isn't right.")) - success_msg("You own numpy!") - ``` diff --git a/docs/source/logic_tests/index.rst b/docs/source/logic_tests/index.rst new file mode 100644 index 00000000..6e757ad9 --- /dev/null +++ b/docs/source/logic_tests/index.rst @@ -0,0 +1,10 @@ +Logic Tests +=========== + +.. toctree:: + :maxdepth: 2 + + test_correct + test_or + test_not + misc diff --git a/docs/source/logic_tests/misc.md b/docs/source/logic_tests/misc.md new file mode 100644 index 00000000..78c7f839 --- /dev/null +++ b/docs/source/logic_tests/misc.md @@ -0,0 +1,61 @@ +Misc +---- + +[TODO: this is a previous version, will update with spec2 functions] + +Several functions, such as `test_correct()`, `test_or()`, `test_not()`, among others, also have arguments that expect another set of tests. This article will explain the different ways of specifying these 'sub-tests'. + +Let's take the example of `test_correct()`; this function takes two sets of tests. The first set is executed, and if it's fine, the second set is left alone. If the first set of tests results in an error, the second set is executed and the feedback is logged. + +### Example 1 + +As an example, suppose you want the student to calculate the mean of a Numpy array `arr` and store it in `res`. A possible solution could be: + + *** =solution + ```{python} + # Import numpy and create array + import numpy as np + arr = np.array([1, 2, 3, 4, 5, 6]) + + # Calculate result + result = np.mean(arr) + ``` + +The first part of the tests here would be to check `result`. If `result` is not correct, you want to check whether `np.mean()` has been called. + +The most concise way to do this, is with lambda functions; you specify two sets of tests, that in this case consist of one test each: + + *** =sct + ```{python} + test_correct(check_object('result').has_equal_value(), + test_function('numpy.mean')) + success_msg("You own numpy!") + ``` + +### Example 2 + +When writing SCTs for more complicated exercises, you'll probably want to pass along several tests to an argument. + +Suppose that you expect the student to create an object `result` once more, but this time it's the sum of calling the `np.mean()` function twice; once on `arr1`, once on `arr2`: + + *** =solution + ```{python} + # Import numpy and create array + import numpy as np + arr1 = np.array([1, 2, 3, 4, 5, 6]) + arr2 = np.array([6, 5, 4, 3, 2, 1]) + + # Calculate result + result = np.mean(arr) + np.mean(arr) + ``` + +Now, in the 'digging deeper' part of `test_correct()`, you want to check the `np.mean()` function twice. +To do this, you'll want to use a function definition; lambda functions are not practical anymore: + + *** =sct + ```{python} + diagnose = [test_function('numpy.mean', index = i) for i in [1,2]] + + test_correct(test_object('result'), diagnose) + success_msg("You own numpy!") + ``` diff --git a/docs/source/logic_tests/test_correct.md b/docs/source/logic_tests/test_correct.md new file mode 100644 index 00000000..ae1a2ac4 --- /dev/null +++ b/docs/source/logic_tests/test_correct.md @@ -0,0 +1,64 @@ +test_correct +------------ + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_correct + :members: +``` + +A wrapper function around `test_or()`, `test_correct()` allows you to add logic to your SCT. Normally, your SCT is simply a script with subsequent `pythonwhat` function calls, all of which have to pass. `test_correct()` allows you to bypass this: you can specify a "sub-SCT" in the `check` part, that should pass. If these tests pass, the "sub-SCT" in `diagnose` is not executed. If the tests don't pass, the "sub-SCT" in `diagnose` is run, typically to dive deeper into what the error might be and give more specific feedback. + +To accomplish this, the lambda function in `check` is executed silently, so that failure will not cause the SCT to stop and generate a feedback message. If the execution passes, all is good and `test_correct()` is abandoned. If it fails, `diagnose` is executed, not silently. If the `diagnose` part fails, the feedback message that it generates is presented to the student. If it passes, the `check` part is executed again, this time not silently, to make sure that a `test_correct()` that contains a failing `check` part leads to a failing SCT. + +### Example 1 + +As an example, suppose you want the student to calculate the mean of a Numpy array `arr` and store it in `res`. A possible solution could be: + + *** =solution + ```{python} + # Import numpy and create array + import numpy as np + arr = np.array([1, 2, 3, 4, 5, 6]) + + # Calculate result + result = np.mean(arr) + ``` + +You want the SCT to pass when the student manages to store the correct value in the object `result`. How `result` was calculated, does not matter to you: as long as `result` is correct, the SCT should accept the submission. If something about `result` is not correct, you want to dig a little deeper and see if the student used the `np.mean()` function correctly. The following SCT will do just that: + + *** =sct + ```{python} + test_correct(lambda: test_object('result'), + lambda: test_function('numpy.mean')) + success_msg("You own numpy!") + ``` + +Notice that you have to use lambda functions to use Python as a functional programming language. +Let's go over what happens when the student submits different pieces of code: + +- The student submits `result = np.mean(arr)`, exactly the same as the solution. `test_correct()` executes the first lambda function, `test_object('result')`. This test passes, so `test_correct()` is exited and `test_function()` is not executed. The SCT passes. +- The student submits `result = np.sum(arr) / arr.size`, which also leads to the correct value in `result`. `test_correct()` executes the first lambda function, `test_object('result')`. This test passes, so `test_correct()` is exited and `test_function()` is not executed. So the entire SCT passes even though `np.mean()` was not used. +- The student submits `result = np.mean(arr + 1)`. `test_correct()` executes the first lambda function, `test_object('result')`. This test fails, so `test_correct()` heads over to second, 'diagnose' lambda function and executes `test_function('numpy.mean')`. This function will fail, because the argument passed to `numpy.mean()` in the student submission does not correspond to the argument passed in the solution. A meaningful, specific feedback message is presented to the student: you did not correctly specify the arguments inside `np.mean()`. +- The student submits `result = np.mean(arr) + 1`. `test_correct()` executes the first lambda function, `test_object('result')`. This test fails, so `test_correct()` heads over to the second, 'diagnose' lambda function and executes `test_function('numpy.mean'). This function passes, because `np.mean()` is called in exactly the same way in the student code as in the solution. Because there is something wrong - `result` is not correct - the 'check' lambda function, `test_object('result')` is executed again, and this time its feedback on failure is presented to the student. The student gets the message that `result` does not contain the correct value. + +### Multiple functions in `diagnose` and `check` + +You can also use `test_correct()` with entire 'sub-SCTs' that are composed of several SCT calls. In this case, you have to define an additional function that executes the tests you want to perform in this sub-SCT, and pass this function to `test_correct()`. + +### Why to use `test_correct()` + +You will find that `test_correct()` is an extremely powerful function to allow for different ways of solving the same problem. You can use `test_correct()` to check the end result of a calculation. If the end result is correct, you can go ahead and accept the entire exercise. If the end result is incorrect, you can use the `diagnose` part of `test_correct()` to dig a little deeper. + +It is also perfectly possible to use `test_correct()` inside another `test_correct()`, although things can get funky with the lambda functions in this case. + +### Wrapper around `test_or()` + +`test_correct()` is a wrapper around `test_or()`. `test_correct(diagnose, check)` is equivalent with: + + def diagnose_and_check() + diagnose() + check() + + test_or(diagnose_and_check, check) + +Note that in each of the `test_or` cases here, the submission has to pass the SCTs specified in `check`. diff --git a/docs/source/logic_tests/test_not.md b/docs/source/logic_tests/test_not.md new file mode 100644 index 00000000..9a1c2fcf --- /dev/null +++ b/docs/source/logic_tests/test_not.md @@ -0,0 +1,2 @@ +test_not +-------- diff --git a/docs/source/logic_tests/test_or.md b/docs/source/logic_tests/test_or.md new file mode 100644 index 00000000..7763a28c --- /dev/null +++ b/docs/source/logic_tests/test_or.md @@ -0,0 +1,28 @@ +test_or +------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_or + :members: +``` + +This function simply tests whether one of the SCTs you specify inside it passes. You should pass your SCTs within lambda expressions or custom defined functions (like you do for functions like `test_correct()`, `test_for_loop()`, or `test_function_definition`). + +Suppose you want to check whether people correctly printed out any integer between 3 and 7. A solution could be: + + *** =solution + ```{python} + print(4) + ``` + +To test this in a robust way, you could use `test_output_contains()` with a suitable regular expression that covers everything, or you can use `test_or()` with three separate `test_output_contains()` functions. + + *** =sct + ```{python} + test_or(lambda: test_output_contains('4'), + lambda: test_output_contains('5'), + lambda: test_output_contains('6')) + success_msg("Nice job!") + ``` + +You can consider `test_or()` a logic-inducing function. The different calls to `pythonwhat` functions that are in your SCT are actually all tests that _have_ to pass: they are `AND` tests. With `test_or()` you can add chunks of `OR` tests in there. diff --git a/docs/source/part_checks.rst b/docs/source/part_checks.rst new file mode 100644 index 00000000..ce1568ba --- /dev/null +++ b/docs/source/part_checks.rst @@ -0,0 +1,446 @@ +Part Checks +================ + +.. role:: python(code) + :language: python + +Check Syntax +-------------- + +In Brief +~~~~~~~~ + +While functions beginning with ``test_``, such as ``test_student_typed`` look over some code or output, ``check_`` functions allow us to zoom in on parts of that student and solution code. + +For example, [check_list_comp](#check_list_comp) examines list comprehensions by breaking them into 3 parts: ``body``, ``comp_iter``, and ``ifs``. This is shown below. + + +:code:`[i*2 for i in range(10) if i>2]` => :code:`[BODY for i in COMP_ITER if IFS]` + +Each of these 3 parts may be tested individually using the simple test functions. +For example, in order to test the body of the comprehension above, we could create the following exercise. + +.. code:: python + + *** =solution + ```{python} + L2 = [i**2 for i in range(0,10) if i>2] + ``` + + *** =sct + ```{python} + (Ex().check_list_comp(1) # focus on first list comp + .check_body().test_student_typed('i\*2') # focus on its body for test + ) + ``` + +In the SCT, ``check_list_comp`` gets the first comprehension, and will fail with feedback if no comprehensions were used in th submission code. ``check_body`` gets ``i**2`` in the solution code, and whatever corresponds to BODY in the submission code. + +(Note: the parentheses around the entire statement are just syntactic sugar, to let us chain commands in python without using ``\`` at the end of each line.) + +Full Example +~~~~~~~~~~~~ + +This section expands the above example to run tests on each part: body, iter, and ifs. + +.. code-block:: python + + *** =solution + ```{python} + L2 = [i*2 for i in range(0,10) if i>2] + ``` + + *** =sct + ```{python} + list_comp = Ex().check_list_comp(1, missing_msg="Did you include a list comprehension?") + list_comp.check_body().test_student_typed('i\*2') + list_comp.check_iter().has_equal_value() + list_comp.check_ifs(1).multi([has_equal_value(context_vals=[i]) for i in range(0,10)]) + ``` + +In this SCT, the first line focuses on the first list comprehension, and assigns it to ``list_comp``, so we can test each part in turn. As a reminder, the code corresponding to each part in the solution code is.. + +* BODY: ``i**2`` +* COMP_ITER: ``range(0,10)`` +* IFS: [``i>2``] + +Note that IFS is represented as a list, and the index 1 was passed to `check_ifs` because a list comprehension may have multiple if statements. Since the test on BODY, is explained in the [In Brief section](#In_Brief), we will focus on the tests on ITER and and IFS. + +check_iter +^^^^^^^^^^^^^^ + +In the line ``list_comp.check_iter().equal_value()``, ``check_iter`` gets the ITER part in the solution and submission code, while ``has_equal_value`` tells pythonwhat to run those parts and see if they return equal values. Below are example solution and submission codes, with the ITER part they would produce + +================ ============================================ ==================== + type code ITER part +================ ============================================ ==================== + **solution** :python:`[i*2 for i in range(0,10) if i>2]` :code:`range(0,10)` +**submission** :python:`[i*2 for i in range(10) if i>2]` :code:`range(10)` +================ ============================================ ==================== + +In this case, ``equal_value`` will run each part, and then confirm that :python:`range(0,10) == range(10)`. For more on functions that run code, like ``has_equal_value`` see [Expressions Tests](processes). + +check_ifs +^^^^^^^^^^^^^ + +The line + +.. code-block:: python + + list_comp.check_ifs(1).multi([has_equal_value(context_vals=[i] for i in range(0,10))]) + +is a doozy, but can be broken down into + +.. code-block:: python + + equal_tests = [has_equal_value(context_vals=[i] for i in range(0,10))] # collection of has_equal_tests + list_comp.check_ifs(1).multi(equal_tests) # focus on IFS run equal_tests` + +In this case ``equal_tests`` is a list of ``has_equal_value`` tests that we'll want to perform. ``check_ifs(1)`` grabs the first IFS part, and ``multi(equal_tests)`` runs each ``has_equal_value`` test on that part. + +Notice that ``has_equal_value`` was given a context_val argument. This is because the list comprehension creates a temporary variable that needs to be defined when we run the IFS code. + +================ ============================================== ================ =============== + type code IFS part context value +================ ============================================== ================ =============== + **solution** :python:`[i*2 for i in range(0,10) if i>2]` :python:`if i>2` ``i`` + **submission** :python:`[j*2 for j in range(0,10) if j>2]` :python:`if j>2` ``j`` +================ ============================================== ================ =============== + +In this case, the context_vals argument is a list of values, with one for each (in this case only a single) context value. In this way, ``has_equal_value`` assigns ``i`` and ``j`` to the same value, before running the IFs part. By creating a list of ``has_equal_tests`` with context vals spanning ``range(0,10)``, we test the IFS across a range of values. + +Nested Part Example +~~~~~~~~~~~~~~~~~~~~ + +Check functions may be combined to focus on parts within parts, such as + +.. code:: python + + *** =solution + ```{python} + [i*2 if i> 5 else 0 for i in range(0,10)] + ``` + +In this case, a representation with the parts in caps and wrapping the inline if expression with ``{BODY=...}`` is + +.. code:: + + [{BODY=BODY if TEST else ORELSE} for i in ITER] + +in order to test running the inline if expression we could go from list_comp => body => if_exp. One possible SCT is shown below. + +.. code:: python + + *** =sct + ```{python} + (Ex().check_list_comp(1) # first comprehension + .check_body().set_context(i=6) # comp's body + .check_if_exp(1).has_equal_value() # body's inline IFS + ) + ``` + +Note that rather than using the ``context_vals`` argument of ``has_equal_value`` we use ``set_context`` to define the context variable (``i`` in the solution code) on the body of the list comprehension. This makes it very clear when the context value was introduced. It is worth pointing out that of the parts a list comprehension has, BODY and IFS, but not ITER have ``i`` as a context value. This is because in python ``i`` is undefined in the ITER part. Context values are listed in the [see cheatsheet below]. + +Testing only the body of the list comprehension +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If we left out the ``check_if_exp`` above, the resulting SCT, + +.. code:: python + + (Ex().check_list_comp(1).check_body().set_context(i=6) + #.check_if_exp(1) + .has_equal_value() + ) + +would still run the same code for the solution (the inline if expression), since it's the only thing in the BODY of the list comprehension. However it wouldn't check if an if expression was used, allowing a wider range of passing and failing submissions (for better or worse!). Moreover, `has_equal_value` may be used multiple times during the chaining, as it doesn't change what the focus is. + +Helper Functions +---------------- + +multi +~~~~~~~ + +Runs multiple subtests. + + +Comma separated arguments +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For example, this code without multi, + +.. code:: + + Ex().check_if_exp(0).check_body().has_equal_value() + Ex().check_if_exp(0).check_test().has_equal_value() + + +is equivalent to + +.. code:: + + Ex().check_if_exp(0).multi( + check_body().has_equal_value(), + check_test().has_equal_value() + ) + +List or generator of subtests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Rather than one or more subtest args, multi can take a single list or generator of subtests. +For example, the code below checks that the body of a list comprehension has equal value +for 10 possible values of the iterator variable, ``i``. + +.. code:: + + Ex().check_list_comp(0) + .check_body() + .multi(set_context(i=x).has_equal_value() for x in range(10)) + +Chaining off multi +^^^^^^^^^^^^^^^^^^^ + +Multi returns the same state, or focus, it was given, so whatever comes after multi will run +the same as if multi wasn't used. For example, the code below tests a list comprehension's body, +followed by its iterator. + +.. code:: + + Ex().check_list_comp(0) \ + .multi(check_body().has_equal_value()) \ + .check_iter().has_equal_value() + + +set_context +~~~~~~~~~~~~~ + +Sets the value of a temporary variable, such as ``ii`` in the list comprehension below. + +.. code:: + + [ii + 1 for ii in range(3)] + +Variable names may be specified using positional or keyword arguments. + +Example +^^^^^^^^ + +**Solution Code** + +.. code:: + + ltrs = ['a', 'b'] + for ii, ltr in enumerate(ltrs): + print(ii) + +**SCT** + +.. code:: + + Ex().check_for_loop(0).check_body() \ + .set_context(ii=0, ltr='a').has_equal_output() \ + .set_context(ii=1, ltr='b').has_equal_output() + +Note that if a student replaced `ii` with `jj` in their submission, `set_context` would still work. +It uses the solution code as a reference. While we specified the target variables ``ii`` and ``ltr`` +by name in the SCT above, they may also be given by position.. + +.. code:: + + Ex().check_for_loop(0).check_body().set_context(0, 'a').has_equal_output() + +Instructor Errors +^^^^^^^^^^^^^^^^^^^ + +If you are unsure what variables can be set, it's often easiest to take a guess. +When you try to set context values that don't match any target variables in the solution code, +``set_context`` raises an exception that lists the ones available. + + + +with_context +~~~~~~~~~~~~~~ + +Runs subtests after setting the context for a ``with`` statement. + +This function takes arguments in the same form as ``multi``. +Note also that ``with_context`` was the default behavior for ``test_with`` in pythonwhat version 1. + +Context Managers Explained +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +With statements are special in python in that they enter objects called a context manager at the beginning of the block, +and exit them at the end. For example, the object returned by ``open('fname.txt')`` below is a context manager. + +.. code:: + + with open('fname.txt') as f: + print(f.read()) + +This code runs by + +1. assigning ``f`` to the context manager returned by ``open('fname.txt')`` +2. calling ``f.__enter__()`` +3. running the block +4. calling ``f.__exit__()`` + +``with_context`` was designed to emulate this sequence of events, by setting up context values as in step (1), +and replacing step (3) with any sub-tests given as arguments. + + +fail +~~~~~~ + +Fails. This function takes a single argument, ``msg``, that is the feedback given to the student. +Note that this would be a terrible idea for grading submissions, but may be useful while writing SCTs. +For example, failing a test will highlight the code as if the previous test/check had failed. + +As a trivial SCT example, + +.. code:: + + Ex().check_for_loop(0).check_body().fail() # fails boo + +This can also be helpful for debugging SCTs, as it can be used to stop testing as a given point. + +Check Functions +---------------- + +**Arguments** + +* **index**: index or key corresponding to the node or part of interest. + This applies to all functions in the **check** column in the table below. + However, apart from that, it only applies when there is more than one of a specific part to choose from --- + ``check_ifs``, ``check_args``, ``check_handlers``, and ``check_context``. + (e.g. ``Ex().check_list_comp(0).check_ifs(0)``) +* **missing_msg**: optional feedback message if node or part doesn't exist. + + +Note that code in all caps indicates the name of a piece of code that may be inspected using, ``check_{part}``, +where ``{part}`` is replaced by the name in caps (e.g. ``check_if_else(0).check_test()``). +Target variables are those that may be set using ``set_context``. +These variables may only be set in places where python would set them. +For example, this means that a list comprehension's ITER part has no target variables, +but its BODY does. + ++------------------------+------------------------------------------------------+-------------------+ +| check | parts | target variables | ++========================+======================================================+===================+ +|check_if_else | .. code:: | | +| | | | +| | if TEST: | | +| | BODY | | +| | else: | | +| | ORELSE | | +| | | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_while | .. code:: python | | +| | | | +| | while TEST: | | +| | BODY | | +| | else: | | +| | ORELSE | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_list_comp | .. code:: | ``i`` | +| | | | +| | [BODY for i in ITER if IFS[0] if IFS[1]] | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_generator_exp | .. code:: | ``i`` | +| | | | +| | (BODY for i in ITER if IFS[0] if IFS[1]) | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_dict_comp | .. code:: | ``k``, ``v`` | +| | | | +| | {KEY : VALUE for k, v in ITER if IFS[0]} | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_for_loop | .. code:: | ``i`` | +| | | | +| | for i in ITER: | | +| | BODY | | +| | else: | | +| | ORELSE | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_try_except | .. code:: python | ``e`` | +| | | | +| | try: | | +| | BODY | | +| | except BaseException as e: | | +| | HANDLERS['BaseException'] | | +| | except: | | +| | HANDLERS['all'] | | +| | else: | | +| | ORELSE | | +| | finally: | | +| | FINALBODY | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_with | .. code:: python | `f`` | +| | | | +| | with CONTEXT_TEST as f: | | +| | BODY | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_function_def | .. code:: python | argument names | +| | | | +| | def f(ARGS[0], ARGS[1]): | | +| | BODY | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ +|check_lambda | .. code:: | argument names | +| | | | +| | lambda ARGS[0], ARGS[1]: BODY | | +| | | | +| | | | ++------------------------+------------------------------------------------------+-------------------+ + +More +------ + +elif statements +~~~~~~~~~~~~~~~~ + +In python, when an if-else statement has an elif clause, it is held in the ORELSE part, + +.. code:: python + + if TEST: + BODY + ORELSE # elif and else portion + +In this sense, an if-elif-else statement is represented by python as nested if-elses. For example, the final ``else`` below + +.. code:: python + + if x: print(x) # line 1 + elif y: print(y) # "" 2 + else: print('none') # "" 3 + +can be checked with the following SCT + +.. code:: python + + (Ex().check_if_else(1) # lines 1-3 + .check_orelse().check_if_else(1) # lines 2-3 + .check_orelse().has_equal_output() # line 3 + ) + + +function definition / lambda args +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + + def f(a, b): + BODY + +*args and **kwargs +^^^^^^^^^^^^^^^^^^ + +testing default values +^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/parts_cheatsheet.rst b/docs/source/parts_cheatsheet.rst deleted file mode 100644 index 072af1fb..00000000 --- a/docs/source/parts_cheatsheet.rst +++ /dev/null @@ -1,318 +0,0 @@ -Part Checks -================ - -.. role:: python(code) - :language: python - -Check Syntax --------------- - -In Brief -~~~~~~~~ - -While functions beginning with ``test_``, such as ``test_student_typed`` look over some code or output, ``check_`` functions allow us to zoom in on parts of that student and solution code. - -For example, [check_list_comp](#check_list_comp) examines list comprehensions by breaking them into 3 parts: ``body``, ``comp_iter``, and ``ifs``. This is shown below. - - -:code:`[i*2 for i in range(10) if i>2]` => :code:`[BODY for i in COMP_ITER if IFS]` - -Each of these 3 parts may be tested individually using the simple test functions. -For example, in order to test the body of the comprehension above, we could create the following exercise. - -.. code:: python - - *** =solution - ```{python} - L2 = [i**2 for i in range(0,10) if i>2] - ``` - - *** =sct - ```{python} - (Ex().check_list_comp(1) # focus on first list comp - .check_body().test_student_typed('i\*2') # focus on its body for test - ) - ``` - -In the SCT, ``check_list_comp`` gets the first comprehension, and will fail with feedback if no comprehensions were used in th submission code. ``check_body`` gets ``i**2`` in the solution code, and whatever corresponds to BODY in the submission code. - -(Note: the parentheses around the entire statement are just syntactic sugar, to let us chain commands in python without using ``\`` at the end of each line.) - -Full Example -~~~~~~~~~~~~ - -This section expands the above example to run tests on each part: body, iter, and ifs. - -.. code-block:: python - - *** =solution - ```{python} - L2 = [i*2 for i in range(0,10) if i>2] - ``` - - *** =sct - ```{python} - list_comp = Ex().check_list_comp(1, missing_msg="Did you include a list comprehension?") - list_comp.check_body().test_student_typed('i\*2') - list_comp.check_iter().has_equal_value() - list_comp.check_ifs(1).multi([has_equal_value(context_vals=[i]) for i in range(0,10)]) - ``` - -In this SCT, the first line focuses on the first list comprehension, and assigns it to ``list_comp``, so we can test each part in turn. As a reminder, the code corresponding to each part in the solution code is.. - -* BODY: ``i**2`` -* COMP_ITER: ``range(0,10)`` -* IFS: [``i>2``] - -Note that IFS is represented as a list, and the index 1 was passed to `check_ifs` because a list comprehension may have multiple if statements. Since the test on BODY, is explained in the [In Brief section](#In_Brief), we will focus on the tests on ITER and and IFS. - -``check_iter`` -^^^^^^^^^^^^^^ - -In the line ``list_comp.check_iter().equal_value()``, ``check_iter`` gets the ITER part in the solution and submission code, while ``has_equal_value`` tells pythonwhat to run those parts and see if they return equal values. Below are example solution and submission codes, with the ITER part they would produce - -================ ============================================ ==================== - type code ITER part -================ ============================================ ==================== - **solution** :python:`[i*2 for i in range(0,10) if i>2]` :code:`range(0,10)` -**submission** :python:`[i*2 for i in range(10) if i>2]` :code:`range(10)` -================ ============================================ ==================== - -In this case, `equal_value` will run each part, and then confirm that `range(0,10) == range(10)`. For more on functions that run code, like `has_equal_value` see [Expressions Tests](processes). - -``check_ifs`` -^^^^^^^^^^^^^ - -The line - -.. code-block:: python - - list_comp.check_ifs(1).multi([has_equal_value(context_vals=[i] for i in range(0,10))]) - -is a doozy, but can be broken down into - -.. code-block:: python - - equal_tests = [has_equal_value(context_vals=[i] for i in range(0,10))] # collection of has_equal_tests - list_comp.check_ifs(1).multi(equal_tests) # focus on IFS run equal_tests` - -In this case ``equal_tests`` is a list of ``has_equal_value`` tests that we'll want to perform. ``check_ifs(1)`` grabs the first IFS part, and ``multi(equal_tests)`` runs each ``has_equal_value`` test on that part. - -Notice that `has_equal_value` was given a context_val argument. This is because the list comprehension creates a temporary variable that needs to be defined when we run the IFS code. - -================ ============================================== ================ =============== - type code IFS part context value -================ ============================================== ================ =============== - **solution** :python:`[i*2 for i in range(0,10) if i>2]` :python:`if i>2` ``i`` - **submission** :python:`[j*2 for j in range(0,10) if j>2]` :python:`if j>2` ``j`` -================ ============================================== ================ =============== - -In this case, the context_vals argument is a list of values, with one for each (in this case only a single) context value. In this way, ``has_equal_value`` assigns ``i`` and ``j`` to the same value, before running the IFs part. By creating a list of ``has_equal_tests`` with context vals spanning ``range(0,10)``, we test the IFS across a range of values. - -Nested Part Example -~~~~~~~~~~~~~~~~~~~~ - -Check functions may be combined to focus on parts within parts, such as - -.. code:: python - - *** =solution - ```{python} - [i*2 if i> 5 else 0 for i in range(0,10)] - ``` - -In this case, a representation with the parts in caps and wrapping the inline if expression with ``{BODY=...}`` is - -.. code:: - - [{BODY=BODY if TEST else ORELSE} for i in ITER] - -in order to test running the inline if expression we could go from list_comp => body => if_exp. One possible SCT is shown below. - -.. code:: python - - *** =sct - ```{python} - (Ex().check_list_comp(1) # first comprehension - .check_body() # comp's body - .set_context(i=6) - .check_if_exp(1) - .has_equal_value() # body's inline if - ) - ``` - -Note that rather than using the ``context_vals`` argument of ``has_equal_value`` we use the ``set_context`` to define the context variable (``i`` in the solution code) on the body of the list comprehension. This makes it very clear when the context value was introduced. It is worth pointing out that of the parts a list comprehension has, BODY and IFS, but not ITER have ``i`` as a context value in pythonwhat (since in python ``i`` is undefined in the ITER part). Context values are listed in the [see cheatsheet below]. - -Testing only the body of the list comprehension -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If we left out the ``check_if_exp`` above, the resulting SCT, - -.. code:: python - - (Ex().check_list_comp(1).check_body().set_context(i=6) - #.check_if_exp(1) - .has_equal_value() - ) - -would still run the same code for the solution (the inline if expression), since it's the only thing in the BODY of the list comprehension. However it wouldn't check if an if expression was used, allowing a wider range of passing and failing submissions (for better or worse!). Moreover, `has_equal_value` may be used multiple times during the chaining, as it doesn't change what the focus is. - -Table of Example Checks on Nested Parts ----------------------------------------- - -+------------------------+------------------------------+-------------------------------------+ -| description | solution code | sct | -+========================+==============================+=====================================+ -|nested ifs (i.e. elifs) | .. code:: - -+------------------------+ - - -Helper Functions ----------------- - -`multi` -~~~~~~~ - -`set_context` -~~~~~~~~~~~~~ - -`with_context` -~~~~~~~~~~~~~~ - -Cheatsheet ----------- - -differences between spec1 and spec2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -* comp_iter is the argument name for spec1 (e.g. test_list_comp) was renamed to iter (e.g. check_list_comp(1).check_iter) -* `test_function_definition` was shortened to `check_function_def`. - -check_list_comp -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - [BODY for i in ITER if IFS[0] if IFS[1]] - - -check_dict_comp -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - { KEY : VALUE for k, v in ITER if IFS[0] if IFS[1] } - -check_generator_exp -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: python - - (BODY for i in ITER if IFS[0] if IFS[1]) - - -check_for_loop -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: python - - for i in ITER: - BODY - else: - ORELSE - -_yes, you can put an else statement at the end!_ - -check_if_else -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - if TEST: - BODY - else: - ORELSE - - -or, in the case of elif statements... - -.. code:: python - - if TEST: - BODY - ORELSE - -Nested Examples -^^^^^^^^^^^^^^^^ - -In this sense, an if-elif-else statement is represented by python as nested if-elses. For example, the final ``else`` below - -.. code:: python - - if x: print(x) # line 1 - elif y: print(y) # "" 2 - else: print('none') # "" 3 - -can be checked with the following SCT - -.. code:: python - - (Ex().check_if_else(1) # lines 1-3 - .check_orelse().check_if_else(1) # lines 2-3 - .check_orelse().has_equal_output() # line 3 - ) - - -check_lambda -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - lambda x: BODY - -check_try_except -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - try: - BODY - except BaseException: - HANDLERS['BaseException'] - except: - HANDLERS['all'] - else: - ORELSE - finally: - FINALBODY - - -check_while -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - while TEST: - BODY - else: - ORELSE - - -check_with -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - with CONTEXT_TEST as context_var: - BODY - - -check_function_def -~~~~~~~~~~~~~~~~~~ - -.. code:: python - - def f(a, b): - BODY - diff --git a/docs/source/pythonwhat.wiki/index.rst b/docs/source/pythonwhat.wiki/index.rst new file mode 100644 index 00000000..bdff85ed --- /dev/null +++ b/docs/source/pythonwhat.wiki/index.rst @@ -0,0 +1,29 @@ +Legacy Tests +============ + +The functions below combine both part checks and simple tests from pythonwhat v1. +In some cases, they allow very specific checks that are not yet exposed to SCT creators in v2 +(such as whether iterator variables have the exact same names). + +.. toctree:: + :maxdepth: 2 + + test_data_frame + test_dictionary + test_operator + test_expression_output + test_expression_result + test_object_after_expression + test_object_accessed + parts_cheatsheet + test_comprehension + test_for_loop + test_if_else + test_if_exp + test_try_except + test_while_loop + test_with + test_function_definition + test_function + test_function_v2.md + test_lambda_function diff --git a/docs/source/pythonwhat.wiki/parts_cheatsheet.md b/docs/source/pythonwhat.wiki/parts_cheatsheet.md new file mode 100644 index 00000000..c79a0537 --- /dev/null +++ b/docs/source/pythonwhat.wiki/parts_cheatsheet.md @@ -0,0 +1,89 @@ +parts_cheatsheet +---------------- + +### test_list_comp + +```{python} +[BODY for i in COMP_ITER if IFS[0] if IFS[1]] +``` + +### test_dict_comp + +```{python} +{ KEY : VALUE for k, v in COMP_ITER if IFS[0] if IFS[1] } +``` +### test_generator_exp + +```{python} +(BODY for i in COMP_ITER if IFS[0] if IFS[1]) +``` + +### test_for_loop +```{python} +for i in FOR_ITER: + BODY +else: + ORELSE +``` + +_yes, you can put an else statement at the end!_ + +### test_if_else + +```{python} +if TEST: + BODY +else: + ORELSE +``` + +or, in the case of elif statements... + +```{python} +if TEST: + BODY +ORELSE +``` + +### test_lambda +```{python} +lambda x: BODY +``` + +### test_try_except + +```{python} +try: + BODY +except BaseException: + HANDLERS['BaseException'] +except: + HANDLERS['all'] +else: + ORELSE +finally: + FINALBODY +``` + +### test_while + +```{python} +while TEST: + BODY +else: + ORELSE +``` + +### test_with + +```{python} +with CONTEXT_TEST as context_var: + BODY +``` + +### test_function_definition + +```{python} +def f(a, b): + BODY +``` diff --git a/docs/source/pythonwhat.wiki/test_comprehension.md b/docs/source/pythonwhat.wiki/test_comprehension.md new file mode 100644 index 00000000..0c26f37a --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_comprehension.md @@ -0,0 +1,201 @@ +test comprehensions +------------------- + + def test_list_comp(index=1, + not_called_msg=None, + comp_iter=None, + iter_vars_names=False, + incorrect_iter_vars_msg=None, + body=None, + ifs=None, + insufficient_ifs_msg=None, + expand_message=True) + + def test_generator_exp(index=1, + not_called_msg=None, + comp_iter=None, + iter_vars_names=False, + incorrect_iter_vars_msg=None, + body=None, + ifs=None, + insufficient_ifs_msg=None, + expand_message=True) + + def test_dict_comp(index=1, + not_called_msg=None, + comp_iter=None, + iter_vars_names=False, + incorrect_iter_vars_msg=None, + key=None, + value=None, + ifs=None, + insufficient_ifs_msg=None, + expand_message=True) + +Currently, functionality to test list comprehensions, generator expressions and dictionary comprehensions is implemented. If you look at the signatures, you'll see that the arguments for `test_list_comp()` and `test_generator_exp()` are identical. Syntactically, there is close to no difference between list comprehensions and generator expressions, so all tests and settings apply for both cases. For `test_dict_comp()` there's only a small difference: the arguments `key` and `value` instead of the `body` argument, so that you can test the `key` part of the dictionary comprehension seperately from the `value` comprehension. + +The above functions work pretty similarly to `test_for_loop()`, with some additions and customizations here and there. Let's go over the argments: + +- `index`: the number of the comprehension in the submission to test. (this is specific to each comprehension, if there's one list and one dict comprehension you need `index=1` twice.) +- `not_called_msg`: Custom message in case the comprehension was not coded (or there weren't enough comprehensions). +- `comp_iter`: sub SCT to check the sequence part of the comprehension. Specify this through another function definition or a lambda function. +- `iter_vars_names`: whether or not the iterator variables should match the ones in the solution. +- `incorrect_iter_vars_msg`: Custom message in case the iterator variables don't match the solution (if `iter_vars_names` is `True`) or if the number of iterator variables doesn't correspond to the solution. +- `body`, `key`, `value`: sub SCTs to check the body part (for list comps and generator expressions) or the key and value part of a dictionary comprehension. +- `ifs`: list of sub-SCTs to check each of the ifs specified inside the comprehension. If you specify `ifs`, make sure that the number of sub-SCTs corresponds exactly to the number of ifs that are in the solution. +- `insufficient_ifs`: custom message in case the student coded less ifs than the corresponding comp in the solution. +- `expand_message`: whether or not to expand feedback messages from sub-SCTs with more information about where in the list comprehension they occur. + +### Example 1: List comprehension + +Suppose you want the student to code a list comprehension like below: + + *** =solution + ```{python} + x = {'a': 2, 'b':3, 'c':4, 'd':'test'} + [key + str(val) for key,val in x.items() if isinstance(key, str) if isinstance(val, int)] + ``` + +The following SCT will test several parts of this list comprehension, and relies on automatic feedback messages everywhere: + + *** =sct + ```{python} + test_list_comp(index=1, + comp_iter=lambda: test_expression_result(), + iter_vars_names=True, + body=lambda: test_expression_result(context_vals = ['a', 2]), + ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False]), + lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False])]) + ``` + +By setting `iter_vars_names` to `True`, `pythonwhat` will check that the student actually used the iterator variables `key` and `val`. Notice that in the sub SCT for the body, `context_vals` are used to set the `key` and `val` iterator variables before the expression is tested. This is similar to how things work in `test_for_loop()`. Notice also that inside the list of if sub-SCTs, `do_eval` is false, because the values `key` and `val` are not available there (setting context vals is currently only possible inside `test_expression_*()` functions). + +`test_list_comp()` will generate a bunch of meaningful automated messages depending on which error the student made: + + submission: + feedback: "The system wants to check the first list comprehension you defined but hasn't found it." + + submission: [key for key in x.keys()] + feedback: "Check your code in the iterable part of the first list comprehension. Unexpected expression: expected `dict_items([('a', 2), ('b', 3), ('c', 4), ('d', 'test')])`, got `dict_keys(['a', 'b', 'c', 'd'])` with values." + + submission: [a + str(b) for a,b in x.items()] + feedback: "Have you used the correct iterator variables in the first list comprehension? Make sure you use the correct names!" + + submission: [key + '_' + str(val) for key,val in x.items()] + feedback: "Check your code in the body of the first list comprehension. Unexpected expression: expected `a2`, got `a_2` with values." + + submission: [key + str(val) for key,val in x.items()] + feedback: "Have you used 2 ifs inside the first list comprehension?" + + submission: [key + str(val) for key,val in x.items() if hasattr(key, 'test') if hasattr(key, 'test')] + feedback: "Check your code in the first if of the first list comprehension. Have you called `isinstance()`?" + + submission: [key + str(val) for key,val in x.items() if isinstance(key, str) if hasattr(key, 'test')] + feedback: "Check your code in the second if of the first list comprehension. Have you called `isinstance()`?" + + submission: [key + str(val) for key,val in x.items() if isinstance(key, str) if isinstance(key, str)] + feedback: "Check your code in the second if of the first list comprehension. Did you call `isinstance()` with the correct arguments?" + + submission: [key + str(val) for key,val in x.items() if isinstance(key, str) if isinstance(val, str)] + feedback: "Great work!" + +NOTE: the "check your code in the ... of the first list comprehension" parts are included because `expand_message = True`. + +You can also update SCT to override all automatically generated messages, either inside `test_list_comp()` itself or inside the sub-SCTs: + + *** =sct + ```{python} + test_list_comp(index=1, + not_called_msg='notcalled', + comp_iter=lambda: test_expression_result(incorrect_msg = 'iterincorrect'), + iter_vars_names=True, + incorrect_iter_vars_msg='incorrectitervars', + body=lambda: test_expression_result(context_vals = ['a', 2], incorrect_msg = 'bodyincorrect'), + ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], not_called_msg = 'notcalled1', incorrect_msg = 'incorrect2'), + lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False], not_called_msg = 'notcalled2', incorrect_msg = 'incorrect2')], + insufficient_ifs_msg='insufficientifs') + ``` + +In this case, you get the following feedback for different submissions: + + submission: + feedback: "notcalled" + + submission: [key for key in x.keys()] + feedback: "Check your code in the iterable part of the first list comprehension. iterincorrect" + + submission: [a + str(b) for a,b in x.items()] + feedback: "incorrectitervars" + + submission: [key + '_' + str(val) for key,val in x.items()] + feedback: "Check your code in the body of the first list comprehension. bodyincorrect" + + submission: [key + str(val) for key,val in x.items()] + feedback: "insufficientifs" + + submission: [key + str(val) for key,val in x.items() if hasattr(key, 'test') if hasattr(key, 'test')] + feedback: "Check your code in the first if of the first list comprehension. notcalled1" + + submission: [key + str(val) for key,val in x.items() if isinstance(key, str) if hasattr(key, 'test')] + feedback: "Check your code in the second if of the first list comprehension. notcalled2" + + submission: [key + str(val) for key,val in x.items() if isinstance(key, str) if isinstance(key, str)] + feedback: "Check your code in the second if of the first list comprehension. incorrect2" + + submission: [key + str(val) for key,val in x.items() if isinstance(key, str) if isinstance(val, str)] + feedback: "Great work!" + + +### Example 2: Generator Expressions + +An example here won't be necessary, because it works the exact same way as in Example 1, with the only difference that in automated feedback, "list comprehension" is replaced with "generator expression". + +### Example 3: Dictionary Comprehensions + +Suppose you want the student to code a dictionary comprehension like below: + + *** =solution + ```{python} + x = {'a': 2, 'b':3, 'c':4, 'd':'test'} + [key + str(val) for key,val in x.items() if isinstance(key, str) if isinstance(val, int)] + ``` + +The following SCT will test several parts of this list comprehension, and relies on automatic feedback messages everywhere: + + *** =sct + ```{python} + test_list_comp(index=1, + comp_iter=lambda: test_expression_result(), + iter_vars_names=True, + body=lambda: test_expression_result(context_vals = ['a', 2]), + ifs=[lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False]), + lambda: test_function_v2('isinstance', params = ['obj'], do_eval = [False])]) + ``` + +Again, customized messages are generated for different cases: + + submission: + feedback: "The system wants to check the first dictionary comprehension you defined but hasn't found it." + + submission: { a:a for a in lst[1:2] } + feedback: "Check your code in the iterable part of the first dictionary comprehension. Unexpected expression: expected `['this', 'is', 'a', 'list']`, got `['is']` with values." + + submission: { a:a for a in lst } + feedback: "Have you used the correct iterator variables in the first dictionary comprehension? Make sure you use the correct names!" + + submission: { el + 'a':str(el) for el in lst } + feedback: "Check your code in the key part of the first dictionary comprehension. Unexpected expression: expected `a`, got `aa` with values." + + submission: { el:str(el) for el in lst } + feedback: "Check your code in the value part of the first dictionary comprehension. Unexpected expression: expected `1`, got `a` with values." + + submission: { el:len(el) for el in lst } + feedback: "Have you used 1 ifs inside the first dictionary comprehension?" + + submission: { el:len(el) for el in lst if isinstance('a', str)} + feedback: "Check your code in the first if of the first dictionary comprehension. Did you call `isinstance()` with the correct arguments?" + + submission: { el:len(el) for el in lst if isinstance(el, str)} + feedback: "Great work!" + + diff --git a/docs/source/pythonwhat.wiki/test_data_frame.md b/docs/source/pythonwhat.wiki/test_data_frame.md new file mode 100644 index 00000000..fb0b403c --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_data_frame.md @@ -0,0 +1,37 @@ +test_data_frame +--------------- + + def test_data_frame(name, + columns=None, + undefined_msg=None, + not_data_frame_msg=None, + undefined_cols_msg=None, + incorrect_msg=None) + +Test a pandas DataFrame. This methods makes it possible to test the columns of a DataFrame object independently. Only the contents will be tested. Customisable error messages are possible for when there is no object in the process with name `name`, for when that object is no pandas DataFrame, when there are columns you want to test for which are not defined and when some columns contain bad values. `columns` contains a list of column names, and defaults to `None`. If it's `None`, all columns that are found in the data frame created by the solution will be tested. + +### Example 1 + +Suppose we have the following solution: + + *** =solution + ```{python} + # import pandas + import pandas as pd + + # Create dataframe with columns a and b + my_df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + ``` + +To test this we simply use: + + *** =sct + ```{python} + test_import("pandas") + test_data_frame("my_df", columns = ["a", "b"]) + success_msg("Great job!") + ``` + +This SCT will first test if `pandas` is correctly imported, and will then check if the student created a Pandas DataFrame called `my_df`. If it was not defined, a message is generated that you can override with `undefined_msg`. If the object was defined but it isn't a Pandas DataFrame, as message is generated that you can override wiht `not_data_frame_msg`. If `my_df` is a Pandas DataFrame, `test_data_frame()` goes on to check if all columns that are specified in the `columns` argument are defined in the data frame, and next whether these columns are correct. The messages that are generated in case of an incorrect submission can be overrided with `undefined_cols_msg` and `incorrect_msg`, respectively. + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_dictionary.md b/docs/source/pythonwhat.wiki/test_dictionary.md new file mode 100644 index 00000000..51785d76 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_dictionary.md @@ -0,0 +1,43 @@ +test_dictionary +--------------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_dictionary.test_dictionary +``` + + def test_dictionary(name, + keys=None, + undefined_msg=None, + not_dictionary_msg=None, + key_missing_msg=None, + incorrect_value_msg=None) + +Test a dictionary. Consider this function an advanced version of `test_object`, where you can specify messages that are explicit to test dictionaries. `test_dictionary` takes a step-by-step approach to checking the correspondence of the dictionary between student and solution process: + +- Step 1: Is the object specified in `name` actually defined? +- Step 2: Is the object specified in `name` actually a dictionary? +- Step 3: For each key, is the key specified in the dictionary? +- Step 4: For each key, is the value corresponding to the key correct when comparing to the solution? + +For Step 3 and Step 4, you can control which keys have to be tested through the `keys` argument. If you don't specify this argument, `test_dictionary()` will look for all keys and compare the values that are specified in the corresponding dictionary in the solution process. + +### Example: step by step + +Suppose you want the student to create a dictionary `x`, that contains three keys: `"a"`, `"b"` and `"c"`. The following solution and sct could be used for this: + + *** =solution + ```{python} + x = {'a': 123, 'b':456, 'c':789} + ``` + + *** =sct + ```{python} + test_dictionary('x') + ``` + +- Step 1: if the student submits an empty script, the feedback _Are you sure you defined the dictionary `x`?_ will be presented. You can override this by specifying `undefined_msg` yourself. +- Step 2: if the student submits `x = 123`, the feedback _`x` is not a dictionary._ will be presented. You can override this message by specifying `not_dictionary_msg` yourself. +- Step 3: if the student submits `x = {'a':123, 'b':456, 'd':78}`, the feedback _Have you specified a key `c` inside `x`?_ will be presented. You can override this by specifying `key_missing_msg` yourself. +- Step 4: if the student submits `x = {'a':123, 'b':456, 'c':78}`, the feedback _Have you specified the correct value for the key `c` inside `x`?_ will be presented. You can override this by specifying `incorrect_value_msg` yourself. + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_expression_output.md b/docs/source/pythonwhat.wiki/test_expression_output.md new file mode 100644 index 00000000..df88df23 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_expression_output.md @@ -0,0 +1,86 @@ +test_expression_output +---------------------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_expression_output.test_expression_output +``` + + def test_expression_output(extra_env=None, + context_vals=None, + incorrect_msg=None, + eq_condition="equal", + expr_code=None, + pre_code=None, + keep_objs_in_env=None) + + +`test_expression_output()` is similar to `test_expression_result()`, but instead of checking the result, it checks the output that a single or a set of expressions generates. Typically, this function is used as a sub-test inside other test functions, such as `test_for_loop()` and `test_with()`. + +By default, the `test_expression_output()` will execute the 'active expression(s)'; if it's used as a top-level SCT function, that is the entire student submission. If it's used inside the `body` of the `test_for_loop()` function, for example, the entire for loop's body will executed and the output will be compared to the solution. With `expr_code`, you can override this default expression tree. With `pre_code`, you can prepend the execution of the default expression tree with some extra code, for example to set some variables. + +Oftentimes, the expression you want to check the output for does not have all variables to its disposal that it requires. Remember that the process in which the expression is evaluated only contains the variables that are available in the global scope. If you're for example running the body of a function definition, this means that the local variables, that are for example passed into the function as arguments, are not all available during execution. To make these variables available, you can set the `extra_env` and `context_vals` arguments. The former is to specify extra variables, with a dictionary. The latter is to specify so called context values; the objects you specify here should not be named; this is to make function definition arguments, iterators in for loops, context variables in `with` constructs, etc, name-independent. + +### Example 1 + +Suppose we want the student to define a function, that loops over the elements in a dictionary, and prints out each key and value, as follows: + + *** =solution + ```{python} + def print_dict(my_dict): + for key, value in my_dict.items(): + print(key + " - " + str(value)) + ``` + +An appropriate SCT for this exercise could be the following (for clarity, we're not using any default messages): + + *** =sct + ```{python} + def fun_body_test(): + def for_iter_test(): + example_dict = {'a': 2, 'b': 3} + test_expression_result(context_vals = [example_dict]) + def for_body_test(): + test_expression_output(context_vals = ['c', 3]) + test_for_loop(for_iter = for_iter_test, body = for_body_test) + + test_function_definition('print_dict', body = fun_body_test) + ``` + +Assuming the student coded the function in the exact same way as the solution, the following things happen: + +- `test_function_definition()` is run first: it checks whether `print_dict` is defined, whether the arguments are correctly named and with the correct defaults. Next, it checks the function definition body: it extracts the body of both the student and the solution code, sets the context values for this 'substate', i.e. `"my_dict"`, and then runs `fun_body_test()`. +- Inside `fun_body_test()`, `test_for_loop()` is executed. This function will find the for loop in the function definition body of both student and solution code, and will then run different tests: + - First, the `for_iter` test is run, which is specified with `for_iter_test()` in this SCT. The `for_iter` part of the `for` loop is extracted, which is `my_dict.items()` in the case of the solution. The context values are still `"my_dict"`. Inside `test_expression_result()`, the context vals are specified, so through `context_vals = [example_dict]`, the variable `my_dict` will now have the value `{'a': 2, 'b':3}` inside the student and solution processes. Next, the currently active expression (`my_dict.items()`) is executed. The result of calling this expression in both student and solution process is compared. + - Second, the `body` test is run, which is specified iwth `for_body_test()` in this SCT. The `body` part of the `for` loop is extracted, which is `print(key + " - " + str(value))` in the case of the solution. Now, the context values are set to the iterator variables of the `for` loop, so `"key"` and `"value"`. Inside `test_expression_output()`, the context vals are specified: `key` is set to be `'c'`, `value` is set to be `3`. Next, the currently active expression (`print(key + " - " + str(value))`) is executed, and the output it generates is fetched. The output of calling this expression in both student and solution process is compared. + +### Example 2 + +Suppose now that inside the `for` loop of `print_dict()` from the previous example, you each time want to print out the length of the entire dictionary: + + *** =solution + ```{python} + def print_dict(my_dict): + for key, value in my_dict.items(): + print("total length: " + str(len(my_dict))) + print(key + " - " + str(value)) + ``` + +The SCT from before won't work out of the box, because now you also need a value for `my_dict` inside `test_expression_output()`, the test of the body of the `for` loop, but this value is not available. You cannot specify this value through `context_vals`, because the context variables are already updated to be `"key"` and `"value"`. To be able to test this appropriately, you'll have to set `extra_env` inside `test_expression_output()`: + + *** =sct + ```{python} + def fun_body_test(): + def for_iter_test(): + example_dict = {'a': 2, 'b': 3} + test_expression_result(context_vals = [example_dict]) + def for_body_test(): + example_dict = {'a': 2, 'b': 3} + test_expression_output(context_vals = ['c', 3], extra_env = {'my_dict': example_dict}) + + test_for_loop(for_iter = for_iter_test, body = for_body_test) + + test_function_definition('print_dict', body = fun_body_test) + ``` + +With this update of the SCT, the exercise will still run fine. + diff --git a/docs/source/pythonwhat.wiki/test_expression_result.md b/docs/source/pythonwhat.wiki/test_expression_result.md new file mode 100644 index 00000000..5ce97de7 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_expression_result.md @@ -0,0 +1,19 @@ +test_expression_result +---------------------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_expression_result.test_expression_result +``` + + def test_expression_result(extra_env=None, + context_vals=None, + incorrect_msg=None, + eq_condition="equal", + expr_code=None, + pre_code=None, + keep_objs_in_env=None, + error_msg=None) + +`test_expression_result()` works pretty much the same as `test_expression_output()` and takes the same arguments. However, in this case, the expression should be a single expression and can't be a 'tree of expressions', such as the entire body of a function definition for example. Currently, the only places where `test_expression_result()` is used, is inside inherently 'single expression parts' of your code, such as the sequence specification of a `for` loop, the expression of a lambda function, etc. + +The example in the [`test_expression_output()` article](https://github.com/datacamp/pythonwhat/wiki/test_expression_output) also explains the use of `test_expression_result()`. diff --git a/docs/source/pythonwhat.wiki/test_for_loop.md b/docs/source/pythonwhat.wiki/test_for_loop.md new file mode 100644 index 00000000..4b4545d1 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_for_loop.md @@ -0,0 +1,88 @@ +test_for_loop +------------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_for_loop + :members: +``` + + + test_for_loop(index=1, + for_iter=None, + body=None, + orelse=None, + expand_message=True) + +As the name suggesets, you can use `test_for_loop()` to test if a for loop was properly coded. Similar to how `test_if_else()` and `test_while_loop()` works, `test_for_loop()` parses the for loop in the student's submission and breaks it up into its composing parts. Next, it also parses the for loop in the solution solution, and compares the parts between student submission and solution. It does this through sub-SCTs that you specify in `cond_test` and `expr_test`. + +### Example 1 + +Suppose you want the student to implement an algorithm that calculates fibonacci's row (until `n = 20`) using a simple for loop. The solution could look like this: + + *** =solution + ```{python} + # Initialise the row + fib = [0, 1] + + # Update the row correctly each loop + for n in range(2, 20): + fib.append(fib[n-2] + fib[n-1]) + ``` + +An SCT to accompany this exercise could be the following: + + *** =sct + ```{python} + def test_for_iter(): + "You have to iterate over `range(2, 20)`" + test_function("range", + not_called_msg=msg, + incorrect_msg=msg) + + def test_for_body(): + msg = "Make sure your row, `fib`, updates correctly" + test_object_after_expression("fib", + extra_env={ "fib": [0, 1, 1, 2] }, + context_vals=[4], + undefined_msg=msg, + incorrect_msg=msg) + test_for_loop(index=1, + for_iter=test_for_iter, + body=test_for_body) + ``` + + +Notice that two self-defined functions, `test_for_iter()` and `test_for_body()` are used to specify the sub-SCTs for the different parts in the `for` loop. With `index = 1`, you tell `pythonwhat` that you want to check the first `for` loop you find in the student submission with the first `for` loop in the solution. + + +The `for_iter` part of `test_for_loop()` tests whether the loop with index 1 loops over the correct range. The tests in this sub-SCT are run on the sequence part of the loop, which in this case for the solution is `range(2, 20)`. With `test_function()`, we can test this. In other cases, you could use e.g. `test_expression_result()`, to test the result of the sequence part. + +The `body` part of `test_for_loop()` tests whether `fib` is updated correctly. The tests in this sub-SCT are run on the body of the loop. The `test_object_after_expression()`. This function will test an object after the active expression is run in the student and solution process. In this case it will check if `fib` is updated the same in the student and solution process after one loop through the body of the `for`. Two important arguments for `test_object_after_expression()` are: + +- `extra_env = { "fib": [0, 1, 1, 2] }`: when running the body of the for loop, the process will be updated with these extra environment variables. In this case this means that before the body is ran, `fib` will be initialised to `[0, 1, 1, 2]`. +- `context_vals = [4]`: this argument contains the values of the loop's variable. In the solution code, for example, there will be one: `n`. This means that `n` will be initialised to `4` in the solution process when the body of the for loop is run. The student can give any name to `n`, as long as the functionality remains the same. + +You may have noticed that the helper functions that are used within `test_for_loop()` contain feedback messages as well. When they are used within a `test_for_loop()`, these messages will automatically be extended with "in the ___ of the for loop on line ___.". To avoid this extension, you could set the option `expand_message = False` in `test_for_loop()`. + +Example 2: Multiple context vals + +If you have multiple context vals, things largely work the same way. Suppose you want somebody to print out the keys and values of a dictionary as follows: + + *** =solution + ```{python} + my_dict = {'a': 1, 'b': 2, 'c': 3} + for k, v in my_dict.items(): + print(k + ' - ' + str(v)) + ``` + +An appropriate SCT would be: + + *** =sct + ```{python} + test_object('my_dict') + test_for_loop(index=1, + for_iter = lambda: test_expression_result(), + body = lambda: test_expression_output(context_vals = ['a', 1])) + ``` + +In this case, when you're checking the output of the body of the `for` loop, you're telling `k` to be `'a'` and `v` to be `1`. diff --git a/docs/source/pythonwhat.wiki/test_function.md b/docs/source/pythonwhat.wiki/test_function.md new file mode 100644 index 00000000..ff5d43f6 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_function.md @@ -0,0 +1,208 @@ +test_function +------------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_function.test_function +``` + + test_function(name, + index=1, + args=None, + keywords=None, + eq_condition="equal", + do_eval=True, + not_called_msg=None, + incorrect_msg=None) + +`test_function()` enables you to test whether the student called a function correctly. The function first tests if the specified function is actually called by the student, and then compares the call with calls of the function in the solution code. Next, it can compare the parameters passed to these functions. Because `test_function()` also uses the student and solution process, this can be done in a very concise way. + +### Example 1 + +Suppose you want the student to call the `round()` function on pi, as follows: + + *** =solution + ```{python} + # This is pi + pi = 3.14159 + + # Round pi to 3 digits + r_pi = round(pi, 3) + ``` + +The following SCT tests whether the `round()` function is used correctly: + + *** =sct + ```{python} + test_function("round") + success_msg("Great job!") + ``` + +This is a very robust way to test whether `round()` is used, much more robust when comparing to `test_student_typed()`. `test_function()` tests whether the student has called the function `round()` and checks whether the values of the arguments are the same as in the solution. So in this case, it tests wether `round()` is used with its first argument equal to `3.14159` and the second argument equal to `3`. `test_function()` figures out the values of these arguments from the solution code and the solution processes that corresponds with it. The above SCT would accept all of the following student submissions: + +- `round(3.14159, 3)` +- `pi = 3.14159; dig = 3; round(pi, dig)` +- `int_part = 3; dec_part = 0.14159; round(int_part + dec_part, 3)` + +By default, `test_function()` tests all arguments that are specified in the solution code. It is also possible to check whether a function is used and only check specific positional arguments. For example, + + *** =sct + ```{python} + test_function("round", args=[0]) + success_msg("Great job!") + ``` + +will only test whether the first argument's value is `3.14159`. A student submission that is `round(pi, 5)` would also pass this SCT. + +With `args`, you can also control whether or not to actually check the values that were passed as parameters. Say you only want to check that the function `round()` was called: + + *** =sct + ```{python} + test_function("round", args=[]) + success_msg("Great job!") + ``` + + +`test_function()` will automatically generate meaningful feedback, but you can also override these messages with `not_called_msg` and `incorrect_msg`. The former controls the message that is thrown if the student didn't call the specified function in the first place. The latter is thrown if the student did not correctly set the arguments in the function call: + + *** =sct + ```{python} + test_function("round", + not_called_msg = "You did not call `round()` to round the irrational number, `pi`.", + incorrect_msg = "Be sure to round `pi` to `3` digits.`) + success_msg("Great job!") + ``` + + + +### Example 2: Multiple function calls + +`index`, which is 1 by default, becomes important when there are several calls of the same function. Suppose that your exercise requires the student to call the `round()` function twice: once on `pi` and once on `e`, Euler's number. A possible solution could be the following: + + *** =solution + ```{python} + # Call round on pi + round(3.14159, 3) + + # Call round on e + round(2.71828, 3) + ``` + +To test both these function calls, you'll need the following SCT: + + *** =sct + ```{python} + test_function("round", index=1) + test_function("round", index=2) + success_msg("Two in a row, great!") + ``` + +The first `test_function()` call, where `index=1`, checks the solution code for the first function call of `round()`, finds it - `round(3.14159, 3)` - and then goes to look through the student code to find a function call of `round()` that matches the arguments. It is perfectly possible that there are 5 function calls of `round()` in the student's submission, and that only the fourth call matches the requirements for `test_function()`. As soon as a function call is found in the student code that passes all tests, `pythonwhat` heads over to the second `test_function()` call, where `index=2`. The same thing happens: the second call of `round()` is found from the solution code, and a match is sought for in the student code. This time, however, the function call that was matched before is now 'blacklisted'; it is not possible that the same function call in the student code causes both `test_function()` calls to pass. + +This means that all of the following student submissions would be accepted: + + - `round(3.14159, 3); round(2.71828, 3)` + - `round(2.71828, 3); round(3.14159, 3)` + - `round(3.14159, 3); round(123.456); round(2.71828, 3)` + - `round(2.71828, 3); round(123.456); round(3.14159, 3)` + +Of course, you can also specify all other arguments to customize your test, such as `do_eval`, `args`, `not_called_msg` and `incorrect_msg`. + +### Example 3: Custom feedback + +By default `test_function()` checks all arguments and keywords that are specified in the solution; if you specify `incorrect_msg`, any error to one of these arguments will replaced by the same custom message. If you want to provide different custom error messages for different arguments, you can do so with multiple function calls. To, for example, provide different feedback for the first and second argument of the `round()` function: + + *** =sct + ```{python} + test_function("round", args = [0], index=1, incorrect_msg = 'first arg wrong!') + test_function("round", args = [1], index=1, incorrect_msg = 'second arg wrong!') + success_msg("Well done") + ``` + +**NOTE**: currently, `test_function()` automatically checks all arguments and keywords that you specify in corresponding function call in the solution. Therefore, if you want to give specific feedback, make sure to select a single argument or a single keyword. To check the first argument, you can best use `args = [0], keywords = []`, to test a keyword named `check`, you'll want to use `args = [], keywords = ['check']`. + +### Example 4: Methods + +Python also features methods, i.e. functions that are called on objects. For testing such a thing, you can also use `test_function()`. Consider the following solution code, that creates a connection to an SQLite Database with `sqlalchemy`. + + *** =solution + ```{python} + from urllib.request import urlretrieve + from sqlalchemy import create_engine, MetaData, Table + engine = create_engine('sqlite:///census.sqlite') + metadata = MetaData() + connection = engine.connect() + from sqlalchemy import select + census = Table('census', metadata, autoload=True, autoload_with=engine) + stmt = select([census]) + + # execute the query and fetch the results. + connection.execute(stmt).fetchall() + ``` + +To test the last chained method calls, you can use the following SCT. Notice from the second `test_function()` call here that you have to describe the entire chain (leaving out the arguments that are passed to `execute()`). This way, you explicitly list the order in which the methods should be called. + + *** =sct + ``` + test_function("connection.execute", do_eval = False) + test_function("connection.execute.fetchall") + ``` + +**NOTE**: currently, it is not possible to easily test the arguments inside chained method calls, methods inside arguments, etc. We are working on a massive update of `pythonwhat` to easily support this very customized testing, with virtually no limit to 'how deep you want the tests to go'. More on this later! + +### `do_eval` + +With `do_eval`, you can control how arguments are compared between student and solution code. + +- If `do_eval` is `True`, the evaluated version of the arguments are compared; +- If `do_eval` is `False`, the 'string version' of the argumetns are compared; +- If `do_eval` is `None`, the arguments are not compared; in this case, `test_function()` simply checks if you specified the arguments, without further checks. + + +### Function calls in packages + +If you're testing whether function calls of particular packages are used correctly, you should always refer to these functions with their 'full name'. Suppose you want to test whether the function `show` of `matplotlib.pyplot` was used correctly, you should use + + *** =sct + ```{python} + test_function("matplotlib.pyplot.show") + ``` + +The `test_function()` call can handle it when a student used aliases for the python packages (all `import` and `import * from *` calls are supported). In case there is an error, `test_function()` will automatically generated a feedback message that uses the alias of the student. + +**NOTE:** No matter how you import the function, you always have to refer to the function with its full name, e.g. `package.subpackage1.subpackage2.function`. + +### Argument equality + +Just like with `test_object()`, evaluated arguments are compared using the `==` operator (check out [the section about Object equality](https://github.com/datacamp/pythonwhat/wiki/test_object#object-equality)). For a lot of complex objects, the implementation of `==` causes the object instances to be compared... not their underlying meaning. For example when the solution is: + + *** =solution + from urllib.request import urlretrieve + fn1 = 'https://s3.amazonaws.com/assets.datacamp.com/production/course_998/datasets/Chinook.sqlite' + urlretrieve(fn1, 'Chinook.sqlite') + + # Import packages + from sqlalchemy import create_engine + import pandas as pd + + # Create engine: engine + engine = create_engine('sqlite:///Chinook.sqlite') + + # Execute query and store records in dataframe: df + df = pd.read_sql_query("SELECT * FROM Album", engine) + +And the SCT is: + + *** =sct + test_function("pandas.read_sql_query") + +The SCT will fail even if the student uses this exact solution code. The reason being that the `engine` object is compared in the solution and student process. The engine object is evaluated by `create_engine('sqlite:///Chinook.sqlite')`. As you can try out yourself, `create_engine('sqlite:///Chinook.sqlite') == create_engine('sqlite:///Chinook.sqlite')` will always be `False`, even though they are semantically exactly the same. A better way of testing this code would be: + + *** =sct + test_correct( + lambda: test_object("df"), + lambda: test_function("pandas.read_sql_query", do_eval=False) + ) + +This SCT will not do exactly the same, but it will test enough in practice 99% of the time. Check out [the section about Object equality](https://github.com/datacamp/pythonwhat/wiki/test_object#object-equality) for complex objects that do have a good equality implementation. + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_function_definition.md b/docs/source/pythonwhat.wiki/test_function_definition.md new file mode 100644 index 00000000..a0439dcc --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_function_definition.md @@ -0,0 +1,194 @@ +test_function_definition +------------------------ + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_function_definition + :members: +``` + + def test_function_definition(name, + arg_names=True, + arg_defaults=True, + body=None, + results=None, + outputs=None, + errors=None, + not_called_msg=None, + nb_args_msg=None, + other_args_msg=None, + arg_names_msg=None, + arg_defaults_msg=None, + wrong_result_msg=None, + wrong_output_msg=None, + no_error_msg=None, + wrong_error_msg=None, + expand_message=True): + + +In more advanced courses, you'll sometimes want students to define their own functions. With `test_function_definition()` it is possible to test such user-defined functions in a robust way. This function allows you to test four things: + +1. The argument names of the function (including if the correct defaults are used) +2. The body of the functions (does it output correctly, are the correct functions used) +3. The return value with a certain input +4. The output value with a certain input + +### Example 1 + +Say you want a student to write a very basic function to set numbers in a base from 1 up until 9 to a decimal. To not overcomplicate things you just ask them to implement the basic functionality; they don't have to catch any exceptions. A solution to the exercise can like like this: + + *** =solution + ```{python} + def to_decimal(number, base = 2): + print("Converting %d from base %s to base 10" % (number, base)) + number_str = str(number) + number_range = range(len(number_str)) + multipliers = [base ** ((len(number_str) - 1) - i) for i in number_range] + decimal = sum([int(number_str[i]) * multipliers[i] for i in number_range]) + return decimal + ``` + +You could test the function like this: + + *** =sct + ```{python} + # All of the following test_function_definition() functions are done on the same + # function definition. + + # Test the function, see that the defaults of the arguments are the same. + # For this function, we don't care about the argument names of the function. + # Note: generally, we DO care about the names of the arguments, since they can + # be used as keywords. arg_defaults and arg_names will be set to True by default. + + test_function_definition("to_decimal", arg_defaults = True, arg_names = False) + + # Here, a feedback message will be generated. You can overwrite this feedback + # message by using: + # test_function_definition("to_decimal", arg_defaults = True, arg_names = False, + # arg_defaults_msg = "Use the correct default argument values!") + # In the following tests, I'll always use the standard feedback messages, remember they + # can almost always be overwritten. + + # We want to test whether the function returns the correct things with certain inputs. + + test_function_definition("to_decimal", arg_names = False, arg_defaults = False, # Already tested this + results = [ + [1001101, 2], + ]1212357, 8] + ) + + # This will run to_decimal(1001101, 2) and to_decimal(1212357, 8) in student and solution + # process, and match the results. If they don't match, a feedback message will be generated. + # Note: here we've set arg_defaults to False, because we already tested this in the first + # test_function_definition. + + # We want to test the output of the function with certain inputs. + + test_function_definition("to_decimal", arg_names = False, arg_defaults = False, # Already tested this + outputs = [ + [1234, 6], + [8888888, 9] + ) + + # This will run to_decimal(1234, 6) and to_decimal(8888888, 9) in solution and student + # process and compare their printed output. + + # Finally, we might want them to use a certain function. For this we can do tests specifically + # on the body of the function. Remember you can use lambda functions or custom functions for this + # (also see wiki about test_if_else(), test_for_loop() and test_while_loop(). + + test_function_definition("to_decimal", arg_names = False, arg_defaults = False, # Already tested this + body = lambda: test_function("sum", args = [], incorrect_msg = "you should use the `sum()` function.")) + + # This will test the body of the function definition, and see if the function sum() is used. + # Note that the generated feedback will be preceded by: 'In your definition of `to_decimal()`, ...' + # So if the last test doesn't pass, this feedback will be generated: + # In your definition of `to_decimal()`, you should use the `sum()` function. + ``` + +Pitfall: you have to watch out when using `test_function()` in a body test, you should never test arguments +that are only defined within the scope of the function (e.g. function parameters). This is the reason why +we used `args = []` in the last test, because the argument used in `sum()` can not be calculated to verify +in the global scope. This is something which would require architectural changes in the `pythonwhat` package. + + +### Example 2: User-defined errors + +In some cases, you'll want the student to code resilience against incorrect inputs or behavior. To test this, you can use the `errors`, `no_error_msg` and `wrong_error_msg` arguments. The first is similar to `results`, and specifies the input arguments as a list of tuples or a list of lists, that have to generate an error. With `no_error_msg` you can control the message that is presented if running one of these argument sets does not generate an error, while it should. With `wrong_error_msg`, you control the message that is presented if the type of the error (or exception) that is thrown does not correspond to the type that is thrown when the function is called in the solution process. + +Suppose you want the student to code up a function `inc`, that increments a number if it's positive. If it's not, you want the function to raise a `ValueError`. A solution could look like this: + + *** =solution + ```{python} + def inc(num): + if num < 0: + raise ValueError('num is negative') + return(num + 1) + ``` + +To test this, we can use the following SCT (we're only focussing on the `errors` part here; of course you can extend the `test_function_definition()` call with more checks on arguments, `results`, body, etc.): + + *** =sct + ```{python} + test_function_definition("inc", errors = [[-1]]) + ``` + +If the student submits the following code: + +``` +def inc(num): + return(num + 1) +``` + +the SCT will see it's incorrect and throw the message: _Calling `inc(-1)` doesn't result in an error, but it should!_ + +If the student submits the following code: + +``` +def inc(num): + if num < 0: + raise NameError('num is negative') + return(num + 1) +``` + +the SCT will see it's incorrect and throw the message: _Calling `inc(-1)` should result in a `ValueError`, instead got a `NameError`._ + +Currently, there isn't a way to test the actual message you pass with errors you raise. + +### Example 3: `*args` and `**kwargs` + +When defining a function in Python, it also possible to specify so-called 'unordered non-keyword arguments', with a `*`, and 'unordered keyword arguments'. Typically, these are called `args` and `kwargs` respectively, but this is not required. + +Have a look at the following example: + + *** =solution + ```{python} + def my_fun(x, y = 4, z = ['a', 'b'], *args, **kwargs): + k = len(args) + l = len(kwargs) + print("just checking") + return k + l + ``` + +An SCT to check this function definition: + + *** =sct + ```{python} + def inner_test(): + context = ['r', 's', ['c', 'd'], ['t', 'u'], {'a': 2, 'b': 3, 'd':4}] + test_object_after_expression('k', context_vals = context) + test_object_after_expression('l', context_vals = context) + test_function_definition("my_fun", body = inner_test, + results = [{'args': ['r', 's', ['c', 'd'], 't', 'u', 'v'], 'kwargs': {'a': 2, 'b': 3, 'd': 4}}], + outputs = [{'args': ['r', 's', ['c', 'd'], 't', 'u', 'v'], 'kwargs': {'a': 2, 'b': 3, 'd': 4}}]) + ``` + +There are different things to note: + +- By default, the names of the `*` argument and the `**` argument are checked, if they are defined in the solution. This is controlled through `arg_names`, just like for 'regular' arguments. To override the automatic message that is thrown if the `*` or `**` arg is not specified or not appropriately named, use `other_args_msg`. +- The `*` and `**` args are also part of the context values that you can specify in 'inner tests'. They are appended to the normal arguments: first the `*`, then the `**` argument. You can see in the `context` object, that the penultimate element is used to specify the `*args` argument, and the last element, a dictionary, is used to specify the `**` argument. +- Before, you saw that `results`, `outputs`, and `errors` should be a list of lists, where the inner list is the list of arguments. To also cater for explicitly keyworded arguments, you can also specify a list of dictionaries. Each dictionary represents one call of the user-defined fucntion and should contain two elements: `'args'` and `'kwargs'`. Behind the scenes, the function will be called as: `my_fun([*d['args'], **d['kwargs']])`, where `d` is the two-key dictionary. + + +### Sidenote + +Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_function_v2.md b/docs/source/pythonwhat.wiki/test_function_v2.md new file mode 100644 index 00000000..6b9c0da4 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_function_v2.md @@ -0,0 +1,297 @@ +test_function_v2 +---------------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_function.test_function_v2 +``` + + test_function_v2(name, + index=1, + params=None, + signature=None, + eq_condition="equal", + do_eval=True, + not_called_msg=None, + params_not_matched_msg=None, + params_not_specified_msg=None, + incorrect_msg=None) + +`test_function_v2()` enables you to test whether the student called a function correctly. The function first tests if the specified function is actually called by the student, and then compares the call with calls of the function in the solution code. Next, it can compare the arguments passed to these functions. Because `test_function_v2()` also uses the student and solution process, this can be done in a very concise way. `test_function_v2()` is an improved version of [`test_function()`](https://github.com/datacamp/pythonwhat/wiki/test_function), where: + +- there is resilience against different ways of calling a function (arguments vs keywords), +- you have to be specific about which parameters you want to check, +- you can specify parameter-specific evaluation forms (`do_eval` can be a list), +- you can specify parameter-specific custom messages (`params_not_matched_msg` and `params_not_specified_msg` can be lists), +- you have more control over messaging in general. + +### Example 1 + +Suppose you want the student to call the `round()` function on pi, as follows: + + *** =solution + ```{python} + # This is pi + pi = 3.14159 + + # Round pi to 3 digits + r_pi = round(pi, 3) + ``` + +The following SCT tests whether the `round()` function is used correctly: + + *** =sct + ```{python} + test_function_v2("round", params=["number", "ndigits"]) + success_msg("Great job!") + ``` + +`test_function_v2()` tests whether the student has called the function `round()` and checks whether the values of the arguments are the same as in the solution. So in this case, it tests whether `round()` is used and the `number` and `ndigits` parameters, that `round()` expects, are specified correctly, i.e. equal to `3.14159` and `3` respectively. `test_function_v2()` figures out the values of these arguments from the solution code and the solution process that corresponds with it. The above SCT would accept all of the following student submissions: + +- `round(3.14159, 3)` +- `round(number=3.14159, 3)` +- `round(number=3.14159, ndigits=3)` +- `round(ndigits=3, number=3.14159)` +- `pi=3.14159; dig=3; round(pi, dig)` +- `pi=3.14159; dig=3; round(number=pi, dig)` +- `int_part = 3; dec_part = 0.14159; round(int_part + dec_part, 3)` + +In `params`, you have to explicitly list all the parameters that you want to test. If you only want to check the `number` parameter, for example, you can use: + + *** =sct + ```{python} + test_function_v2("round", params=["number"]) + success_msg("Great job!") + ``` + +This SCT will only test whether the `number` parameter was specified to be `3.14159`. If a student submits `round(pi, 5)`, this would also pass this SCT. + +If you specify `params` to be an empty list, which is the default, you are simply checking whether the `round()` function was called in the first place: + + *** =sct + ```{python} + test_function_v2("round") # same as test_function_v2("round", params=[]) + success_msg("Great job!") + ``` + +`test_function_v2()` will automatically generate meaningful feedback, but you can also override these messages through the different `*_msg` parameters that `test_function_v2()` features: + +- `not_called_msg`: message if the student didn't call the specified function or didn't call the specified function often enough (if you're testing multiple calls of the same function in the same submission). +- `params_not_matched_msg`: message if the function call of the student was invalid, i.e. if the way of specifying the different parameters was invalid. +- `params_not_specified_msg`: message if the student did not specify all parameters that are specified inside `params`. This argument can either be a string, to give the same message for each parameter that is missing, or a list of strings with the same length as `params`. In case of a missing parameter, `test_function_v2()` will present the corresponding message. +- `incorrect_msg`: message if the student did not specify all parameters correctly, so when his or her specifications don't correspond with the solution. This argument can again be a single string, or a list of parameter-specific feedback messages. + +Below is an example of an SCT that specified all feedback messages. This is not required, though; you can depend on the automatic feedback messages for the `not_called_msg`, `params_not_specified_msg` and `incorrect_msg` and only manually specify the `params_not_matched_msg`, for example. + + *** =sct + ```{python} + test_function_v2("round", params=["number", "ndigits"] + not_called_msg="You did not call `round()` to round the irrational number, `pi`.", + params_not_matched_msg="Are you sure you correctly called the `round()` function?", + params_not_specified_msg="Make sure to specify both the `number` and `ndigits` parameter!", + incorrect_msg=["Make sure to correctly specify `number`; it should be `pi`, or `3.14159`.", + "Have you specified `ndigits` so that `pi` is rounded to 3 digits?"]) + success_msg("Great job!") + ``` + + +### Example 2: Multiple function calls + +`index`, which is 1 by default, becomes important when there are several calls of the same function. Suppose that your exercise requires the student to call the `round()` function twice: once on `pi` and once on `e`, Euler's number. A possible solution could be the following: + + *** =solution + ```{python} + # Call round on pi + round(3.14159, 3) + + # Call round on e + round(2.71828, 3) + ``` + +To test both these function calls, you'll need the following SCT: + + *** =sct + ```{python} + test_function_v2("round", params=["number","ndigits"], index=1) + test_function_v2("round", params=["number","ndigits"], index=2) + success_msg("Two in a row, great!") + ``` + +The first `test_function_v2()` call, where `index=1`, checks the solution code for the first function call of `round()`, finds it - `round(3.14159, 3)` - and then goes to look through the student code to find a function call of `round()` that matches the arguments. It is perfectly possible that there are 5 function calls of `round()` in the student's submission, and that only the fourth call matches the requirements for `test_function_v2()`. As soon as a function call is found in the student code that passes all tests, `pythonwhat` heads over to the second `test_function_v2()` call, where `index=2`. The same thing happens: the second call of `round()` is found from the solution code, and a match is sought for in the student code. This time, however, the function call that was matched before is now 'blacklisted'; it is not possible that the same function call in the student code causes both `test_function_v2()` calls to pass. + +This means that all of the following student submissions would be accepted: + + - `round(3.14159, 3); round(2.71828, 3)` + - `round(2.71828, 3); round(3.14159, 3)` + - `round(number=3.14159, ndigts=3); round(number=2.71828, 3)` + - `round(number=2.71828, 3); round(number=3.14159, 3)` + - `round(3.14159, 3); round(123.456); round(2.71828, 3)` + - `round(2.71828, 3); round(123.456); round(3.14159, 3)` + +Of course, you can also specify all other arguments to customize your test to perfection, such as custom messages and `do_eval` (example 3). + +### Example 3: `do_eval` + +With `do_eval`, you can control how parameter specifications are compared between student and solution code. There are two ways to specify `do_eval`: you can specify a single value, that will be used for comparing all `params` that you specified. However, you can also specify a list of values, with the same length as `params`; the way in which parameter specifications are compared becomes parameter specific. In both cases, there are three valid values: + +- `True`, where the evaluated version of the student and solution arguments is compared. +- `False`, where the 'string version' of the arguments is compared; +- `None`, in which case the arguments are not compared; `test_function_v2` simply checks if the parameter(s) in question has/have been specified. + +Say, for example, you want to check if a student called the `round()` function and specified the parameters `number` and `ndigits`. You want to test the actual equality of `number`, but you don't care about the value of `ndigits`, you just want to make sure the student specified it, nothing more. + +The following solution and SCT implement this train of thought (custom feedback messages have not been specified, although this is perfectly possible): + + *** =solution + ```{python} + # This is pi + pi = 3.14159 + + # Round pi to 3 digits + r_pi = round(pi, 3) + ``` + + *** =sct + ```{python} + test_function_v2("round", + params=["number", "ndigits"], + do_eval=[True, None]) + success_msg("Great job!") + ``` + +All of the following submissions would be accepted by this SCT: + +- `round(pi, 3)` +- `round(number=pi, ndigits=3)` +- `round(number=pi, ndigits=4)` +- `round(pi, 4)` +- `round(pi, 0)` + +### Example 4: Function calls in packages + +If you're testing whether function calls of particular packages are used correctly, you should always refer to these functions with their 'full name'. Suppose you want to test whether the function `show` of `matplotlib.pyplot` was used correctly, you should use + + *** =sct + ```{python} + test_function_v2("matplotlib.pyplot.show") + ``` + +The `test_function_v2()` call can handle it when a student used aliases for the python packages (all `import` and `import * from *` calls are supported). In case there is an error, `test_function_v2()` will automatically generated a feedback message that uses the alias that the student used. + +**NOTE:** No matter how you import the function, you always have to refer to the function with its full name, e.g. `package.subpackage1.subpackage2.function`. + +### Example 5: Manual signatures + +To implement resilience against different ways of specify function parameters, the `inspect` module is used, that is part of Python's basic distribution. Through `inspect.signature()` a function's parameters can be inferred, and then 'bound' to the arguments that the student specified. However, this signature is not available for all of Python's functions. More specifically, Python's built-in functions that are implemented in C don't allow a signature to be extracted from them. `pythonwhat` already includes manually specified signatures for functions such as `print()`, `str()`, `hasattr()`, etc, but it's still possible that some signatures are missing. + +That's why `test_function_v2()` features a `signature` parameter, that is `None` by default. If `pythonwhat` can't retrieve a signature for the function you want to test, you can pass an object of the class `inspect.Signature` to the `signature` parameter. + +Suppose, for the sake of example, that `test_function_v2()` can't find a signature for the `round()` function (you will be informed by this through automated testing; running the solution against an SCT that depends on a signature that is not found will throw a backend error). To be able to implement this function test, you can use the `sig_from_params()` function: + + *** =sct + ```{python} + sig = sig_from_params(param("number", param.POSITIONAL_OR_KEYWORD), + param("ndigits", param.POSITIONAL_OR_KEYWORD, default=0)) + test_function_v2("round", params=["number", "ndigits"], signature=sig) + ``` + +`param` is an alias of the `Parameter` class that's inside the `inspect` module. You can pass `sig_from_params()` as many parameters as you want. The first argument of `param()` should be the name of the parameter, the second argument should be the 'kind' of parameter. `param.POSITIONAL_OR_KEYWORD` tells `test_function_v2` that the parameter can be specified either through a positional argument or through a keyword argument. Other common possibilities are `param.POSITIONAL_ONLY` and `param.KEYWORD_ONLY` (for a full list, refer to the [Python docs on `inspect`](https://docs.python.org/3.4/library/inspect.html#inspect.Parameter)). The third, optional argument, allows you to specify a default value for the parameter. + +**NOTE:** If you find vital Python functions that are used very often and that are not included in `pythonwhat` by default, you can [let us know](mailto:content-engineering@datacamp.com) and we'll add the function to our [list of manual signatures](https://github.com/datacamp/pythonwhat/blob/master/pythonwhat/signatures.py). + +### Example 6: Methods + +Python also features methods, i.e. functions that are called on objects. For testing such a thing, you can also use `test_function_v2()`. Consider the following solution code, that creates a connection to an SQLite Database with `sqlalchemy`. + + *** =solution + ```{python} + # Prepare everything +from urllib.request import urlretrieve +from sqlalchemy import create_engine, MetaData, Table +engine = create_engine('sqlite:///census.sqlite') +metadata = MetaData() +connection = engine.connect() +from sqlalchemy import select +census = Table('census', metadata, autoload=True, autoload_with=engine) +stmt = select([census]) + + # execute the query and fetch the results. + connection.execute(stmt).fetchall() + ``` + +To test the last chained method calls, you can use the following SCT. Notice from the second `test_function_v2()` call here that you have to describe the entire chain (leaving out the arguments that are passed to `execute()`). This way, you explicitly list the order in which the methods should be called. + + *** =sct + ```{python} + test_function_v2("connection.execute", params = ["object"], do_eval = False) + test_function_v2("connection.execute.fetchall") + ``` + +**NOTE**: currently, it is not possible to easily test the arguments inside chained method calls, methods inside arguments, etc. We are working on a massive update of `pythonwhat` to easily support this very customized testing, with virtually no limit to 'how deep you want the tests to go'. More on this later! + +### Example 7: Signatures for methods + +In the previous example, you might have noticed that `test_funtion_v2()` was capable to infer that `connection` is a `Connection` object, and that `execute()` is a method of the `Connection` class. For checking method calls that aren't chained, this is possible, but for chained method calls, such as `connection.execute.fetchall`, this is not possible. In those cases you'll have to manually specify a signature. With `sig_from_obj()` you can specify the function from which to extract a signature. + +The following full example shows how it's done: + + *** =pre_exercise_code + ```{python} + class Test(): + def __init__(self, a): + self.a = a + + def set_a(self, value): + self.a = value + return(self) + x = Test(123) + ``` + + *** =solution + ```{python} + x.set_a(843).set_a(102) + ``` + + *** =sct + ```{python} + sig = sig_from_obj('x.set_a') + test_function_v2('x.set_a.set_a', params=['value'], signature=sig) + ``` + +**NOTE**: You can also use the `sig_from_params()` function to manually build the signature from scratch, but this this more work than simply specifying the function object as a string from which to extract the signature. + + +### Extra: Argument equality + +Just like with `test_object()`, evaluated arguments are compared using the `==` operator (check out [the section about Object equality](https://github.com/datacamp/pythonwhat/wiki/test_object#object-equality)). For a lot of complex objects, the implementation of `==` causes the object instances to be compared... not their underlying meaning. For example when the solution is: + + *** =solution + ``` + from urllib.request import urlretrieve + fn1 = 'https://s3.amazonaws.com/assets.datacamp.com/production/course_998/datasets/Chinook.sqlite' + urlretrieve(fn1, 'Chinook.sqlite') + from sqlalchemy import create_engine + import pandas as pd + engine = create_engine('sqlite:///Chinook.sqlite') + + # Execute query and store records in dataframe: df + df = pd.read_sql_query("SELECT * FROM Album", engine) + ``` + +And the SCT is: + + *** =sct + ``` + test_function_v2("pandas.read_sql_query", params = ['sql', 'con'], do_eval = [True, False]) + ``` + +The SCT will fail even if the student uses this exact solution code. The reason being that the `engine` object is compared in the solution and student process. The engine object is evaluated by `create_engine('sqlite:///Chinook.sqlite')`. As you can try out yourself, `create_engine('sqlite:///Chinook.sqlite') == create_engine('sqlite:///Chinook.sqlite')` will always be `False`, even though they are semantically exactly the same. A better way of testing this code would be: + + *** =sct + test_correct( + lambda: test_object("df"), + lambda: test_function_v2("pandas.read_sql_query", do_eval=False) + ) + +This SCT will not do exactly the same, but it will test enough in practice 99% of the time. Check out [the section about Object equality](https://github.com/datacamp/pythonwhat/wiki/test_object#object-equality) for complex objects that DO have a good equality implementation. + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_if_else.md b/docs/source/pythonwhat.wiki/test_if_else.md new file mode 100644 index 00000000..df894987 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_if_else.md @@ -0,0 +1,64 @@ +test_if_else +------------ + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_if_else.test_if_else +``` + + test_if_else(index=1, + test=None, + body=None, + orelse=None, + expand_message=True) + +`test_if_else()` allows you to robustly check `if` statements, optionally extended with `elif` and `else` components. For each of the components of an if-else construct `test_if_else()` takes several 'sub-SCTs'. These 'sub-SCTs', that you have to pass in the form of lambda functions or through a function that defines all tests, are executed on these separate parts of the submission. + +### Example 1 + +Suppose an exercise asks the student to code up the following if-else construct: + + *** =solution + ```{python} + # a is set to 5 + a = 5 + + # If a < 5, print out "It's small", else print out "It's big" + if a < 5: + + else: + print("It's big") + ``` + +The `if-else` construct here consists of three parts: + +- The condition to check: `a < 5`. The `test` argument of `test_if_else()` specifies the sub-SCT to test this. +- The body of the `if` statement: `print("It's small")`. The `body` argument of `test_if_else()` specifies the sub-SCT to test this. +- The else part: `print("It's big")`. The `orelse` argument of `test_if_else()` specifies the sub-SCT to ttest this. + +You can thus write our SCT as follows. Notice that for the `test` argument a function is used to specify different tests; for the `body` and `orelse` arguments two lambda functions suffise. + + *** =sct + ```{python} + def sct_on_condition_test(): + test_expression_result({"a": 4}) + test_expression_result({"a": 5}) + test_expression_result({"a": 6}) + + test_if_else(index = 1, + test = sct_on_condition_test, + body = lambda: test_function("print") + orelse = lambda: test_function("print")) + ``` + +#### The `test` part + +Have a look at the `sct_on_condition_test()`, that is used to specify the sub-SCT for the `test` part of the if-else-construct, so `a < 5`. It contains three calls of the `test_expression_result` function. These functions are executed in a 'narrow scope' that only considers the condition of the student code, and the condition of the solution code. + +More specifically, `test_expression_result({"a": 5})` will check whether executing the `if` condition that the student coded when `a` equals 5 leads to the same result as executing the `if` condition that is coded in the solution when `a` equals 5. That way, you can robustly check the validity of the `if` test. There are three `test_expression_result()` calls to see if the condition makes sense for different inputs. + +Suppose that the student incorrectly used the condition `a < 6` instead of `a < 5`. `test_expression_result({"a": 5})` will see what the result is of `a < 6` if `a` equals 5. The result is `True`. Next, it checks the result of `a < 5`, the `if` condition of the solution, which is `False`. There is a mismatch between the 'student result' and the 'solution result', and a meaningful feedback messages is generated. + +#### The `body` and `orelse` parts + +In a similar way, the functions that are used as lambda functions in both the `body` and `orelse` part, will also be executed in a 'narrow scope', where only the `body` and `orelse` part of the student's submission and the solution are used. + diff --git a/docs/source/pythonwhat.wiki/test_if_exp.md b/docs/source/pythonwhat.wiki/test_if_exp.md new file mode 100644 index 00000000..606845cf --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_if_exp.md @@ -0,0 +1,58 @@ +test_if_exp +----------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_if_else.test_if_exp +``` + +`test_if_exp` is a wrapper around `test_if_else`, which tells it to look for inline `if` expressions. As such, it uses the same arguments. [See `test_if_else` for more info](test_if_else). + +### What is an inline `if` expression? + +An inline `if` expression looks like.. + +```{python} +x = 'a' if True else 'b' +``` + +This is in contrast to an `if` block, which looks like.. + +```{python} +if True: + x = 'a' +else: + x = 'b' +``` + +### Parts + +This test tries to break code into 3 parts, BODY, TEST, and ORELSE. +The table below shows an example inline `if` expression on the left, +and the parts that would be extracted on the right. + +| code | parts breakdown | +| ------------------------- | --------------------- | +| `x = 'a' if True else 'b'` | `x = BODY if TEST else ORELSE` | + +### Nested `if` expressions + +Just like `test_if_else`, `test_if_exp` will not find a nested `if` expression. +Instead, the nested portion will be inside one of the parts. +For example, below is an exercise with an `if` expression in the ORELSE part of another `if` expression. + +*** =solution +```{python} +x = 'a' if True else ('b' if False else 'c') +``` + +*** =sct +```{python} +test_if_exp(orelse=lambda: test_if_exp(orelse=lambda: test_student_typed('c'))) +``` + +The SCT above tests that the student typed 'c' in the ORELSE part of the inner `if` expression. +In parts, this looks like.. + +```{python} +BODY1 if TEST1 else (ORELSE1 = BODY2 if TEST2 else ORELSE2) +``` diff --git a/docs/source/pythonwhat.wiki/test_lambda_function.md b/docs/source/pythonwhat.wiki/test_lambda_function.md new file mode 100644 index 00000000..79ffe488 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_lambda_function.md @@ -0,0 +1,86 @@ +test_lambda_function +-------------------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_lambda_function + :members: +``` + + def test_lambda_function(index, + arg_names=True, + arg_defaults=True, + body=None, + results=None, + errors=None, + not_called_msg=None, + nb_args_msg=None, + arg_names_msg=None, + arg_defaults_msg=None, + wrong_result_msg=None, + no_error_msg=None, + expand_message=True) + +With `test_function_definition()`, you can only test user-defined functions that have a name. There is an important class of functions in Python that go by the name of lambda functions. These functions are anonymous, so they don't necessarily require a name. To be able to test user-coded lambda functions, the `test_lambda_function()` is available. If you're familiar with `test_function_definition()`, you'll notice some similarities. However, instead of the `name`, you now have to pass the `index`; this means you have to specify the lambda function definition to test by number (test the first, second, third ...). Also, because we don't necessarily have an object represents a lambda function (because it can be anonymous), some tricky things are required to correctly specify the arguments `errors` and `results`; the example will give more details. + +### Example 1 + +Suppose we want the student to code a lambda function that takes two arguments, `word` and `echo`, the latter of which should have default value 1. The lambda function should return the product of `word` and `echo`. A solution to this challenge could be the following: + + *** =solution + ```{python} + echo_word = lambda word, echo = 1: word * echo + ``` + +To test this lambda function definition, you can use the following SCT: + + *** =sct + ``` + test_lambda_function(1, + body = lambda: test_student_typed('word'), + results = ["lam('test', 2)"], + errors = ["lam('a', '2')"]) + ``` + +With `1`, we tell `pythonwhat` to test the first lambda function it comes across. Through body, we can specify sub-SCTs to be tested on the body of the lambda function (similar to how `test_function_definition` does it). With `results` and `errors`, you can test the lambda function definition for different input arguments. Notice here that you have to specify a list of function calls as a string. The function you have to call is `lam()`; behind the scenes, this `lam` will be replaced by the actual lambda function the student and solution defined. This means that `lam('test', 2)` will be converted into: + + ``` + (lambda word, echo = 1: word * echo)('test', 2) + ``` + +That way, the system can run the function call, and compare the results between function and solution. Things work the same way for `errors`. + +As usual, the `test_lambda_function()` will generate a bunch of meaningful automated messages depending on which error the student made (you can override all these messages through the `*_msg` argument): + + submission: + feedback: "The system wants to check the first lambda function you defined but hasn't found it." + + submission: echo_word = lambda wrd: wrd * 1 + feedback: "You should define the first lambda function with 2 arguments, instead got 1." + + submission: echo_word = lambda wrd, echo: wrd * echo + feedback: "In your definition of the first lambda function, the first argument should be called word, instead got wrd." + + submission: echo_word = lambda word, echo = 2: word * echo + feedback: "In your definition of the first lambda function, the second argument should have 1 as default, instead got 2." + + submission: echo_word = lambda word, echo = 1: 2 * echo + feedback: "In your definition of the first lambda function, could not find the correct pattern in your code." + + submission: echo_word = lambda word, echo = 1: word * echo + 1 + feedback: "Calling the the first lambda function with arguments ('test', 2) should result in testtest, instead got an error." + + submission: echo_word = lambda word, echo = 1: word * echo * 2 + feedback = "Calling the first lambda function with arguments ('test', 2) should result in testtest, instead got testtesttesttest" + + submission: echo_word = lambda word, echo = 1: word * int(echo) + feedback: "Calling the first lambda function with the arguments ('a', '2') doesn't result in an error, but it should!" + + submission: echo_word = lambda word, echo = 1: word * echo + feedback: "Great job!" (pass) + + +### What about testing usage? + +This is practically impossible to do in a robust way; we suggest you do this in an indirect way (checking the output that should be generated, checking the object that should be created, etc). + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_object_accessed.md b/docs/source/pythonwhat.wiki/test_object_accessed.md new file mode 100644 index 00000000..014d333d --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_object_accessed.md @@ -0,0 +1,34 @@ +test_object_accessed +-------------------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_object_accessed.test_object_accessed +``` + + def test_object_accessed(name, + times=1, + not_accessed_msg=None) + +With `test_object()`, you can check whether a student correctly created an object. However, in some cases, you might also be interested whether the student actually used this object to for example assign another object. `test_object_accessed()` makes this possible; it is also possible to test object attributes. + +The `name` argument should be a string that specifies the name of the object, or the attribute of a certain object, for which you want to check if it was accesses. If the object resides inside a package, such as `pi` in the `math` package, use `"math.pi"`. With `times`, you can specify how often the object or attribute should have been accessed. With `not_accessed_msg` you can override the automatically generated feedback message in case `name` hasn't been accessed often enough according to `times`. + +### Example + +To show how everything works, suppose you have the following submission of a student: + + ``` + import numpy as np + import math as m + arr = np.array([1, 2, 3]) + x = arr.shape + print(arr.data) + print(m.e) + ``` + +Let's have a look at some SCT function calls that either pass or fail and why. + +- `test_object_accessed("arr")` - PASS: The object `arr` is accessed twice (in `arr.shape` and `arr.data`) +- `test_object_accessed("arr", times=3)` - FAIL: The objet `arr` is only accessed twice. +- `test_object_accessed("arr.shape")` - PASS: The `shape` attribute of `arr` is accessed once. +- `test_object_accessed("math.e")` - PASS: The object `e` inside the `math` package is accessed once (the student uses the alias `m`, but that is not a problem. In case of an error, the automatically generated message will take this into account.) diff --git a/docs/source/pythonwhat.wiki/test_object_after_expression.md b/docs/source/pythonwhat.wiki/test_object_after_expression.md new file mode 100644 index 00000000..43a08897 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_object_after_expression.md @@ -0,0 +1,91 @@ +test_object_after_expression +---------------------------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_object_after_expression + :members: +``` + + test_object_after_expression(name, + extra_env=None, + context_vals=None, + undefined_msg=None, + incorrect_msg=None, + eq_condition="equal", + pre_code=None, + keep_objs_in_env=None) + +`test_object_after_expression()` is a function that is primarily used to check the correctness of the body of control statements. Through `extra_env` and `context_vals` you can adapt the student/solution environment with manual elements. Next, the 'currently active expression tree', such as the body of a for loop, is executed, and the resulting environment is inspected. This is done for both the student and the solution code, and afterwards the value of the object that you specify in `name` is checked for equality. With pre_code, you can prepend the execution of the default expression tree with some extra code, for example to set some variables. + +### Example 1: Function defintion + +Suppose you want to student to code up a function `shout()`, that adds three exclamation marks to every word you pass it: + + *** =solution + ```{python} + def shout(word): + shout_word = word + '!!!' + return shout_word + ``` + +To test whether the student did this appropriately, you want to first test whether `shout` is a user-defined function, and then whether inside the function, a new variable `shout_word` is created. Finally, you also want to check whether the result of calling `shout('hello')` is correct. The following SCT will do that for us: + + *** =sct + ```{python} + test_function_definition('shout', + body = test_object_after_expression('shout_word', context_vals = ['anything']), + results = [('hello')]) + ``` + +Let's focus on the `body` argument of `test_function_definition()` here, that uses `test_object_after_expression()`. For the other elements, refer to the [`test_function_definition()`](https://github.com/datacamp/pythonwhat/wiki/test_function_definition) article. + +The first argument of `test_object_after_expression()` tells the system to check the value of `shout_word` after executing the body of the function definition. Which part of the code to execute, the 'expression', is implicitly specified by `pythonwhat`. However, to run correctly, this expression has to know what `word` is. You can specify a value of `word` through the `context_vals` argument. It's a simple list: the first element of the list will be the value for the first argument of the function definition, the second element of the list will be the value for the second argument of the list, and so on. Here, there's only one argument, so a list with a single element, a string (that will be the value of the `word` variable), suffises. + +`test_object_after_expression()` will execute the expression, and run it on the solution side and the student side. On the solution side, the value of `shout_word` after the execution will be `'anything!!!'`. If the value on the student code is the same, we can rest assured that `shout_word` has been appropriately defined by the student and the test passes. + +### Example 2: for loop + +Suppose you want the student to build up a dictionary of word counts based on a list, as follows: + + *** =solution + ```{python} + words = ['it', 'is', 'a', 'the', 'is', 'the', 'a', 'the', 'it'] + counts = {} + for word in words: + if word in counts: + counts[word] += 1 + else: + counts[word] = 1 + ``` + +To check whether the `counts` list was correctly built, you can simply use `test_object()`, but you can also go deeper if it goes wrong. This calls for a `test_correct()` in combination with `test_object()` and `test_for()`, that in its turn uses `test_object_after_expression()`: + + + ``` =solution + def check_test(): + test_object('counts') + + def diagnose_test(): + body_test(): + test_object_after_expression('counts', + extra_env = {'counts': {'it': 1}}, + context_vals = ['it']) + test_object_after_expression('counts', + extra_env = {'counts': {'it': 1}}, + context_vals = ['is']) + test_for_loop(index = 1, + test = test_expression_result(), # Check if correct iterable used + body = body_test) + + test_correct(check_test, diagnose_test) + ``` + +Let's focus on the `body_test` for the for loop. Here, we're using `test_object_after_expression()` twice. + +In the first function call, we override the environment so that `counts` is a dictionary with a single key and value. Also, the context value, `word` in this case (the iterator of the `for` loop), is set to `it`. In this case, the body of the for loop - making abstraction of the if-else test - should increment the value of the value, without adding a new key. + +In the second function call, we override the environment so that `counts` is again a dictionary with a single key and value. This time, the context value is set to `is`, so a value that is not yet in the `counts` dictionary, so this should lead to a `counts` dictionary with two elements. + +As in the first example, `test_object_after_expression()` sets the environment variables and context values, runs the expression (in this case the entire body of the `for` loop), and then inspects the value of `counts` after this expression. The combination of the two `test_object_after_expression()` calls here, will indirectly check whether both the if and else part of the body has been correctly implemented. + + diff --git a/docs/source/pythonwhat.wiki/test_operator.md b/docs/source/pythonwhat.wiki/test_operator.md new file mode 100644 index 00000000..75f35668 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_operator.md @@ -0,0 +1,62 @@ +test_operator +------------- + +**THIS FUNCTION IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE** + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_operator + :members: + +``` + + def test_operator(index=1, + eq_condition="equal", + used=None, + do_eval=True, + not_found_msg=None, + incorrect_op_msg=None, + incorrect_result_msg=None) + +Suppose you want the student to do some very basic operations using the `*` and the `**` operator. You could just ask the student to do some calculations and assign the result to a variable, `result` for example, and check that variable using `test_object()`. However this won't allow you to give the student very tailored feedback. Suppose you want to check if the student uses `**` and tell him/her if he/she doesn't! `test_object()` won't allow you to check this kind of specifics as it only checks resulting objects in both processes. Luckily, you can use another helper function, `test_operator()`. + +Say you want the student to calculate the future value of \$100 after 6 years. The interest rate 6% and you are using compound interest. This means the result has to be `100 * 1.06 ** 6`, so the solution code would be. + + *** =solution + ```{python} + # Calculate the future value of 100 dollar: result + result = 100 * 1.06 ** 6 + + # Print out the result + print(result) + ``` + +The SCT might look something like this, + + *** =sct + ```{python} + test_operator(index=1) + test_object("result") + test_function("print") + success_msg("Great!") + ``` + +You can learn about `test_object()` and `test_function()` in the other articles, so those won't be deatiled here. Let's focus on `test_operator()` instead. This function will extract the first operator group from the solution code (`100 * 1.06 ** 6`), run it in the solution process, and compare the result with the result from running the first operator in the student code in the student process. In total, three steps will be tested: + +- Did the student define enough operations? +- Does the student use the same operators as the solution? +- Is the result of the operation for the student the same as the one in the solution? + +`test_operator()` takes some additional arguments for further customization and tailored feedback messages. For example, you can use it as follows to just check whether the student used the `**` operator in his/her first operation and give custom feedback: + + *** =sct + ```{python} + test_operator(index=1, used=["**"], do_eval=False, + incorrect_op_msg="A little tip: you should use `**` to do this calculation.") + test_object("result") + test_function("print") + success_msg("Great!") + ``` + +This SCT will be more forgiving, but the result is still checked with `test_object()` so the student will still have to calculate the correct value. This time, however, it is not checked by `test_operator()` because `do_eval = False`. `used = ["**"]` is used to tell the system to only check on the `**` operator for the first operation group. + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/pythonwhat.wiki/test_try_except.md b/docs/source/pythonwhat.wiki/test_try_except.md new file mode 100644 index 00000000..0b210b56 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_try_except.md @@ -0,0 +1,71 @@ +test_try_except +--------------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_try_except + :members: +``` + + def test_try_except(index=1, + not_called_msg=None, + body=None, + handlers={}, + except_missing_msg = None, + orelse=None, + orelse_missing_msg=None, + finalbody=None, + finalbody_missing_msg=None, + expand_message=True) + +With `test_try_except`, you can check whether the student correctly coded a `try-except` block. + +As usual, `index` controls which try-except block to check. With `not_called_msg` you can choose a custom message to override the automatically defined message in case not enough try-except blocks weren't found in the student code. `body` is a sub-sct to test the code of the `try` block. `orelse` and `finalbody` work the same way, but here there are also `_msg` arguments to provide custom messages in case these parts ar missing. Finally, there's also `handlers` and `except_missing_msg`. `handlers` should be a dictionary, where the keys are the error classes you expect the student to capture (for the general `except:`, use `'all'`), and the values are sub-SCTs for each of these `except` blocks. An `except` block is only checked for existence and correctness if you mention it inside `handlers`. If it is not available, an automatic message will be generated, but this can ge overriden with `expect_missing_msg`. + + +Note: For more information on sub-SCTs, visit [the dedicated article](https://github.com/datacamp/pythonwhat/wiki/Sub-SCTs). + +### Example 1 + +Suppose you want to student to code up the following (completely useless) piece of Python code: + + *** =solution + ```{python} + try: + x = max([1, 2, 'a']) + except TypeError as e: + x = 'typeerror' + except ValueError: + x = 'valueerror' + except (ZeroDivisionError, IOError) as e: + x = e + except : + x = 'someerror' + else : + passed = True + finally: + print('done') + ``` + +To test each and every part of this model solution, you can use the following SCT: + + *** =sct + ```{python} + import collections + handlers = collections.OrderedDict() + handlers['TypeError'] = lambda: test_object_after_expression('x') + handlers['ValueError'] = lambda: test_object_after_expression('x') + handlers['ZeroDivisionError'] = lambda: test_object_after_expression('x', context_vals = ['anerror']) + handlers['IOError'] = lambda: test_object_after_expression('x', context_vals = ['anerror']) + handlers['all'] = lambda: test_object_after_expression('x') + test_try_except(index = 1, + body = lambda: test_function("max"), + handlers = handlers, + orelse = lambda: test_object_after_expression('passed'), + finalbody = lambda: test_function('print')) + ``` + +Notice that: + +- We use the `OrderedDict()` from the `collections` module so that the dictionary we pass in the `handlers` argument is always gone through in the same order. +- We can use `context_vals` to initalize the context value, `e` in this case. + diff --git a/docs/source/pythonwhat.wiki/test_while_loop.md b/docs/source/pythonwhat.wiki/test_while_loop.md new file mode 100644 index 00000000..ea8a90bd --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_while_loop.md @@ -0,0 +1,37 @@ +test_while_loop +--------------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_while_loop + :members: +``` + + test_while_loop(index=1, + test=None, + body=None, + orelse=None, + expand_message=True) + +Since a lot of the logic of `test_if_else()` and `test_for_loop()` can be applied to `test_while_loop()`, this article is limited to an example. For more info see the wiki on `test_if_else()` and `test_for_loop()`, or the documentation of `test_while_loop()`. + + *** =solution + ```{python} + a = 10 + while a > 5: + print("%s is bigger than 5" % a) + a -= 1 + ``` + + *** =sct + ```{python} + def sct_on_condition_test(): + test_expression_result({"a": 4}) + test_expression_result({"a": 5}) + test_expression_result({"a": 6}) + + test_while_loop(index = 1, + test = sct_on_condition_test, + body = lambda: test_expression_output({"a":4})) + ``` + + diff --git a/docs/source/pythonwhat.wiki/test_with.md b/docs/source/pythonwhat.wiki/test_with.md new file mode 100644 index 00000000..a52df604 --- /dev/null +++ b/docs/source/pythonwhat.wiki/test_with.md @@ -0,0 +1,59 @@ +test_with +--------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_with.test_with +``` + + def test_with(index, + context_vals=False, + context_tests=None, + body=None, + undefined_msg=None, + context_vals_len_msg=None, + context_vals_msg=None, + expand_message=True) + +In Python, one can build so-called context managers with the `with` statement. + +Have a look at an example of such a context manager: + + with open('something.txt') as file1, open('something_else.csv') as file2: # the contexts + # body of the with statement + # do something with file1 and file2 + # ... + +Two important parts can be distinguished: the contexts that are being opened and the body, in which operations are done with these contexts. In this example, two contexts are defined: `open('something.txt')` and `open('something_else.csv')`. The context can be given names (this is optional). In the example, the first one will be called `file1` after the `with` statement, and the second one `file2`. + +`test_with()` is written to allow you to test all these parts of the `with` statement separately. + +### Example 1 + +Suppose you want the student to code something as follows: + + *** =solution + ```{python} + with open('moby_dick.txt') as moby, open('lotr.txt') as lotr: + print("First line of Moby Dick: %r." % moby.readline()) + print("First line of The Lord of The Rings: The Two Towers: %r." % lotr.readline()) + + +In this case you want to test two things: you want the student to open up the correct context and you want them to print out the correct information. Let's assume that how the context are named is not important to you. The solution uses `moby` and `lotr`, but the student can use any name he or she wants. Note these names will not be tested by default, but you can change that by setting `context_vals = True`. + +In the SCT, we specify a sub-SCT for `context_tests` and for `body`. The former tests the contexts, the latter tests the body. As before, you can specify these sub-SCTs through lambda functions or a separate function definition: + + *** =sct + ```{python} + def test_with_body(): + test_function('print', 1) + test_function('print', 2) + + test_with(1, + context_tests = [ + lambda: test_function('open'), + lambda: test_function('open') + ], + body = test_with_body) + ``` + +Different from before, htough, `context_tests` expects a list of lambda functions or customly defined functions. The index in this list of functions represents the context against which the SCTs will be tested. The first lambda/custom function in `context_tests` will be tested against the first context. The second lambda/custom function in `context_tests` will be tested against the second context. If only one function is given in `context_tests`, only the first context will be tested. The `body` argument requires one lambda/custom function to be passed, this contains the sub-SCT that is run against the `with` statements' body. diff --git a/docs/source/simple_tests.md b/docs/source/simple_tests.md deleted file mode 100644 index add2c8a0..00000000 --- a/docs/source/simple_tests.md +++ /dev/null @@ -1,3 +0,0 @@ -Simple Tests -================ - diff --git a/docs/source/simple_tests/index.rst b/docs/source/simple_tests/index.rst new file mode 100644 index 00000000..a6c0fbbd --- /dev/null +++ b/docs/source/simple_tests/index.rst @@ -0,0 +1,16 @@ +Simple Tests +============ + +Simple tests are the most basic tests available in pythonwhat. +They don't focus on specific pieces of a submission (like part checks [LINK]), or re-run any code (like expression tests [LINK]). +Instead, they simply look at things like imports, printed output, or raw code text. +A final, common use is to test the value of a variable in the final environment (that is, after the submission of solution code have been run). + +.. toctree:: + :maxdepth: 2 + + test_import + test_object + test_output_contains + test_student_typed + test_mc diff --git a/docs/source/simple_tests/test_import.md b/docs/source/simple_tests/test_import.md new file mode 100644 index 00000000..9b199cff --- /dev/null +++ b/docs/source/simple_tests/test_import.md @@ -0,0 +1,52 @@ +test_import +----------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_import + :members: +``` + + def test_import(name, + same_as=True, + not_imported_msg=None, + incorrect_as_msg=None): + +With `test_import` you can test whether a student correctly imported a certain package. As an option, you can also specify whether or not the same alias should be used. + +Python features many ways to import packages. All of these different methods revolve around the `import`, `from` and `as` keywords. Suppose you want students to import `matplotlib.pyplot` as `plt` (the common way of importing the plotting tools in `matplotlib`. A possible solution of your exercises could be the following: + + *** =solution + ```{python} + # Import plotting tools + import matplotlib.pyplot as plt + ``` + +Below is a possible SCT for this exercise: + + *** =sct + ```{python} + test_import("matplotlib.pyplot") + success_msg("You nailed it!") + ``` + +Here, `test_import` will parse both the student's submission as well as the solution, and figure out which packages were imported and how. Next, it checks if the `matplotlib.pyplot` package was imported and under which alias. If the student did this and imported it as `plt`, all is good. If, however, the student submitted `import matplotlib` (import entire package instead of module) or `import matplotlib.pyplot as pppplot` (incorrect alias), `test_import()` will fail and generate the appropriate messages. + +As usual, you can override these messages with your own: + + *** =sct + ```{python} + test_import("matplotlib.pyplot"), + not_imported_msg = "You can import pyplot by using `import matplotlib.pyplot`.", + incorrect_as_msg = "You should set the correct alias for `matplotlib.pyplot`, import it `as plt`.") + success_msg("You nailed it!") + ``` + +With `same_as`, you can control whether or not the alias should be exactly the same. By default `same_as=True`, so the alias (`plt` in the example) should also be used by student. If you set it to `False`: + + *** =sct + ```{python} + test_import("matplotlib.pyplot", same_as=False) + success_msg("You nailed it!") + ``` + +The SCT will also pass if the student uses `import matplotlib.pyplot as pppplot`, a submission that wouldn't be accepted if `same_as=True`. diff --git a/docs/source/simple_tests/test_mc.md b/docs/source/simple_tests/test_mc.md new file mode 100644 index 00000000..7d9f8154 --- /dev/null +++ b/docs/source/simple_tests/test_mc.md @@ -0,0 +1,34 @@ +test_mc +------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_mc + :members: +``` + + test_mc(correct, msgs) + +Multiple choice exercises are straightforward to test. Use `test_mc()` to provide tailored feedback for both the incorrect options, as the correct option. Below is the code for a multiple choice exercise example, with an SCT that uses `test_mc`: + + --- type:MultipleChoiceExercise lang:python xp:50 skills:2 + ## The author of Python + + Who is the author of the Python programming language? + + *** =instructions + - Roy Co + - Ronald McDonald + - Guido van Rossum + + *** =hint + Just google it! + + *** =sct + ```{python} + test_mc(correct = 3, + msgs = ["That's someone who makes soups.", + "That's a clown who likes burgers.", + "Correct! Head over to the next exercise!"]) + ``` + +The first argument of `test_mc()`, `correct`, should be the number of the correct answer in this list. Here, the correct answer is Guido van Rossum, corresponding to 3. The `msgs` argument should be a list of strings with a length equal to the number of options. We encourage you to provide feedback messages that are informative and tailored to the (incorrect) option that people selected. Make sure to correctly order the feedback message such that it corresponds to the possible answers that are listed in the instructions tab. Notice that there's no need for `success_msg()` in multiple choice exercises, as you have to specify the success message inside `test_mc()`, along with the feedback for incorrect options. diff --git a/docs/source/simple_tests/test_object.md b/docs/source/simple_tests/test_object.md new file mode 100644 index 00000000..21520501 --- /dev/null +++ b/docs/source/simple_tests/test_object.md @@ -0,0 +1,99 @@ +test_object +----------- + +```eval_rst +.. autofunction:: pythonwhat.test_funcs.test_object.test_object +``` + + test_object(name, + eq_condition="equal", + do_eval=True, + undefined_msg=None, + incorrect_msg=None) + +`test_object()` enables you to test whether a student correctly defined an object. + +As explained on the [wiki home](https://github.com/datacamp/pythonwhat/wiki), both the student's submission as well as the solution code are executed, in separate processes. `test_object()` looks at these processes and checks if the object specified in `name` is available in the student process. Next, it checks whether the object in the student and solution process correspond. In case of a failure along the way, `test_object()` will generate a meaningful feedback message that you can override. + +### Example 1 + +Suppose we have the following solution: + + *** =solution + ```{python} + # Create a variable x, equal to 3 * 5 + x = 3 * 15 + ``` + +To test this we simply use: + + *** =sct + ```{python} + test_object("x") + success_msg("Great job!") + ``` + +This SCT will test if the variable `x` is defined, and has the same ending value in the student process as in the solution process. All of the following student submissions would be accepted by `test_object()`: + +- `x = 15` +- `x = 12 + 3` +- `x = 3; x += 12` + +How the object `x` came about in the student's submission, does not matter: only the end result, the actual content of `x`, matters. + +`do_eval=True` by default; if you set it to `False`, only the existence of an object `x` will be checked; its contents will not be compared to the object `x` that's in the solution process. + +### Object equality + +When comparing more complex objects in Python, chances are they don't use the equality operation you desire. Python objects are compared using the `==` operator, and objects can overwrite its implementation to fit the object's needs. Internally, `test_object()` uses the `==` operation to compare objects, this means you could encounter undesirable behaviour. Sometimes `==` just compares the actual object instances, and objects which are semantically alike wont be according to `test_object()`. + +Say, for example, that you have the following solution: + + *** =solution + from urllib.request import urlretrieve + fn1 = 'https://s3.amazonaws.com/assets.datacamp.com/production/course_998/datasets/Chinook.sqlite' + urlretrieve(fn1, 'Chinook.sqlite') + + # Import packages + from sqlalchemy import create_engine + import pandas as pd + + # Create engine: engine + engine = create_engine('sqlite:///Chinook.sqlite') + + # Open engine connection + con = engine.connect() + +An SCT for this exercise could be the following: + + *** =sct + test_object("engine") + test_object("con") + +Now, if the student enters the exact same code as the solution, the SCT will still fail. How? Well if you try this out: `create_engine('sqlite:///Chinook.sqlite') == create_engine('sqlite:///Chinook.sqlite')` you will notice that it returns `False`. This means the exact same execution doesn't lead to the the exact same object (although they might be semantically equal). We can't use `test_object` like that here. There are several ways to solve this: + +#### Workaround + + *** =sct + test_object("engine", do_eval=False) + test_function("create_engine") + test_object("con", do_eval=False) + test_function("engine.connect") + +This will check whether the objects `engine` and `con` are declared, without checking for it's value. With `test_function()` we check whether they used the correct functions. This will not test the exact same thing as the first SCT, but it's effective 99% of the time. + +#### Equality operations hardcoded in `pythonwhat` + +A side note here, complex objects that are used a lot have an custom implementation of equality built for them in `pythonwhat`. These objects can be tested with a regular `test_object(...)`, without having to use `do_eval=False`. At the moment, the more complex classes that can be tested are: + +- `numpy.ndarray` +- `pandas.DataFrame` +- `pandas.Series` + +Of course primitive classes like `str`, `int`, `list`, `dict`, ... can be tested without any problems too, as well as objects of which the class has a semantically correct implementation of the `==` operator. + +#### Manually define a converter + +As explained in the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes), objects are extracted from their respected processes by 'dilling' and 'undilling' them. However, you can manually set a 'converter' with the `set_converter()` function. This will override the default dilling and undilling behavior, and enables you to make simplified representations of custom objects, testing only exactly what you want to test. Learn more about it [here](https://github.com/datacamp/pythonwhat/wiki/Processes). + +**NOTE**: Behind the scenes, `pythonwhat` has to fetch the value of objects from sub-processes. The required 'dilling' and 'undilling' can cause issues for exotic objects. For more information on this and possible errors that can occur, read the [Processes article](https://github.com/datacamp/pythonwhat/wiki/Processes). diff --git a/docs/source/simple_tests/test_output_contains.md b/docs/source/simple_tests/test_output_contains.md new file mode 100644 index 00000000..75bc8ae4 --- /dev/null +++ b/docs/source/simple_tests/test_output_contains.md @@ -0,0 +1,40 @@ +test_output_contains +-------------------- + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_output_contains + :members: +``` + + test_output_contains(text, + pattern=True, + no_output_msg=None) + +We can test the output of the student contains with `test_output_contains()`. This function will compare the given text with the text in the student's output and see if we have a match. You can use regular expressions or not, that's completely up to you. + +Here's an example of an exercise with `test_student_typed()`, suppose the solution looks like this: + + *** =solution + ```{python} + # Print the "This is some ... stuff" to the shell + print("This is some weird stuff") + ``` + +The following SCT tests whether the student outputs `This is some weird stuff`: + + *** =sct + ```{python} + test_output_contains("This is some weird stuff", pattern = False) + success_msg("Great job!") + ``` + +Notice that we set `pattern` to `False`, this will cause `test_output_contains()` to search for the pure string, no patterns are used. This SCT is not robust, because it won't be accepted if the student submits `print("This is some cool stuff")`, for example. Therefore, it's a good idea to use [regular expressions](https://docs.python.org/3.5/library/re.html). `pattern=True` by default, so there's no need to specify this: + + *** =sct + ```{python} + test_output_contains(/This is some \w* stuff/, + no_output_msg = "Print out `This is some ... stuff` to the output, fill in `...` with a word you like.") + success_msg("Great job!") + ``` + +Now, different printouts will be accepted. Notice that we also specified `no_output_msg` here. If the pattern is not found in the output generated, this message will be shown instead of a message that's automatically generated by `pythonwhat`. diff --git a/docs/source/simple_tests/test_student_typed.md b/docs/source/simple_tests/test_student_typed.md new file mode 100644 index 00000000..9b5f73c0 --- /dev/null +++ b/docs/source/simple_tests/test_student_typed.md @@ -0,0 +1,44 @@ +test_student_typed +------------------ + +```eval_rst +.. automodule:: pythonwhat.test_funcs.test_student_typed + :members: +``` + + test_student_typed(text, + pattern=True, + not_typed_msg=None) + +`test_student_typed()` will look through the student's submission to find a match with the string specified in `text`. With `pattern`, you can declare whether or not to use regular expressions. + +Suppose the solution of an exercise looks like this: + + *** =solution + ```{python} + # Calculate the sum of all single digit numbers and assign the result to 's' + s = sum(range(10)) + + # Print the result to the shell + print(s) + ``` + +The following SCT tests whether the student typed `"sum(range("`: + + *** =sct + ```{python} + test_student_typed("sum(range(", pattern = False) + success_msg("Great job!") + ``` + +Notice that we set `pattern` to `False`, this will cause `test_student_typed()` to search for the pure string, no patterns are used. This SCT is not that robust though, it won't accept something like `sum( range(10) )`. This is why we should almost always use [regular expressions](https://docs.python.org/3.5/library/re.html) in `test_student_typed`. For example: + + *** =sct + ```{python} + test_student_typed("sum\s*\(\s*range\s*\(", not_typed_msg="You didn't use `range()` inside `sum()`.") + success_msg("Great job!") + ``` + +We also used `not_typed_msg` here, which will control the feedback given to the student when `test_student_typed()` doesn't pass. Note that also `success_msg()` is used here, this is the message that is shown when the SCT has passed. + +In general, **you should avoid using `test_student_typed()`**, as it imposes severe restrictions on how a student can solve an exercise. Often, there are different ways to solve an exercise. Unless you have a very advanced regular expression, `test_student_typed()` will not be able to accept all these different approaches. For the example above, `test_function()` would be more appropriate. diff --git a/docs/source/spec2_summary.rst b/docs/source/spec2_summary.rst index 4d4f7551..934a38cf 100644 --- a/docs/source/spec2_summary.rst +++ b/docs/source/spec2_summary.rst @@ -5,8 +5,8 @@ Spec2 Improvements :language: python -Lambdaless test_ functions -------------------------- +Lambdaless test functions +--------------------------- Sometimes you want to pass a test function as an argument to another test function. For the examples below, we'll use the following solution code @@ -127,7 +127,7 @@ That is, if you want ``test_if_exp`` below to run immediately, do not write and expect the SCTs to run in a predictable order. -If you want create a bunch of sub-tests, but don't want to preface each with F(), you can use the pythonwhat v2 function multi, as below. +If you want to create a bunch of sub-tests, but don't want to preface each with F(), you can use the pythonwhat v2 function multi, as below. .. code:: @@ -137,19 +137,90 @@ If you want create a bunch of sub-tests, but don't want to preface each with F() Context values for nested parts ------------------------------- +Context values may now be defined for nested parts. For example, the print statement below, + +.. code:: + + for i in range(2): # outer for loop part + for j in range(3): # inner for loop part + print(i + j) + +may be tested by setting context values at each level, + +.. code:: + + (Ex() + .check_for_loop(0).check_body().set_context(i = 1) # outer for + .check_for_loop(0).check_body().set_context(j = 2) # inner for + .has_equal_output() + ) + +For more on context valus see [PROCESSES LINK HERE]. + Can call code chunks that before could only be split up ------------------------------------------------------- +Entire code pieces, such as the inline if statement below, + +.. code:: + + 'yes' if True else 'no' + +may be tested using something like, + +.. code:: + + Ex().check_if_exp(0).has_equal_value() + Argument checking ----------------- +The arguments of a function definition, such as + +.. code:: + + def f(a=1): print(a) + +are now parts and may be checked as below.. + +.. code:: + + (Ex().check_function_def('f') # does f exist? + .check_args('a') # does a exist? + .is_default() # is it a default argument? + .has_equal_value() # is it's default equal to solution? + ) + +For more on the argument part, see [PART CHEATSHEET LINK HERE]. + + Deprecate test_expression_result and friends -------------------------------------------- -Long live `has_equal_value`, `has_equal_output`, `has_equal_error` +In pythonwhat v1, the functions + +* test_expression_result +* test_expression_output +* test_object_after_expression + +and various arguments of test_function_definition, test_lambda_function ran code and then +checked the result, printed output, or errors against eachother. + +These functions have been deprecated in favor of similar function.. + +* `has_equal_value` +* `has_equal_output` +* `has_equal_error` + +These functions include identical arguments as the above. Feedback messages may use templating (via str.format or Jinja2) ----------------------------------------------------------------- +**This feature is not stable, and should not be used in production** + + Cleaned up internals -------------------- + +Yayyyy. diff --git a/docs/source/test_functions.rst b/docs/source/test_functions.rst deleted file mode 100644 index 5d88f512..00000000 --- a/docs/source/test_functions.rst +++ /dev/null @@ -1,128 +0,0 @@ -Old Syntax -================= - -Simple ------- - -test_import -~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_import - :members: - -test_mc -~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_mc - :members: - -test_object -~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_object - :members: - -test_operator -~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_operator - :members: - -test_output_contains -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_output_contains - :members: - -test_student_typed -~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_student_typed - :members: - -test_expression_output -~~~~~~~~~~~~~~~~~~~~~~ - -.. autofunction:: pythonwhat.test_funcs.test_expression_output.test_expression_output - - -test_expression_result -~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_expression_result - :members: - -test_object_after_expression -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_object_after_expression - :members: - - - -Composite ---------- - -test_for_loop -~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_for_loop - :members: - -test_if_else -~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_if_else - :members: - -test_while_loop -~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_while_loop - :members: - -test_comp -~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_comp - :members: - -test_try_except -~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_try_except - :members: - -test_function_definition -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_function_definition - :members: - -test_function -~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_function - :members: - -test_lambda_function -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_lambda_function - :members: - - -Logic ------- - -test_correct -~~~~~~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_correct - :members: - -test_or -~~~~~~~ - -.. automodule:: pythonwhat.test_funcs.test_or - :members: diff --git a/pythonwhat/test_funcs/test_correct.py b/pythonwhat/test_funcs/test_correct.py index 5edc6a05..91c110be 100644 --- a/pythonwhat/test_funcs/test_correct.py +++ b/pythonwhat/test_funcs/test_correct.py @@ -5,6 +5,9 @@ from .test_or import test_or def test_correct(check, diagnose, state=None): + """Allows feedback from a diagnostic SCT, only if a check SCT fails. + + """ rep = Reporter.active_reporter rep.set_tag("fun", "test_correct") diff --git a/pythonwhat/test_funcs/test_or.py b/pythonwhat/test_funcs/test_or.py index 24e41ed3..83b26032 100644 --- a/pythonwhat/test_funcs/test_or.py +++ b/pythonwhat/test_funcs/test_or.py @@ -3,6 +3,8 @@ from pythonwhat.check_funcs import multi def test_or(*tests, state=None): + """Test whether at least one SCT passes.""" + rep = Reporter.active_reporter rep.set_tag("fun", "test_or")