Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue a warning when a parameter is not consumed by a task #3170

Merged
merged 1 commit into from
Jun 4, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions luigi/parameter.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ class OptionalParameterTypeWarning(UserWarning):
pass


class UnconsumedParameterWarning(UserWarning):
"""Warning class for parameters that are not consumed by the task."""


class Parameter:
"""
Parameter whose value is a ``str``, and a base class for other parameter types.
Expand Down
19 changes: 19 additions & 0 deletions luigi/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@

import luigi

from luigi import configuration
from luigi import parameter
from luigi.task_register import Register
from luigi.parameter import ParameterVisibility
from luigi.parameter import UnconsumedParameterWarning

Parameter = parameter.Parameter
logger = logging.getLogger('luigi-interface')
Expand Down Expand Up @@ -429,6 +431,23 @@ def list_to_tuple(x):
return tuple(x)
else:
return x

# Check for unconsumed parameters
conf = configuration.get_config()
if not hasattr(cls, "_unconsumed_params"):
cls._unconsumed_params = set()
if task_family in conf.sections():
for key, value in conf[task_family].items():
composite_key = f"{task_family}_{key}"
if key not in result and composite_key not in cls._unconsumed_params:
warnings.warn(
"The configuration contains the parameter "
f"'{key}' with value '{value}' that is not consumed by the task "
f"'{task_family}'.",
UnconsumedParameterWarning,
)
cls._unconsumed_params.add(composite_key)

# Sort it by the correct order and make a list
return [(param_name, list_to_tuple(result[param_name])) for param_name, param_obj in params]

Expand Down
49 changes: 49 additions & 0 deletions test/task_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,55 @@ class ATaskWithBadParam(luigi.Task):
with self.assertRaisesRegex(ValueError, r"ATaskWithBadParam\[args=\(\), kwargs={}\]: Error when parsing the default value of 'bad_param'"):
ATaskWithBadParam()

@with_config(
{
"TaskA": {
"a": "a",
"b": "b",
"c": "c",
},
"TaskB": {
"a": "a",
"b": "b",
"c": "c",
},
}
)
def test_unconsumed_params(self):
class TaskA(luigi.Task):
a = luigi.Parameter(default="a")

class TaskB(luigi.Task):
a = luigi.Parameter(default="a")

with warnings.catch_warnings(record=True) as w:
warnings.filterwarnings(
action="ignore",
category=Warning,
)
warnings.simplefilter(
action="always",
category=luigi.parameter.UnconsumedParameterWarning,
)

TaskA()
TaskB()

assert len(w) == 4
expected = [
("b", "TaskA"),
("c", "TaskA"),
("b", "TaskB"),
("c", "TaskB"),
]
for i, (expected_value, task_name) in zip(w, expected):
assert issubclass(i.category, luigi.parameter.UnconsumedParameterWarning)
assert str(i.message) == (
"The configuration contains the parameter "
f"'{expected_value}' with value '{expected_value}' that is not consumed by "
f"the task '{task_name}'."
)


class ExternalizeTaskTest(LuigiTestCase):

Expand Down