From fa323b4865543a16f7a2e2a5d1621400ecc55544 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 30 Oct 2025 22:40:06 +0100 Subject: [PATCH 1/5] Fix passing repeated tasks to the functional interface. --- .../how_to_guides/functional_interface.ipynb | 268 ++++++++++++++++-- src/_pytask/collect.py | 48 +++- tests/test_execute.py | 58 ++++ ..._repeated_tasks_functional_interface.ipynb | 80 ++++++ 4 files changed, 423 insertions(+), 31 deletions(-) create mode 100644 tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb diff --git a/docs/source/how_to_guides/functional_interface.ipynb b/docs/source/how_to_guides/functional_interface.ipynb index 76b6eedc..b7e26244 100644 --- a/docs/source/how_to_guides/functional_interface.ipynb +++ b/docs/source/how_to_guides/functional_interface.ipynb @@ -85,11 +85,11 @@ { "data": { "text/html": [ - "
Platform: darwin -- Python 3.12.2, pytask 0.5.4.dev16+g8ed43db.d20250607, pluggy 1.6.0\n",
+       "
Platform: darwin -- Python 3.12.12, pytask 0.5.6.dev17+g2a33ab7d5.d20251030, pluggy 1.6.0\n",
        "
\n" ], "text/plain": [ - "Platform: darwin -- Python \u001b[1;36m3.12\u001b[0m.\u001b[1;36m2\u001b[0m, pytask \u001b[1;36m0.5\u001b[0m.\u001b[1;36m4.\u001b[0mdev16+g8ed43db.d20250607, pluggy \u001b[1;36m1.6\u001b[0m.\u001b[1;36m0\u001b[0m\n" + "Platform: darwin -- Python 3.12.12, pytask 0.5.6.dev17+g2a33ab7d5.d20251030, pluggy 1.6.0\n" ] }, "metadata": {}, @@ -121,11 +121,11 @@ { "data": { "text/html": [ - "
Collected 3 tasks.\n",
+       "
Collected 3 tasks.\n",
        "
\n" ], "text/plain": [ - "Collected \u001b[1;36m3\u001b[0m tasks.\n" + "Collected 3 tasks.\n" ] }, "metadata": {}, @@ -147,7 +147,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "59c1696ceef94890b5833506da638d2d", + "model_id": "35f4210e20704ca7b8130462a887bebf", "version_major": 2, "version_minor": 0 }, @@ -174,9 +174,9 @@ "
╭─────────────────────────┬─────────╮\n",
        "│ Task                     Outcome │\n",
        "├─────────────────────────┼─────────┤\n",
-       "│ task_create_first_file .       │\n",
-       "│ task_create_second_file.       │\n",
-       "│ task_merge_files       .       │\n",
+       "│ task_create_first_file .       │\n",
+       "│ task_create_second_file.       │\n",
+       "│ task_merge_files       .       │\n",
        "╰─────────────────────────┴─────────╯\n",
        "
\n" ], @@ -184,9 +184,9 @@ "╭─────────────────────────┬─────────╮\n", "│\u001b[1m \u001b[0m\u001b[1mTask \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mOutcome\u001b[0m\u001b[1m \u001b[0m│\n", "├─────────────────────────┼─────────┤\n", - "│ \u001b]8;id=761467;file:///None\u001b\\task_create_first_file \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", - "│ \u001b]8;id=184911;file:///None\u001b\\task_create_second_file\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", - "│ \u001b]8;id=880445;file:///None\u001b\\task_merge_files \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=780914;file://None\u001b\\task_create_first_file \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=865151;file://None\u001b\\task_create_second_file\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=487888;file://None\u001b\\task_merge_files \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", "╰─────────────────────────┴─────────╯\n" ] }, @@ -241,11 +241,11 @@ { "data": { "text/html": [ - "
──────────────────────────────────────────── Succeeded in 0.1 seconds ─────────────────────────────────────────────\n",
+       "
──────────────────────────────────────────── Succeeded in 0.05 seconds ────────────────────────────────────────────\n",
        "
\n" ], "text/plain": [ - "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.1 seconds\u001b[0m\u001b[38;2;19;124;57m ─────────────────────────────────────────────\u001b[0m\n" + "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.05 seconds\u001b[0m\u001b[38;2;19;124;57m ────────────────────────────────────────────\u001b[0m\n" ] }, "metadata": {}, @@ -273,7 +273,7 @@ { "data": { "text/plain": [ - "Session(config={'pm': , 'markers': {'filterwarnings': 'Add a filter for a warning to a task.', 'persist': 'Prevent execution of a task if all products exist and even if something has changed (dependencies, source file, products). This decorator might be useful for expensive tasks where only the formatting of the file has changed. The state of the files which have changed will also be remembered and another run will skip the task with success.', 'skip': 'Skip a task and all its dependent tasks.', 'skip_ancestor_failed': 'Internal decorator applied to tasks if any of its preceding tasks failed.', 'skip_unchanged': 'Internal decorator applied to tasks which have already been executed and have not been changed.', 'skipif': 'Skip a task and all its dependent tasks if a condition is met.', 'task': 'Mark a function as a task regardless of its name. Or mark tasks which are repeated in a loop. See this tutorial for more information: [link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/].', 'try_first': 'Try to execute a task a early as possible.', 'try_last': 'Try to execute a task a late as possible.'}, 'config': None, 'database_url': sqlite:////Users/tobiasr/git/pytask/.pytask/pytask.sqlite3, 'editor_url_scheme': 'file', 'export': <_ExportFormats.NO: 'no'>, 'hook_module': None, 'ignore': ['.codecov.yml', '.gitignore', '.pre-commit-config.yaml', '.readthedocs.yml', '.readthedocs.yaml', 'readthedocs.yml', 'readthedocs.yaml', 'environment.yml', 'pyproject.toml', 'setup.cfg', 'tox.ini', '.git/*', '.venv/*', '.pixi/*', '*.egg-info/*', '.ipynb_checkpoints/*', '.mypy_cache/*', '.nox/*', '.tox/*', '_build/*', '__pycache__/*', 'build/*', 'dist/*', 'pytest_cache/*'], 'paths': [], 'layout': 'dot', 'output_path': 'dag.pdf', 'rank_direction': <_RankDirection.TB: 'TB'>, 'expression': '', 'marker_expression': '', 'nodes': False, 'strict_markers': False, 'directories': False, 'exclude': [None, '.git/*', '/Users/tobiasr/git/pytask/.pytask/*'], 'mode': <_CleanMode.DRY_RUN: 'dry-run'>, 'quiet': False, 'capture': , 'debug_pytask': False, 'disable_warnings': False, 'dry_run': False, 'force': False, 'max_failures': inf, 'n_entries_in_table': 15, 'pdb': False, 'pdbcls': None, 's': False, 'show_capture': , 'show_errors_immediately': False, 'show_locals': False, 'show_traceback': True, 'sort_table': True, 'trace': False, 'verbose': 1, 'stop_after_first_failure': False, 'check_casing_of_paths': True, 'pdb_cls': '', 'tasks': [, , at 0x115ac0a40>], 'task_files': ('task_*.py',), 'command': 'build', 'root': PosixPath('/Users/tobiasr/git/pytask'), 'filterwarnings': []}, collection_reports=[CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('768f2f6b-85b9-4ed4-8b08-22e66ec68143'), 'after': [], 'is_generator': False, 'duration': (1749282510.968756, 1749282510.969585)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('2fe578c8-ab38-4d0d-bb34-e12df5b5975b'), 'after': [], 'is_generator': False, 'duration': (1749282511.055661, 1749282511.056195)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_second_file', function= at 0x115ac0a40>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d096742c-b6fd-4b68-9b43-ae5d3c775001'), 'after': [], 'is_generator': False, 'duration': (1749282511.046389, 1749282511.046774)}), exc_info=None)], dag=, hook=, tasks=[TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('768f2f6b-85b9-4ed4-8b08-22e66ec68143'), 'after': [], 'is_generator': False, 'duration': (1749282510.968756, 1749282510.969585)}), TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('2fe578c8-ab38-4d0d-bb34-e12df5b5975b'), 'after': [], 'is_generator': False, 'duration': (1749282511.055661, 1749282511.056195)}), TaskWithoutPath(name='task_create_second_file', function= at 0x115ac0a40>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d096742c-b6fd-4b68-9b43-ae5d3c775001'), 'after': [], 'is_generator': False, 'duration': (1749282511.046389, 1749282511.046774)})], dag_report=None, execution_reports=[ExecutionReport(task=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('768f2f6b-85b9-4ed4-8b08-22e66ec68143'), 'after': [], 'is_generator': False, 'duration': (1749282510.968756, 1749282510.969585)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_create_second_file', function= at 0x115ac0a40>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d096742c-b6fd-4b68-9b43-ae5d3c775001'), 'after': [], 'is_generator': False, 'duration': (1749282511.046389, 1749282511.046774)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('2fe578c8-ab38-4d0d-bb34-e12df5b5975b'), 'after': [], 'is_generator': False, 'duration': (1749282511.055661, 1749282511.056195)}), outcome=, exc_info=None, sections=[])], exit_code=, collection_start=1749282510.959347, collection_end=1749282510.9617531, execution_start=1749282510.962504, execution_end=1749282511.06384, n_tasks_failed=0, scheduler=TopologicalSorter(dag=, priorities={'82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a': 0, '45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948': 0, '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829': 0}, _nodes_processing=set(), _nodes_done={'2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829', '82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a', '45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948'}), should_stop=False, warnings=[])" + "Session(config={'pm': , 'markers': {'filterwarnings': 'Add a filter for a warning to a task.', 'persist': 'Prevent execution of a task if all products exist and even if something has changed (dependencies, source file, products). This decorator might be useful for expensive tasks where only the formatting of the file has changed. The state of the files which have changed will also be remembered and another run will skip the task with success.', 'skip': 'Skip a task and all its dependent tasks.', 'skip_ancestor_failed': 'Internal decorator applied to tasks if any of its preceding tasks failed.', 'skip_unchanged': 'Internal decorator applied to tasks which have already been executed and have not been changed.', 'skipif': 'Skip a task and all its dependent tasks if a condition is met.', 'task': 'Mark a function as a task regardless of its name. Or mark tasks which are repeated in a loop. See this tutorial for more information: [link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/].', 'try_first': 'Try to execute a task a early as possible.', 'try_last': 'Try to execute a task a late as possible.'}, 'config': None, 'database_url': sqlite:////Users/tobiasr/git/pytask/.pytask/pytask.sqlite3, 'editor_url_scheme': 'file', 'export': <_ExportFormats.NO: 'no'>, 'hook_module': None, 'ignore': ['.codecov.yml', '.gitignore', '.pre-commit-config.yaml', '.readthedocs.yml', '.readthedocs.yaml', 'readthedocs.yml', 'readthedocs.yaml', 'environment.yml', 'pyproject.toml', 'setup.cfg', 'tox.ini', '.git/*', '.venv/*', '.pixi/*', '*.egg-info/*', '.ipynb_checkpoints/*', '.mypy_cache/*', '.nox/*', '.tox/*', '_build/*', '__pycache__/*', 'build/*', 'dist/*', 'pytest_cache/*'], 'paths': [], 'layout': 'dot', 'output_path': 'dag.pdf', 'rank_direction': <_RankDirection.TB: 'TB'>, 'expression': '', 'marker_expression': '', 'nodes': False, 'strict_markers': False, 'directories': False, 'exclude': [None, '.git/*', '/Users/tobiasr/git/pytask/.pytask/*'], 'mode': <_CleanMode.DRY_RUN: 'dry-run'>, 'quiet': False, 'capture': , 'debug_pytask': False, 'disable_warnings': False, 'dry_run': False, 'explain': False, 'force': False, 'max_failures': inf, 'n_entries_in_table': 15, 'pdb': False, 'pdbcls': None, 's': False, 'show_capture': , 'show_errors_immediately': False, 'show_locals': False, 'show_traceback': True, 'sort_table': True, 'trace': False, 'verbose': 1, 'stop_after_first_failure': False, 'check_casing_of_paths': True, 'pdb_cls': '', 'tasks': [, , at 0x10fd6e200>], 'task_files': ('task_*.py',), 'command': 'build', 'root': PosixPath('/Users/tobiasr/git/pytask'), 'filterwarnings': []}, collection_reports=[CollectionReport(outcome=, node=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('ea334f40-41a6-46f4-8a90-2ceee3b46db9'), 'after': [], 'is_generator': False, 'duration': (1761860160.3823931, 1761860160.3829582)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_second_file', function= at 0x10fd6e200>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('a4ecfefd-61b0-4b89-8bd6-3a57a4fd6f8a'), 'after': [], 'is_generator': False, 'duration': (1761860160.370384, 1761860160.370671)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d557f1a5-d539-49e0-882b-d39a6f9d0b81'), 'after': [], 'is_generator': False, 'duration': (1761860160.3600278, 1761860160.361962)}), exc_info=None)], dag=, hook=, tasks=[TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('ea334f40-41a6-46f4-8a90-2ceee3b46db9'), 'after': [], 'is_generator': False, 'duration': (1761860160.3823931, 1761860160.3829582)}), TaskWithoutPath(name='task_create_second_file', function= at 0x10fd6e200>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('a4ecfefd-61b0-4b89-8bd6-3a57a4fd6f8a'), 'after': [], 'is_generator': False, 'duration': (1761860160.370384, 1761860160.370671)}), TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d557f1a5-d539-49e0-882b-d39a6f9d0b81'), 'after': [], 'is_generator': False, 'duration': (1761860160.3600278, 1761860160.361962)})], dag_report=None, execution_reports=[ExecutionReport(task=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d557f1a5-d539-49e0-882b-d39a6f9d0b81'), 'after': [], 'is_generator': False, 'duration': (1761860160.3600278, 1761860160.361962)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_create_second_file', function= at 0x10fd6e200>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('a4ecfefd-61b0-4b89-8bd6-3a57a4fd6f8a'), 'after': [], 'is_generator': False, 'duration': (1761860160.370384, 1761860160.370671)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('ea334f40-41a6-46f4-8a90-2ceee3b46db9'), 'after': [], 'is_generator': False, 'duration': (1761860160.3823931, 1761860160.3829582)}), outcome=, exc_info=None, sections=[])], exit_code=, collection_start=1761860160.3489978, collection_end=1761860160.351701, execution_start=1761860160.3525, execution_end=1761860160.3991098, n_tasks_failed=0, scheduler=TopologicalSorter(dag=, priorities={'45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948': 0, '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829': 0, '82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a': 0}, _nodes_processing=set(), _nodes_done={'45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948', '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829', '82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a'}), should_stop=False, warnings=[])" ] }, "execution_count": 4, @@ -285,6 +285,237 @@ "session" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Repeated Tasks\n", + "\n", + "You can also create multiple tasks with the same function name by using the `@task` decorator in a loop. This is useful when you want to run the same operation with different parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from pytask import Product\n", + "\n", + "# Create repeated tasks with different parameters\n", + "tasks = []\n", + "for i in range(3):\n", + "\n", + " def create_file(value: int, path: Annotated[Path, Product]) -> None:\n", + " path.write_text(f\"Result: {value}\")\n", + "\n", + " t = task(kwargs={\"value\": i * 100, \"path\": Path(f\"output_{i}.txt\")})(create_file)\n", + " tasks.append(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
────────────────────────────────────────────── Start pytask session ───────────────────────────────────────────────\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[39m────────────────────────────────────────────── \u001b[0mStart pytask session\u001b[39m ───────────────────────────────────────────────\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
Platform: darwin -- Python 3.12.12, pytask 0.5.6.dev17+g2a33ab7d5.d20251030, pluggy 1.6.0\n",
+       "
\n" + ], + "text/plain": [ + "Platform: darwin -- Python 3.12.12, pytask 0.5.6.dev17+g2a33ab7d5.d20251030, pluggy 1.6.0\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
Root: /Users/tobiasr/git/pytask\n",
+       "
\n" + ], + "text/plain": [ + "Root: \u001b[35m/Users/tobiasr/git/\u001b[0m\u001b[95mpytask\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
Collected 3 tasks.\n",
+       "
\n" + ], + "text/plain": [ + "Collected 3 tasks.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n",
+       "
\n" + ], + "text/plain": [ + "\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ebd3051cde6f429fafdc830e688fa1b3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
╭────────────────────────┬─────────╮\n",
+       "│ Task                    Outcome │\n",
+       "├────────────────────────┼─────────┤\n",
+       "│ create_file[0-path0]  .       │\n",
+       "│ create_file[100-path1].       │\n",
+       "│ create_file[200-path2].       │\n",
+       "╰────────────────────────┴─────────╯\n",
+       "
\n" + ], + "text/plain": [ + "╭────────────────────────┬─────────╮\n", + "│\u001b[1m \u001b[0m\u001b[1mTask \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mOutcome\u001b[0m\u001b[1m \u001b[0m│\n", + "├────────────────────────┼─────────┤\n", + "│ \u001b]8;id=149853;file://None\u001b\\create_file[0-path0] \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=248728;file://None\u001b\\create_file[100-path1]\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=290712;file://None\u001b\\create_file[200-path2]\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "╰────────────────────────┴─────────╯\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n",
+       "
\n" + ], + "text/plain": [ + "\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[2m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
╭─────────── Summary ────────────╮\n",
+       "  3  Collected tasks            \n",
+       "  3  Succeeded        (100.0%)  \n",
+       "╰────────────────────────────────╯\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[38;2;19;124;57m╭─\u001b[0m\u001b[38;2;19;124;57m──────────\u001b[0m\u001b[1;38;2;242;242;242m Summary \u001b[0m\u001b[38;2;19;124;57m───────────\u001b[0m\u001b[38;2;19;124;57m─╮\u001b[0m\n", + "\u001b[38;2;19;124;57m│\u001b[0m \u001b[38;2;242;242;242m \u001b[0m\u001b[38;2;242;242;242m3\u001b[0m\u001b[38;2;242;242;242m \u001b[0m\u001b[38;2;242;242;242m \u001b[0m\u001b[38;2;242;242;242mCollected tasks\u001b[0m\u001b[38;2;242;242;242m \u001b[0m\u001b[38;2;242;242;242m \u001b[0m\u001b[38;2;242;242;242m \u001b[0m\u001b[38;2;242;242;242m \u001b[0m \u001b[38;2;19;124;57m│\u001b[0m\n", + "\u001b[38;2;19;124;57m│\u001b[0m \u001b[38;2;242;242;242;48;2;19;124;57m \u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m3\u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m \u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m \u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57mSucceeded \u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m \u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m \u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m(100.0%)\u001b[0m\u001b[38;2;242;242;242;48;2;19;124;57m \u001b[0m \u001b[38;2;19;124;57m│\u001b[0m\n", + "\u001b[38;2;19;124;57m╰────────────────────────────────╯\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
──────────────────────────────────────────── Succeeded in 0.06 seconds ────────────────────────────────────────────\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.06 seconds\u001b[0m\u001b[38;2;19;124;57m ────────────────────────────────────────────\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "session = pytask.build(tasks=tasks)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Cleanup\n", + "for i in range(3):\n", + " Path(f\"output_{i}.txt\").unlink()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -296,7 +527,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -314,6 +545,7 @@ " disable_warnings: \u001b[33m'bool'\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m,\n", " dry_run: \u001b[33m'bool'\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m,\n", " editor_url_scheme: \u001b[33m\"Literal['no_link', 'file', 'vscode', 'pycharm'] | str\"\u001b[39m = \u001b[33m'file'\u001b[39m,\n", + " explain: \u001b[33m'bool'\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m,\n", " expression: \u001b[33m'str'\u001b[39m = \u001b[33m''\u001b[39m,\n", " force: \u001b[33m'bool'\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m,\n", " ignore: \u001b[33m'Iterable[str]'\u001b[39m = (),\n", @@ -363,6 +595,8 @@ "editor_url_scheme\n", " An url scheme that allows to click on task names, node names and filenames and\n", " jump right into you preferred editor to the right line.\n", + "explain\n", + " Explain why tasks need to be executed by showing what changed.\n", "expression\n", " Same as ``-k`` on the command line. Select tasks via expressions on task ids.\n", "force\n", @@ -426,7 +660,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -452,7 +686,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.12.12" } }, "nbformat": 4, diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index d1ed0c3a..847f8881 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -50,6 +50,7 @@ from _pytask.shared import to_list from _pytask.shared import unwrap_task_function from _pytask.task_utils import COLLECTED_TASKS +from _pytask.task_utils import parse_collected_tasks_with_task_marker from _pytask.task_utils import task as task_decorator from _pytask.typing import is_task_function @@ -108,27 +109,46 @@ def _collect_from_paths(session: Session) -> None: def _collect_from_tasks(session: Session) -> None: """Collect tasks from user provided tasks via the functional interface.""" + # First pass: collect and group tasks by path + tasks_by_path: dict[Path | None, list[Any]] = {} + non_task_objects = [] + for raw_task in to_list(session.config.get("tasks", ())): if is_task_function(raw_task): if not hasattr(raw_task, "pytask_meta"): raw_task = task_decorator()(raw_task) # noqa: PLW2901 path = get_file(raw_task) - name = raw_task.pytask_meta.name - - if has_mark(raw_task, "task"): - # When tasks with @task are passed to the programmatic interface multiple - # times, they are deleted from ``COLLECTED_TASKS`` in the first iteration - # and are missing in the later. See #625. - with suppress(ValueError): - COLLECTED_TASKS[path].remove(raw_task) - - # When a task is not a callable, it can be anything or a PTask. Set arbitrary - # values and it will pass without errors and not collected. + name = raw_task.pytask_meta.name # type: ignore[attr-defined] + + if has_mark(raw_task, "task"): + # When tasks with @task are passed to the programmatic interface + # multiple times, they are deleted from ``COLLECTED_TASKS`` in the first + # iteration and are missing in the later. See #625. + with suppress(ValueError): + COLLECTED_TASKS[path].remove(raw_task) + + # Group tasks by path for parametrization + if path not in tasks_by_path: + tasks_by_path[path] = [] + tasks_by_path[path].append(raw_task) + else: + non_task_objects.append((raw_task, path, name)) else: - name = "" - path = None - + # When a task is not a callable, it can be anything or a PTask. Set + # arbitrary values and it will pass without errors and not collected. + non_task_objects.append((raw_task, None, "")) + + # Second pass: apply parametrization to grouped tasks + parametrized_tasks = [] + for path, tasks in tasks_by_path.items(): + # Apply the same parametrization logic as file-based collection + name_to_function = parse_collected_tasks_with_task_marker(tasks) + for name, function in name_to_function.items(): + parametrized_tasks.append((function, path, name)) + + # Third pass: collect all tasks + for raw_task, path, name in parametrized_tasks + non_task_objects: report = session.hook.pytask_collect_task_protocol( session=session, reports=session.collection_reports, diff --git a/tests/test_execute.py b/tests/test_execute.py index 809d0c7e..a3f3c069 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -660,6 +660,64 @@ def test_pass_non_task_to_functional_api_that_are_ignored(): assert len(session.tasks) == 0 +@pytest.mark.skipif( + sys.platform == "win32" and os.environ.get("CI") == "true", + reason="Windows does not pick up the right Python interpreter.", +) +def test_repeated_tasks_via_functional_interface(tmp_path): + """Test that repeated tasks with the same function name work correctly. + + This test ensures that when multiple tasks with the same function name are passed + to pytask.build(), they all get unique IDs and execute correctly, similar to how + file-based collection handles repeated tasks. + """ + source = """ + from pathlib import Path + from typing import Annotated + from pytask import Product, task, build, ExitCode + import sys + + # Create repeated tasks with the same function name + tasks = [] + for i in range(3): + def create_data(value: int, produces: Annotated[Path, Product]): + '''Generate data based on a value.''' + produces.write_text(str(value)) + + t = task( + kwargs={"value": i * 10, "produces": Path(f"output_{i}.txt")}, + )(create_data) + tasks.append(t) + + if __name__ == "__main__": + session = build(tasks=tasks) + + # Verify all tasks were collected and executed + assert session.exit_code == ExitCode.OK, f"Exit code: {session.exit_code}" + assert len(session.tasks) == 3, f"Expected 3 tasks, got {len(session.tasks)}" + assert len(session.execution_reports) == 3 + + # Verify each task executed and produced the correct output + assert Path("output_0.txt").read_text() == "0" + assert Path("output_1.txt").read_text() == "10" + assert Path("output_2.txt").read_text() == "20" + + # Verify tasks have unique names with repeated task IDs + task_names = [task.name for task in session.tasks] + assert len(task_names) == len(set(task_names)), "Task names should be unique" + assert all("create_data[" in name for name in task_names), \\ + f"Task names should contain repeated task IDs: {task_names}" + + sys.exit(session.exit_code) + """ + tmp_path.joinpath("task_module.py").write_text(textwrap.dedent(source)) + result = run_in_subprocess( + (sys.executable, tmp_path.joinpath("task_module.py").as_posix()), + cwd=tmp_path, + ) + assert result.exit_code == ExitCode.OK + + def test_multiple_product_annotations(runner, tmp_path): source = """ from pytask import Product diff --git a/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb b/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb new file mode 100644 index 00000000..4d0e1caa --- /dev/null +++ b/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb @@ -0,0 +1,80 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from typing import Annotated\n", + "\n", + "import pytask\n", + "from pytask import ExitCode\n", + "from pytask import Product\n", + "from pytask import task" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "# Create repeated tasks with the same function name\n", + "tasks = []\n", + "for i in range(3):\n", + "\n", + " def create_data(value: int, produces: Annotated[Path, Product]):\n", + " \"\"\"Generate data based on a value.\"\"\"\n", + " produces.write_text(str(value))\n", + "\n", + " t = task(\n", + " kwargs={\"value\": i * 10, \"produces\": Path(f\"data_{i}.txt\").resolve()},\n", + " )(create_data)\n", + " tasks.append(t)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# Test that all tasks execute correctly\n", + "session = pytask.build(tasks=tasks)\n", + "assert session.exit_code == ExitCode.OK\n", + "assert len(session.tasks) == 3, f\"Expected 3 tasks, got {len(session.tasks)}\"\n", + "assert len(session.execution_reports) == 3, (\n", + " f\"Expected 3 execution reports, got {len(session.execution_reports)}\"\n", + ")\n", + "\n", + "# Verify each file was created with the correct content\n", + "assert Path(\"data_0.txt\").read_text() == \"0\"\n", + "assert Path(\"data_1.txt\").read_text() == \"10\"\n", + "assert Path(\"data_2.txt\").read_text() == \"20\"\n", + "\n", + "# Clean up\n", + "Path(\"data_0.txt\").unlink()\n", + "Path(\"data_1.txt\").unlink()\n", + "Path(\"data_2.txt\").unlink()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From a731a1afd81f6ecbf946bebdd2647f84646f11ed Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 30 Oct 2025 22:55:07 +0100 Subject: [PATCH 2/5] Fix tpying issue. --- src/_pytask/collect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 847f8881..460517c2 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -119,7 +119,7 @@ def _collect_from_tasks(session: Session) -> None: raw_task = task_decorator()(raw_task) # noqa: PLW2901 path = get_file(raw_task) - name = raw_task.pytask_meta.name # type: ignore[attr-defined] + name = raw_task.pytask_meta.name if has_mark(raw_task, "task"): # When tasks with @task are passed to the programmatic interface From 64e4d8b2b9f0ea5c01e9f409e40b89a3db683f07 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 30 Oct 2025 23:04:55 +0100 Subject: [PATCH 3/5] Simplify example. --- .../how_to_guides/functional_interface.ipynb | 33 ++++++++++--------- src/_pytask/collect.py | 24 +++++++------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/docs/source/how_to_guides/functional_interface.ipynb b/docs/source/how_to_guides/functional_interface.ipynb index b7e26244..3306a789 100644 --- a/docs/source/how_to_guides/functional_interface.ipynb +++ b/docs/source/how_to_guides/functional_interface.ipynb @@ -147,7 +147,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "35f4210e20704ca7b8130462a887bebf", + "model_id": "7727ceef87cf49c9a926679362ef6c96", "version_major": 2, "version_minor": 0 }, @@ -184,9 +184,9 @@ "╭─────────────────────────┬─────────╮\n", "│\u001b[1m \u001b[0m\u001b[1mTask \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mOutcome\u001b[0m\u001b[1m \u001b[0m│\n", "├─────────────────────────┼─────────┤\n", - "│ \u001b]8;id=780914;file://None\u001b\\task_create_first_file \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", - "│ \u001b]8;id=865151;file://None\u001b\\task_create_second_file\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", - "│ \u001b]8;id=487888;file://None\u001b\\task_merge_files \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=303527;file://None\u001b\\task_create_first_file \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=36996;file://None\u001b\\task_create_second_file\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=56184;file://None\u001b\\task_merge_files \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", "╰─────────────────────────┴─────────╯\n" ] }, @@ -241,11 +241,11 @@ { "data": { "text/html": [ - "
──────────────────────────────────────────── Succeeded in 0.05 seconds ────────────────────────────────────────────\n",
+       "
──────────────────────────────────────────── Succeeded in 0.03 seconds ────────────────────────────────────────────\n",
        "
\n" ], "text/plain": [ - "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.05 seconds\u001b[0m\u001b[38;2;19;124;57m ────────────────────────────────────────────\u001b[0m\n" + "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.03 seconds\u001b[0m\u001b[38;2;19;124;57m ────────────────────────────────────────────\u001b[0m\n" ] }, "metadata": {}, @@ -273,7 +273,7 @@ { "data": { "text/plain": [ - "Session(config={'pm': , 'markers': {'filterwarnings': 'Add a filter for a warning to a task.', 'persist': 'Prevent execution of a task if all products exist and even if something has changed (dependencies, source file, products). This decorator might be useful for expensive tasks where only the formatting of the file has changed. The state of the files which have changed will also be remembered and another run will skip the task with success.', 'skip': 'Skip a task and all its dependent tasks.', 'skip_ancestor_failed': 'Internal decorator applied to tasks if any of its preceding tasks failed.', 'skip_unchanged': 'Internal decorator applied to tasks which have already been executed and have not been changed.', 'skipif': 'Skip a task and all its dependent tasks if a condition is met.', 'task': 'Mark a function as a task regardless of its name. Or mark tasks which are repeated in a loop. See this tutorial for more information: [link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/].', 'try_first': 'Try to execute a task a early as possible.', 'try_last': 'Try to execute a task a late as possible.'}, 'config': None, 'database_url': sqlite:////Users/tobiasr/git/pytask/.pytask/pytask.sqlite3, 'editor_url_scheme': 'file', 'export': <_ExportFormats.NO: 'no'>, 'hook_module': None, 'ignore': ['.codecov.yml', '.gitignore', '.pre-commit-config.yaml', '.readthedocs.yml', '.readthedocs.yaml', 'readthedocs.yml', 'readthedocs.yaml', 'environment.yml', 'pyproject.toml', 'setup.cfg', 'tox.ini', '.git/*', '.venv/*', '.pixi/*', '*.egg-info/*', '.ipynb_checkpoints/*', '.mypy_cache/*', '.nox/*', '.tox/*', '_build/*', '__pycache__/*', 'build/*', 'dist/*', 'pytest_cache/*'], 'paths': [], 'layout': 'dot', 'output_path': 'dag.pdf', 'rank_direction': <_RankDirection.TB: 'TB'>, 'expression': '', 'marker_expression': '', 'nodes': False, 'strict_markers': False, 'directories': False, 'exclude': [None, '.git/*', '/Users/tobiasr/git/pytask/.pytask/*'], 'mode': <_CleanMode.DRY_RUN: 'dry-run'>, 'quiet': False, 'capture': , 'debug_pytask': False, 'disable_warnings': False, 'dry_run': False, 'explain': False, 'force': False, 'max_failures': inf, 'n_entries_in_table': 15, 'pdb': False, 'pdbcls': None, 's': False, 'show_capture': , 'show_errors_immediately': False, 'show_locals': False, 'show_traceback': True, 'sort_table': True, 'trace': False, 'verbose': 1, 'stop_after_first_failure': False, 'check_casing_of_paths': True, 'pdb_cls': '', 'tasks': [, , at 0x10fd6e200>], 'task_files': ('task_*.py',), 'command': 'build', 'root': PosixPath('/Users/tobiasr/git/pytask'), 'filterwarnings': []}, collection_reports=[CollectionReport(outcome=, node=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('ea334f40-41a6-46f4-8a90-2ceee3b46db9'), 'after': [], 'is_generator': False, 'duration': (1761860160.3823931, 1761860160.3829582)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_second_file', function= at 0x10fd6e200>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('a4ecfefd-61b0-4b89-8bd6-3a57a4fd6f8a'), 'after': [], 'is_generator': False, 'duration': (1761860160.370384, 1761860160.370671)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d557f1a5-d539-49e0-882b-d39a6f9d0b81'), 'after': [], 'is_generator': False, 'duration': (1761860160.3600278, 1761860160.361962)}), exc_info=None)], dag=, hook=, tasks=[TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('ea334f40-41a6-46f4-8a90-2ceee3b46db9'), 'after': [], 'is_generator': False, 'duration': (1761860160.3823931, 1761860160.3829582)}), TaskWithoutPath(name='task_create_second_file', function= at 0x10fd6e200>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('a4ecfefd-61b0-4b89-8bd6-3a57a4fd6f8a'), 'after': [], 'is_generator': False, 'duration': (1761860160.370384, 1761860160.370671)}), TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d557f1a5-d539-49e0-882b-d39a6f9d0b81'), 'after': [], 'is_generator': False, 'duration': (1761860160.3600278, 1761860160.361962)})], dag_report=None, execution_reports=[ExecutionReport(task=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('d557f1a5-d539-49e0-882b-d39a6f9d0b81'), 'after': [], 'is_generator': False, 'duration': (1761860160.3600278, 1761860160.361962)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_create_second_file', function= at 0x10fd6e200>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('a4ecfefd-61b0-4b89-8bd6-3a57a4fd6f8a'), 'after': [], 'is_generator': False, 'duration': (1761860160.370384, 1761860160.370671)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('ea334f40-41a6-46f4-8a90-2ceee3b46db9'), 'after': [], 'is_generator': False, 'duration': (1761860160.3823931, 1761860160.3829582)}), outcome=, exc_info=None, sections=[])], exit_code=, collection_start=1761860160.3489978, collection_end=1761860160.351701, execution_start=1761860160.3525, execution_end=1761860160.3991098, n_tasks_failed=0, scheduler=TopologicalSorter(dag=, priorities={'45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948': 0, '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829': 0, '82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a': 0}, _nodes_processing=set(), _nodes_done={'45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948', '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829', '82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a'}), should_stop=False, warnings=[])" + "Session(config={'pm': , 'markers': {'filterwarnings': 'Add a filter for a warning to a task.', 'persist': 'Prevent execution of a task if all products exist and even if something has changed (dependencies, source file, products). This decorator might be useful for expensive tasks where only the formatting of the file has changed. The state of the files which have changed will also be remembered and another run will skip the task with success.', 'skip': 'Skip a task and all its dependent tasks.', 'skip_ancestor_failed': 'Internal decorator applied to tasks if any of its preceding tasks failed.', 'skip_unchanged': 'Internal decorator applied to tasks which have already been executed and have not been changed.', 'skipif': 'Skip a task and all its dependent tasks if a condition is met.', 'task': 'Mark a function as a task regardless of its name. Or mark tasks which are repeated in a loop. See this tutorial for more information: [link https://bit.ly/3DWrXS3]https://bit.ly/3DWrXS3[/].', 'try_first': 'Try to execute a task a early as possible.', 'try_last': 'Try to execute a task a late as possible.'}, 'config': None, 'database_url': sqlite:////Users/tobiasr/git/pytask/.pytask/pytask.sqlite3, 'editor_url_scheme': 'file', 'export': <_ExportFormats.NO: 'no'>, 'hook_module': None, 'ignore': ['.codecov.yml', '.gitignore', '.pre-commit-config.yaml', '.readthedocs.yml', '.readthedocs.yaml', 'readthedocs.yml', 'readthedocs.yaml', 'environment.yml', 'pyproject.toml', 'setup.cfg', 'tox.ini', '.git/*', '.venv/*', '.pixi/*', '*.egg-info/*', '.ipynb_checkpoints/*', '.mypy_cache/*', '.nox/*', '.tox/*', '_build/*', '__pycache__/*', 'build/*', 'dist/*', 'pytest_cache/*'], 'paths': [], 'layout': 'dot', 'output_path': 'dag.pdf', 'rank_direction': <_RankDirection.TB: 'TB'>, 'expression': '', 'marker_expression': '', 'nodes': False, 'strict_markers': False, 'directories': False, 'exclude': [None, '.git/*', '/Users/tobiasr/git/pytask/.pytask/*'], 'mode': <_CleanMode.DRY_RUN: 'dry-run'>, 'quiet': False, 'capture': , 'debug_pytask': False, 'disable_warnings': False, 'dry_run': False, 'explain': False, 'force': False, 'max_failures': inf, 'n_entries_in_table': 15, 'pdb': False, 'pdbcls': None, 's': False, 'show_capture': , 'show_errors_immediately': False, 'show_locals': False, 'show_traceback': True, 'sort_table': True, 'trace': False, 'verbose': 1, 'stop_after_first_failure': False, 'check_casing_of_paths': True, 'pdb_cls': '', 'tasks': [, , at 0x11346e160>], 'task_files': ('task_*.py',), 'command': 'build', 'root': PosixPath('/Users/tobiasr/git/pytask'), 'filterwarnings': []}, collection_reports=[CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('c59f0999-a828-4b41-b0c0-b274b57273e6'), 'after': [], 'is_generator': False, 'duration': (1761861388.575503, 1761861388.575783)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_create_second_file', function= at 0x11346e160>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('74d5f952-75ab-44f8-b219-984f44e92684'), 'after': [], 'is_generator': False, 'duration': (1761861388.5663831, 1761861388.566726)}), exc_info=None), CollectionReport(outcome=, node=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('801a135a-f9ed-427c-8204-14dd1a68fc9c'), 'after': [], 'is_generator': False, 'duration': (1761861388.581853, 1761861388.582324)}), exc_info=None)], dag=, hook=, tasks=[TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('c59f0999-a828-4b41-b0c0-b274b57273e6'), 'after': [], 'is_generator': False, 'duration': (1761861388.575503, 1761861388.575783)}), TaskWithoutPath(name='task_create_second_file', function= at 0x11346e160>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('74d5f952-75ab-44f8-b219-984f44e92684'), 'after': [], 'is_generator': False, 'duration': (1761861388.5663831, 1761861388.566726)}), TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('801a135a-f9ed-427c-8204-14dd1a68fc9c'), 'after': [], 'is_generator': False, 'duration': (1761861388.581853, 1761861388.582324)})], dag_report=None, execution_reports=[ExecutionReport(task=TaskWithoutPath(name='task_create_second_file', function= at 0x11346e160>, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('74d5f952-75ab-44f8-b219-984f44e92684'), 'after': [], 'is_generator': False, 'duration': (1761861388.5663831, 1761861388.566726)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_create_first_file', function=, depends_on={}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('c59f0999-a828-4b41-b0c0-b274b57273e6'), 'after': [], 'is_generator': False, 'duration': (1761861388.575503, 1761861388.575783)}), outcome=, exc_info=None, sections=[]), ExecutionReport(task=TaskWithoutPath(name='task_merge_files', function=, depends_on={'first': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/first.txt'), name='pytask/docs/source/how_to_guides/first.txt', attributes={}), 'second': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/second.txt'), name='pytask/docs/source/how_to_guides/second.txt', attributes={})}, produces={'return': PathNode(path=PosixPath('/Users/tobiasr/git/pytask/docs/source/how_to_guides/hello_world.txt'), name='pytask/docs/source/how_to_guides/hello_world.txt', attributes={})}, markers=[Mark(name='task', args=(), kwargs={})], report_sections=[], attributes={'collection_id': UUID('801a135a-f9ed-427c-8204-14dd1a68fc9c'), 'after': [], 'is_generator': False, 'duration': (1761861388.581853, 1761861388.582324)}), outcome=, exc_info=None, sections=[])], exit_code=, collection_start=1761861388.557942, collection_end=1761861388.560052, execution_start=1761861388.560833, execution_end=1761861388.587148, n_tasks_failed=0, scheduler=TopologicalSorter(dag=, priorities={'82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a': 0, '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829': 0, '45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948': 0}, _nodes_processing=set(), _nodes_done={'82d6a7ce01a2a50d5d4bd5081d662df92b8c500fbc172f94fb026c9d1d4ebc4a', '2a06f358fc8e621754c133af76f5ac1b3e8ad5172b5803823cb264b30ea5d829', '45a637ca3cc7aa973d4b315cc1bef02217b79918357fd35c6fa61f4e2d2f9948'}), should_stop=False, warnings=[])" ] }, "execution_count": 4, @@ -306,11 +306,12 @@ "tasks = []\n", "for i in range(3):\n", "\n", - " def create_file(value: int, path: Annotated[Path, Product]) -> None:\n", + " def create_file(\n", + " value: int = i * 100, path: Annotated[Path, Product] = Path(f\"output_{i}.txt\")\n", + " ) -> None:\n", " path.write_text(f\"Result: {value}\")\n", "\n", - " t = task(kwargs={\"value\": i * 100, \"path\": Path(f\"output_{i}.txt\")})(create_file)\n", - " tasks.append(t)" + " tasks.append(create_file)" ] }, { @@ -396,7 +397,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ebd3051cde6f429fafdc830e688fa1b3", + "model_id": "eb660941cbb04250b67d08f7f7c134de", "version_major": 2, "version_minor": 0 }, @@ -433,9 +434,9 @@ "╭────────────────────────┬─────────╮\n", "│\u001b[1m \u001b[0m\u001b[1mTask \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1mOutcome\u001b[0m\u001b[1m \u001b[0m│\n", "├────────────────────────┼─────────┤\n", - "│ \u001b]8;id=149853;file://None\u001b\\create_file[0-path0] \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", - "│ \u001b]8;id=248728;file://None\u001b\\create_file[100-path1]\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", - "│ \u001b]8;id=290712;file://None\u001b\\create_file[200-path2]\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=505794;file://None\u001b\\create_file[0-path0] \u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=700433;file://None\u001b\\create_file[100-path1]\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", + "│ \u001b]8;id=284007;file://None\u001b\\create_file[200-path2]\u001b]8;;\u001b\\ │ \u001b[38;2;19;124;57m. \u001b[0m │\n", "╰────────────────────────┴─────────╯\n" ] }, @@ -490,11 +491,11 @@ { "data": { "text/html": [ - "
──────────────────────────────────────────── Succeeded in 0.06 seconds ────────────────────────────────────────────\n",
+       "
──────────────────────────────────────────── Succeeded in 0.03 seconds ────────────────────────────────────────────\n",
        "
\n" ], "text/plain": [ - "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.06 seconds\u001b[0m\u001b[38;2;19;124;57m ────────────────────────────────────────────\u001b[0m\n" + "\u001b[38;2;19;124;57m──────────────────────────────────────────── \u001b[0m\u001b[38;2;19;124;57mSucceeded in 0.03 seconds\u001b[0m\u001b[38;2;19;124;57m ────────────────────────────────────────────\u001b[0m\n" ] }, "metadata": {}, diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 460517c2..d5db1403 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -121,19 +121,17 @@ def _collect_from_tasks(session: Session) -> None: path = get_file(raw_task) name = raw_task.pytask_meta.name - if has_mark(raw_task, "task"): - # When tasks with @task are passed to the programmatic interface - # multiple times, they are deleted from ``COLLECTED_TASKS`` in the first - # iteration and are missing in the later. See #625. - with suppress(ValueError): - COLLECTED_TASKS[path].remove(raw_task) - - # Group tasks by path for parametrization - if path not in tasks_by_path: - tasks_by_path[path] = [] - tasks_by_path[path].append(raw_task) - else: - non_task_objects.append((raw_task, path, name)) + if has_mark(raw_task, "task"): + # When tasks with @task are passed to the programmatic interface + # multiple times, they are deleted from ``COLLECTED_TASKS`` in the first + # iteration and are missing in the later. See #625. + with suppress(ValueError): + COLLECTED_TASKS[path].remove(raw_task) + + # Group tasks by path for parametrization + if path not in tasks_by_path: + tasks_by_path[path] = [] + tasks_by_path[path].append(raw_task) else: # When a task is not a callable, it can be anything or a PTask. Set # arbitrary values and it will pass without errors and not collected. From 27926b5744b87c09eb9d4009c38fd1bb9a5b66f2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 30 Oct 2025 23:08:07 +0100 Subject: [PATCH 4/5] Fix. --- tests/test_execute.py | 10 ++++---- ..._repeated_tasks_functional_interface.ipynb | 25 ++++++++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/test_execute.py b/tests/test_execute.py index a3f3c069..5a2c659c 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -680,14 +680,14 @@ def test_repeated_tasks_via_functional_interface(tmp_path): # Create repeated tasks with the same function name tasks = [] for i in range(3): - def create_data(value: int, produces: Annotated[Path, Product]): + def create_data( + value: int = i * 10, + produces: Annotated[Path, Product] = Path(f"output_{i}.txt") + ) -> None: '''Generate data based on a value.''' produces.write_text(str(value)) - t = task( - kwargs={"value": i * 10, "produces": Path(f"output_{i}.txt")}, - )(create_data) - tasks.append(t) + tasks.append(create_data) if __name__ == "__main__": session = build(tasks=tasks) diff --git a/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb b/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb index 4d0e1caa..11f5ca23 100644 --- a/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb +++ b/tests/test_jupyter/test_repeated_tasks_functional_interface.ipynb @@ -12,8 +12,7 @@ "\n", "import pytask\n", "from pytask import ExitCode\n", - "from pytask import Product\n", - "from pytask import task" + "from pytask import Product" ] }, { @@ -27,14 +26,14 @@ "tasks = []\n", "for i in range(3):\n", "\n", - " def create_data(value: int, produces: Annotated[Path, Product]):\n", + " def create_data(\n", + " value: int = i * 10,\n", + " produces: Annotated[Path, Product] = Path(f\"data_{i}.txt\"),\n", + " ):\n", " \"\"\"Generate data based on a value.\"\"\"\n", " produces.write_text(str(value))\n", "\n", - " t = task(\n", - " kwargs={\"value\": i * 10, \"produces\": Path(f\"data_{i}.txt\").resolve()},\n", - " )(create_data)\n", - " tasks.append(t)" + " tasks.append(create_data)" ] }, { @@ -66,13 +65,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.12.0" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" } }, "nbformat": 4, From 241899fb68421b9c97901e6b055366c67ff14787 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Thu, 30 Oct 2025 23:14:16 +0100 Subject: [PATCH 5/5] Add to changes. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c54af609..18203764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and - {pull}`708` updates mypy and fixes type issues. - {pull}`709` add uv pre-commit check. - {pull}`713` removes uv as a test dependency. Closes {issue}`712`. Thanks to {user}`erooke`! +- {pull}`719` fixes repeated tasks with the same function name in the programmatic interface to ensure all tasks execute correctly. ## 0.5.5 - 2025-07-25