Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[YAML] Interpret PDone as no outputs. #30862

Merged
merged 3 commits into from
Apr 5, 2024
Merged

[YAML] Interpret PDone as no outputs. #30862

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9e98ada
[YAML] Interpret PDone as no outputs.
robertwb Apr 4, 2024
bd1945e
Allow implicit n-arry outputs of chain composite and pipelines.
robertwb Apr 4, 2024
a841c55
Fix change-detector test.
robertwb Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 9 additions & 6 deletions sdks/python/apache_beam/yaml/yaml_transform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ def expand_leaf_transform(spec, scope):
return {f'out{ix}': pcoll for (ix, pcoll) in enumerate(outputs)}
elif isinstance(outputs, beam.PCollection):
return {'out': outputs}
elif outputs is None:
elif outputs is None or isinstance(outputs, beam.pvalue.PDone):
return {}
else:
raise ValueError(
Expand All @@ -518,10 +518,13 @@ class CompositePTransform(beam.PTransform):
@staticmethod
def expand(inputs):
inner_scope.compute_all()
return {
key: inner_scope.get_pcollection(value)
for (key, value) in spec['output'].items()
}
if '__implicit_outputs__' in spec['output']:
return inner_scope.get_outputs(spec['output']['__implicit_outputs__'])
else:
return {
key: inner_scope.get_pcollection(value)
for (key, value) in spec['output'].items()
}

if 'name' not in spec:
spec['name'] = 'Composite'
Expand Down Expand Up @@ -596,7 +599,7 @@ def is_not_output_of_last_transform(new_transforms, value):
for (key, value) in composite_spec['output'].items()
}
else:
composite_spec['output'] = last_transform
composite_spec['output'] = {'__implicit_outputs__': last_transform}
if 'name' not in composite_spec:
composite_spec['name'] = 'Chain'
composite_spec['type'] = 'composite'
Expand Down
3 changes: 2 additions & 1 deletion sdks/python/apache_beam/yaml/yaml_transform_unit_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,8 @@ def test_chain_as_composite(self):
config:
fn: 'lambda x: x*x'
input: {spec['transforms'][0]['__uuid__']}
output: {spec['transforms'][1]['__uuid__']}
output:
'__implicit_outputs__': {spec['transforms'][1]['__uuid__']}
'''
self.assertYaml(expected, result)

Expand Down