Skip to content

Commit

Permalink
[BEAM-5600] Add unimplemented split API to Runner side SDF libraries.
Browse files Browse the repository at this point in the history
This creates an unsupported API to the RemoteBundle and a default split handler that throws.
  • Loading branch information
lukecwik committed Jan 2, 2020
1 parent a109cb5 commit a6c13dc
Show file tree
Hide file tree
Showing 6 changed files with 259 additions and 129 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.beam.runners.fnexecution.control;

import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleSplitResponse;

/**
* A handler which is invoked whenever an active bundle is split. The active bundle will continue to
* keep processing until it is complete.
*
* <p>The returned split response contains a description of work that has been performed containing
* a {@code primary} portion that the SDK is responsible for processing and a {@code residual} which
* the runner is responsible for scheduling for future processing. See <a
* href="https://s.apache.org/beam-breaking-fusion">breaking the fusion barrier</a> for further
* details.
*/
public interface BundleSplitHandler {
void split(ProcessBundleSplitResponse splitResponse);

/** Returns a bundle split handler that throws on any split response. */
static BundleSplitHandler unsupported() {
return new BundleSplitHandler() {
@Override
public void split(ProcessBundleSplitResponse splitResponse) {
throw new UnsupportedOperationException(
String.format(
"%s does not support splitting.", BundleSplitHandler.class.getSimpleName()));
}
};
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,11 @@ public Map<String, FnDataReceiver> getInputReceivers() {
return bundle.getInputReceivers();
}

@Override
public void split(double fractionOfRemainder) {
bundle.split(fractionOfRemainder);
}

@Override
public void close() throws Exception {
bundle.close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,15 @@ public interface RemoteBundle extends AutoCloseable {
*/
Map<String, FnDataReceiver> getInputReceivers();

/**
* Ask the remote bundle to split its current processing based upon its knowledge of remaining
* work. A fraction of 0, is equivalent to asking the SDK to checkpoint.
*
* <p>This method will return after the request has been issued. Any splits will be forwarded to
* the {@link BundleSplitHandler}.
*/
void split(double fractionOfRemainder);

/**
* Closes this bundle. This causes the input {@link FnDataReceiver} to be closed (future calls to
* that {@link FnDataReceiver} will throw an exception), and causes the {@link RemoteBundle} to
Expand Down

0 comments on commit a6c13dc

Please sign in to comment.