-
Notifications
You must be signed in to change notification settings - Fork 334
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
SAMZA-1096: StreamSpec constructors in the ExecutionEnvironments
Author: Jacob Maes <jmaes@linkedin.com> Reviewers: Yi Pan (Data Infrastructure) <nickpan47@gmail.com>,Xinyu Liu <xiliu@linkedin.com>,Navina Ramesh <navina@apache.org> Closes #74 from jmakes/samza-1096
- Loading branch information
Jacob Maes
committed
Mar 6, 2017
1 parent
d104013
commit e6c1eed
Showing
15 changed files
with
810 additions
and
113 deletions.
There are no files selected for viewing
225 changes: 168 additions & 57 deletions
225
docs/learn/documentation/versioned/jobs/configuration-table.html
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
88 changes: 88 additions & 0 deletions
88
samza-core/src/main/java/org/apache/samza/system/AbstractExecutionEnvironment.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, | ||
* software distributed under the License is distributed on an | ||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
* KIND, either express or implied. See the License for the | ||
* specific language governing permissions and limitations | ||
* under the License. | ||
*/ | ||
package org.apache.samza.system; | ||
|
||
import java.util.Map; | ||
import org.apache.samza.config.Config; | ||
import org.apache.samza.config.StreamConfig; | ||
|
||
|
||
public abstract class AbstractExecutionEnvironment implements ExecutionEnvironment { | ||
|
||
private final Config config; | ||
|
||
public AbstractExecutionEnvironment(Config config) { | ||
if (config == null) { | ||
throw new NullPointerException("Parameter 'config' cannot be null."); | ||
} | ||
|
||
this.config = config; | ||
} | ||
|
||
@Override | ||
public StreamSpec streamFromConfig(String streamId) { | ||
StreamConfig streamConfig = new StreamConfig(config); | ||
String physicalName = streamConfig.getPhysicalName(streamId, streamId); | ||
|
||
return streamFromConfig(streamId, physicalName); | ||
} | ||
|
||
/** | ||
* Constructs a {@link StreamSpec} from the configuration for the specified streamId. | ||
* | ||
* The stream configurations are read from the following properties in the config: | ||
* {@code streams.{$streamId}.*} | ||
* <br> | ||
* All properties matching this pattern are assumed to be system-specific with one exception. The following | ||
* property is a Samza property which is used to bind the stream to a system. | ||
* | ||
* <ul> | ||
* <li>samza.system - The name of the System on which this stream will be used. If this property isn't defined | ||
* the stream will be associated with the System defined in {@code job.default.system}</li> | ||
* </ul> | ||
* | ||
* @param streamId The logical identifier for the stream in Samza. | ||
* @param physicalName The system-specific name for this stream. It could be a file URN, topic name, or other identifer. | ||
* @return The {@link StreamSpec} instance. | ||
*/ | ||
/*package private*/ StreamSpec streamFromConfig(String streamId, String physicalName) { | ||
StreamConfig streamConfig = new StreamConfig(config); | ||
String system = streamConfig.getSystem(streamId); | ||
|
||
return streamFromConfig(streamId, physicalName, system); | ||
} | ||
|
||
/** | ||
* Constructs a {@link StreamSpec} from the configuration for the specified streamId. | ||
* | ||
* The stream configurations are read from the following properties in the config: | ||
* {@code streams.{$streamId}.*} | ||
* | ||
* @param streamId The logical identifier for the stream in Samza. | ||
* @param physicalName The system-specific name for this stream. It could be a file URN, topic name, or other identifer. | ||
* @param system The name of the System on which this stream will be used. | ||
* @return The {@link StreamSpec} instance. | ||
*/ | ||
/*package private*/ StreamSpec streamFromConfig(String streamId, String physicalName, String system) { | ||
StreamConfig streamConfig = new StreamConfig(config); | ||
Map<String, String> properties = streamConfig.getStreamProperties(streamId); | ||
|
||
return new StreamSpec(streamId, physicalName, system, properties); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.