-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Redo: Extract all yarn related methods to azkaban-common as shared ut…
…il library (#3145) * Extract all yarn related methods to azkaban-common as shared util library * extract duplicate code to a static method * amend header for test file
- Loading branch information
1 parent
e069212
commit f34a223
Showing
5 changed files
with
198 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| /* | ||
| * Copyright 2022 LinkedIn Corp. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not | ||
| * use this file except in compliance with the License. You may obtain a copy of | ||
| * the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | ||
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | ||
| * License for the specific language governing permissions and limitations under | ||
| * the License. | ||
| */ | ||
| package azkaban.utils; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Set; | ||
| import org.apache.hadoop.fs.Path; | ||
| import org.apache.log4j.Logger; | ||
| import org.apache.hadoop.yarn.api.records.ApplicationId; | ||
| import org.apache.hadoop.yarn.client.api.YarnClient; | ||
| import org.apache.hadoop.yarn.conf.YarnConfiguration; | ||
| import org.apache.hadoop.yarn.exceptions.YarnException; | ||
|
|
||
| public class YarnUtils { | ||
|
|
||
| //Yarn resource configuration directory for the cluster where the job is scheduled by the cluster router | ||
| private static final String YARN_CONF_DIRECTORY_PROPERTY = "env.YARN_CONF_DIR"; | ||
| private static final String YARN_CONF_FILENAME = "yarn-site.xml"; | ||
|
|
||
| /** | ||
| * Uses YarnClient to kill the jobs one by one | ||
| */ | ||
| public static void killAllAppsOnCluster(YarnClient yarnClient, Set<String> applicationIDs, | ||
| Logger log) { | ||
| log.info(String.format("Killing applications: %s", applicationIDs)); | ||
|
|
||
| for (final String appId : applicationIDs) { | ||
| try { | ||
| YarnUtils.killAppOnCluster(yarnClient, appId, log); | ||
| } catch (final Throwable t) { | ||
| log.warn("something happened while trying to kill this job: " + appId, t); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * <pre> | ||
| * Uses YarnClient to kill the job on the Hadoop Yarn Cluster. | ||
| * Using JobClient only works partially: | ||
| * If yarn container has started but spark job haven't, it will kill | ||
| * If spark job has started, the cancel will hang until the spark job is complete | ||
| * If the spark job is complete, it will return immediately, with a job not found on job tracker | ||
| * </pre> | ||
| */ | ||
| public static void killAppOnCluster(final YarnClient yarnClient, final String applicationId, | ||
| final Logger log) throws YarnException, IOException { | ||
|
|
||
| final String[] split = applicationId.split("_"); | ||
| final ApplicationId aid = ApplicationId.newInstance(Long.parseLong(split[1]), | ||
| Integer.parseInt(split[2])); | ||
| log.info("start killing application: " + aid); | ||
| yarnClient.killApplication(aid); | ||
| log.info("successfully killed application: " + aid); | ||
| } | ||
|
|
||
| /** | ||
| * Create, initialize and start a YarnClient connecting to the Yarn Cluster (resource manager), | ||
| * using the resources passed in with props. | ||
| * | ||
| * @param props the properties to create a YarnClient, the path to the "yarn-site.xml" to be used | ||
| */ | ||
| public static YarnClient createYarnClient(Props props) { | ||
| final YarnConfiguration yarnConf = new YarnConfiguration(); | ||
| if (props.containsKey(YARN_CONF_DIRECTORY_PROPERTY)) { | ||
| yarnConf.addResource( | ||
| new Path(props.get(YARN_CONF_DIRECTORY_PROPERTY) + "/" + YARN_CONF_FILENAME)); | ||
| } | ||
| final YarnClient yarnClient = YarnClient.createYarnClient(); | ||
| yarnClient.init(yarnConf); | ||
| yarnClient.start(); | ||
| return yarnClient; | ||
| } | ||
| } |
Oops, something went wrong.