From bf2d73a2f9aad5dd2dcb93e3d452b5235b80e639 Mon Sep 17 00:00:00 2001 From: Janne Valkealahti Date: Mon, 11 Aug 2014 12:53:03 +0100 Subject: [PATCH] SHDP-211 Container Grouping - Initial import from dev branches - Container grouping and clustering implementation. - Internal statemachine which is intentionally left un-documented. - Integration with boot autoconfiguration and configuration properties. - Enhanced libs for building custom client side apps. This brings 'app moded' up to date in terms of how end to end lifecycle is handled. --- build.gradle | 25 + gradle.properties | 2 + settings.gradle | 2 +- .../boot/cli/AbstractApplicationCommand.java | 59 +++ .../yarn/boot/cli/AbstractCli.java | 89 ++++ .../yarn/boot/cli/CliSystemConstants.java | 49 ++ .../boot/cli/YarnClusterCreateCommand.java | 172 +++++++ .../boot/cli/YarnClusterDestroyCommand.java | 71 +++ .../yarn/boot/cli/YarnClusterInfoCommand.java | 78 ++++ .../boot/cli/YarnClusterModifyCommand.java | 136 ++++++ .../boot/cli/YarnClusterStartCommand.java | 71 +++ .../yarn/boot/cli/YarnClusterStopCommand.java | 71 +++ .../boot/cli/YarnClustersInfoCommand.java | 64 +++ .../yarn/boot/cli/YarnKillCommand.java | 62 +++ .../yarn/boot/cli/YarnPushCommand.java | 62 +++ .../yarn/boot/cli/YarnPushedCommand.java | 48 ++ .../yarn/boot/cli/YarnSubmitCommand.java | 60 +++ .../yarn/boot/cli/YarnSubmittedCommand.java | 80 ++++ .../yarn/boot/cli/shell/AnsiString.java | 80 ++++ .../yarn/boot/cli/shell/ClearCommand.java | 45 ++ .../yarn/boot/cli/shell/CommandCompleter.java | 146 ++++++ ...scapeAwareWhiteSpaceArgumentDelimiter.java | 102 ++++ .../yarn/boot/cli/shell/ExitCommand.java | 38 ++ .../boot/cli/shell/ForkProcessCommand.java | 81 ++++ .../yarn/boot/cli/shell/PromptCommand.java | 50 ++ .../boot/cli/shell/RunProcessCommand.java | 63 +++ .../yarn/boot/cli/shell/Shell.java | 204 ++++++++ .../yarn/boot/cli/shell/ShellCommand.java | 45 ++ .../boot/cli/shell/ShellExitException.java | 33 ++ .../yarn/boot/cli/shell/ShellPrompts.java | 56 +++ ...inerClusterAppmasterAutoConfiguration.java | 38 ++ .../boot/YarnAppmasterAutoConfiguration.java | 149 +++++- .../YarnContainerClusterEndpoint.java | 96 ++++ .../mvc/AbstractContainerClusterRequest.java | 62 +++ .../mvc/ContainerClusterCreateRequest.java | 84 ++++ .../mvc/ContainerClusterModifyRequest.java | 44 ++ .../mvc/YarnContainerClusterMvcEndpoint.java | 243 ++++++++++ .../domain/ContainerAllocateDataResource.java | 53 +++ .../mvc/domain/ContainerClusterResource.java | 65 +++ .../domain/ContainerClusterStateResource.java | 39 ++ .../mvc/domain/GridMemberResource.java | 37 ++ .../mvc/domain/GridProjectionResource.java | 65 +++ .../mvc/domain/SatisfyStateDataResource.java | 47 ++ .../YarnContainerClusterEndpointResource.java | 50 ++ .../app/YarnContainerClusterApplication.java | 355 ++++++++++++++ .../YarnContainerClusterClientException.java | 52 +++ .../app/YarnContainerClusterOperations.java | 99 ++++ .../app/YarnContainerClusterTemplate.java | 140 ++++++ .../yarn/boot/app/YarnInfoApplication.java | 5 +- .../yarn/boot/app/YarnKillApplication.java | 5 +- .../yarn/boot/app/YarnPushApplication.java | 5 +- .../yarn/boot/app/YarnSubmitApplication.java | 5 +- .../condition/OnYarnAppmasterCondition.java | 5 + .../SpringYarnAppmasterProperties.java | 118 +++++ .../BootMultiLocalResourcesSelector.java | 61 +++ .../main/resources/META-INF/spring.factories | 3 +- .../springframework/yarn/boot/MockUtils.java | 90 ++++ .../springframework/yarn/boot/TestUtils.java | 94 ++++ .../YarnAppmasterAutoConfigurationTests.java | 80 ++++ .../YarnContainerClusterMvcEndpointTests.java | 411 +++++++++++++++++ ...erApplicationOperationPropertiesTests.java | 56 +++ .../YarnContainerClusterTemplateTests.java | 131 ++++++ ...masterContainerClusterPropertiesTests.java | 282 ++++++++++++ ...masterContainerClusterPropertiesTests1.yml | 49 ++ ...sterContainerClusterPropertiesTests2-2.yml | 8 + ...masterContainerClusterPropertiesTests2.yml | 43 ++ .../fs/DefaultResourceLocalizerTests.java | 17 - .../yarn/am/AbstractAppmaster.java | 58 ++- .../am/allocate/ContainerAllocateData.java | 11 +- .../allocate/DefaultAllocateCountTracker.java | 17 +- .../allocate/DefaultContainerAllocator.java | 177 +++++-- .../AbstractContainerClusterAppmaster.java | 434 ++++++++++++++++++ .../am/cluster/ClusterAllocatingAction.java | 45 ++ .../am/cluster/ClusterDestroyingAction.java | 31 ++ .../yarn/am/cluster/ClusterEvent.java | 35 ++ .../yarn/am/cluster/ClusterState.java | 69 +++ .../am/cluster/ClusterStoppingAction.java | 37 ++ .../yarn/am/cluster/ContainerCluster.java | 34 ++ .../am/cluster/ContainerClusterAppmaster.java | 46 ++ .../yarn/am/cluster/ContainerClusterInfo.java | 38 ++ ...ainerClusterStateMachineConfiguration.java | 115 +++++ .../am/cluster/DefaultContainerCluster.java | 65 +++ .../ManagedContainerClusterAppmaster.java | 27 ++ .../springframework/yarn/am/grid/Grid.java | 87 ++++ .../yarn/am/grid/GridMember.java | 43 ++ .../yarn/am/grid/GridProjection.java | 69 +++ .../yarn/am/grid/GridProjectionFactory.java | 49 ++ .../yarn/am/grid/ProjectedGrid.java | 55 +++ .../am/grid/listener/DefaultGridListener.java | 46 ++ .../DefaultProjectedGridListener.java | 61 +++ .../yarn/am/grid/listener/GridListener.java | 42 ++ .../grid/listener/ProjectedGridListener.java | 59 +++ .../ProjectedGridListenerAdapter.java | 39 ++ .../yarn/am/grid/support/AbstractGrid.java | 134 ++++++ .../am/grid/support/AbstractGridMember.java | 40 ++ .../grid/support/AbstractGridProjection.java | 235 ++++++++++ .../grid/support/AbstractProjectedGrid.java | 159 +++++++ .../am/grid/support/AnyGridProjection.java | 82 ++++ .../yarn/am/grid/support/DefaultGrid.java | 20 + .../am/grid/support/DefaultGridMember.java | 27 ++ .../support/DefaultGridProjectionFactory.java | 67 +++ .../am/grid/support/DefaultProjectedGrid.java | 26 ++ .../grid/support/GridMemberInterceptor.java | 25 + .../support/GridMemberInterceptorChain.java | 67 +++ .../am/grid/support/HostsGridProjection.java | 98 ++++ .../yarn/am/grid/support/ProjectionData.java | 102 ++++ .../am/grid/support/RacksGridProjection.java | 115 +++++ .../am/grid/support/SatisfyStateData.java | 63 +++ .../builders/SpringYarnConfigBuilder.java | 4 +- .../builders/YarnAppmasterBuilder.java | 28 +- .../builders/YarnAppmasterConfigurer.java | 14 +- .../builders/YarnEnvironmentBuilder.java | 141 ++++-- .../builders/YarnEnvironmentConfigurer.java | 56 ++- .../YarnResourceLocalizerBuilder.java | 18 + .../YarnResourceLocalizerConfigurer.java | 23 +- ...DefaultEnvironmentClasspathConfigurer.java | 25 +- .../DefaultLocalResourcesHdfsConfigurer.java | 27 +- ...ultMasterContainerAllocatorConfigurer.java | 91 +++- .../MasterContainerAllocatorConfigurer.java | 85 ++++ .../fs/DefaultMultiResourceLocalizer.java | 62 +++ .../yarn/fs/LocalResourcesFactoryBean.java | 36 +- .../yarn/fs/MultiLocalResourcesSelector.java | 42 ++ .../yarn/fs/MultiResourceLocalizer.java | 42 ++ .../console/ContainerClusterReport.java | 342 ++++++++++++++ .../statemachine/AbstractStateMachine.java | 173 +++++++ .../statemachine/EnumStateMachine.java | 30 ++ .../support/statemachine/StateMachine.java | 32 ++ .../StateMachineSystemConstants.java | 26 ++ .../support/statemachine/action/Action.java | 24 + .../config/EnableStateMachine.java | 42 ++ .../config/EnableStateMachineFactory.java | 42 ++ .../EnumStateMachineConfigurerAdapter.java | 20 + .../config/EnumStateMachineFactory.java | 79 ++++ .../config/StateMachineConfig.java | 39 ++ .../config/StateMachineConfigurerAdapter.java | 73 +++ .../config/StateMachineFactory.java | 24 + .../builders/StateMachineConfigBuilder.java | 36 ++ .../builders/StateMachineConfigurer.java | 28 ++ .../builders/StateMachineStateBuilder.java | 67 +++ .../builders/StateMachineStateConfigurer.java | 24 + .../config/builders/StateMachineStates.java | 59 +++ .../StateMachineTransitionBuilder.java | 63 +++ .../StateMachineTransitionConfigurer.java | 24 + .../builders/StateMachineTransitions.java | 59 +++ .../StateMachineConfiguration.java | 60 +++ .../StateMachineFactoryConfiguration.java | 59 +++ .../DefaultExternalTransitionConfigurer.java | 68 +++ .../configurers/DefaultStateConfigurer.java | 67 +++ .../ExternalTransitionConfigurer.java | 33 ++ .../config/configurers/StateConfigurer.java | 34 ++ .../statemachine/state/AbstractState.java | 49 ++ .../support/statemachine/state/EnumState.java | 36 ++ .../support/statemachine/state/State.java | 26 ++ .../AbstractExternalTransition.java | 30 ++ .../AbstractInternalTransition.java | 29 ++ .../transition/AbstractLocalTransition.java | 29 ++ .../transition/AbstractTransition.java | 86 ++++ .../transition/DefaultExternalTransition.java | 29 ++ .../transition/DefaultInternalTransition.java | 29 ++ .../statemachine/transition/Transition.java | 45 ++ .../transition/TransitionKind.java | 26 ++ .../statemachine/trigger/EventTrigger.java | 31 ++ .../support/statemachine/trigger/Trigger.java | 22 + .../org/springframework/yarn/MockUtils.java | 90 ++++ .../org/springframework/yarn/TestUtils.java | 12 + .../DefaultContainerAllocatorTests.java | 154 ++++++- ...ManagedContainerClusterAppmasterTests.java | 263 +++++++++++ ...edContainerClusterAppmasterMultiTests.java | 209 +++++++++ ...dContainerClusterAppmasterSingleTests.java | 309 +++++++++++++ .../yarn/am/grid/GridTests.java | 41 ++ .../grid/support/AnyGridProjectionTests.java | 122 +++++ .../support/DefaultProjectedGridTests.java | 56 +++ .../support/HostsGridProjectionTests.java | 171 +++++++ .../support/RacksGridProjectionTests.java | 131 ++++++ .../grid/support/TestDNSToSwitchMapping.java | 29 ++ ...YarnConfigurationMasterAllocatorTests.java | 74 +++ .../AbstractStateMachineTests.java | 14 + .../support/statemachine/ActionTests.java | 117 +++++ .../statemachine/ConfigurationTests.java | 96 ++++ .../statemachine/EnumStateMachineTests.java | 195 ++++++++ .../StateMachineFactoryTests.java | 66 +++ .../statemachine/StateMachineTests.java | 112 +++++ .../support/statemachine/TransitionTests.java | 68 +++ 183 files changed, 13327 insertions(+), 169 deletions(-) create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractApplicationCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractCli.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/CliSystemConstants.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterCreateCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterDestroyCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterInfoCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterModifyCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStartCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStopCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClustersInfoCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnKillCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushedCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmitCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmittedCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/AnsiString.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ClearCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/CommandCompleter.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/EscapeAwareWhiteSpaceArgumentDelimiter.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ExitCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ForkProcessCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/PromptCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/RunProcessCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/Shell.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellCommand.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellExitException.java create mode 100644 spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellPrompts.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/ContainerClusterAppmasterAutoConfiguration.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/YarnContainerClusterEndpoint.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/AbstractContainerClusterRequest.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterCreateRequest.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterModifyRequest.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpoint.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerAllocateDataResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterStateResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridMemberResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridProjectionResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/SatisfyStateDataResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/YarnContainerClusterEndpointResource.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterApplication.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterClientException.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterOperations.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplate.java create mode 100644 spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/support/BootMultiLocalResourcesSelector.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/MockUtils.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/TestUtils.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/YarnAppmasterAutoConfigurationTests.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpointTests.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterApplicationOperationPropertiesTests.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplateTests.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterContainerClusterPropertiesTests.java create mode 100644 spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests1.yml create mode 100644 spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2-2.yml create mode 100644 spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2.yml create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/AbstractContainerClusterAppmaster.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterAllocatingAction.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterDestroyingAction.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterEvent.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterState.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterStoppingAction.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerCluster.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterAppmaster.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterInfo.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterStateMachineConfiguration.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/DefaultContainerCluster.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmaster.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/Grid.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridMember.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjection.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjectionFactory.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/ProjectedGrid.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultGridListener.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultProjectedGridListener.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/GridListener.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListener.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListenerAdapter.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGrid.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridMember.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridProjection.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractProjectedGrid.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AnyGridProjection.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGrid.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridMember.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridProjectionFactory.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultProjectedGrid.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptor.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptorChain.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/HostsGridProjection.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/ProjectionData.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/RacksGridProjection.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/SatisfyStateData.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/DefaultMultiResourceLocalizer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiLocalResourcesSelector.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiResourceLocalizer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/console/ContainerClusterReport.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/AbstractStateMachine.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/EnumStateMachine.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachine.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachineSystemConstants.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/action/Action.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachine.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachineFactory.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineConfigurerAdapter.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineFactory.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfig.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfigurerAdapter.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineFactory.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigBuilder.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateBuilder.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStates.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionBuilder.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitions.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineConfiguration.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineFactoryConfiguration.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultExternalTransitionConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultStateConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/ExternalTransitionConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/StateConfigurer.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/AbstractState.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/EnumState.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/State.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractExternalTransition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractInternalTransition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractLocalTransition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractTransition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultExternalTransition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultInternalTransition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/Transition.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/TransitionKind.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/EventTrigger.java create mode 100644 spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/Trigger.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/MockUtils.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/AbstractManagedContainerClusterAppmasterTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterMultiTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterSingleTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/GridTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/AnyGridProjectionTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/DefaultProjectedGridTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/HostsGridProjectionTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/RacksGridProjectionTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/TestDNSToSwitchMapping.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/config/annotation/SpringYarnConfigurationMasterAllocatorTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/AbstractStateMachineTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ActionTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ConfigurationTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/EnumStateMachineTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineFactoryTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineTests.java create mode 100644 spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/TransitionTests.java diff --git a/build.gradle b/build.gradle index 3d69821b0..c6dabb972 100644 --- a/build.gradle +++ b/build.gradle @@ -321,6 +321,7 @@ configure(javaProjects()) { eclipse.classpath.file { whenMerged { classpath -> classpath.entries.removeAll { entry -> entry.toString().contains(".pom") } + classpath.entries.removeAll { entry -> entry.toString().contains("servlet-api") } } } @@ -964,6 +965,7 @@ if (gradle.ext.mr2) { description = 'Spring Yarn Core' dependencies { compile project(":spring-data-hadoop") + compile "org.springframework:spring-messaging:$springVersion" compile("org.apache.hadoop:hadoop-yarn-client:$hadoopVersion") { dep -> exclude group: "org.slf4j", module: "slf4j-log4j12" } @@ -1028,12 +1030,35 @@ if (gradle.ext.mr2) { project('spring-yarn:spring-yarn-boot') { description = 'Spring Yarn Boot' + configurations.all { + exclude group: 'javax.servlet', module: 'servlet-api', version: '2.5' + } dependencies { compile project(":spring-yarn:spring-yarn-core") provided project(":spring-yarn:spring-yarn-batch") provided "org.springframework:spring-web:$springVersion" + provided "org.springframework:spring-webmvc:$springVersion" compile "org.springframework.boot:spring-boot-autoconfigure:$springBootVersion" + compile "org.springframework.boot:spring-boot-actuator:$springBootVersion" + compile "org.apache.httpcomponents:httpclient:$httpclientVersion" runtime "org.yaml:snakeyaml:$snakeYamlVersion" + testRuntime("org.springframework.boot:spring-boot-starter-web:$springBootVersion") { + exclude group:'ch.qos.logback' + } + testCompile("org.mockito:mockito-core:$mockitoVersion") { dep -> + exclude group: "org.hamcrest" + } + testCompile "com.jayway.jsonpath:json-path:$jsonpathVersion" + testCompile "com.jayway.jsonpath:json-path-assert:$jsonpathVersion" + } + } + + project('spring-yarn:spring-yarn-boot-cli') { + description = 'Spring Yarn Boot Cli' + dependencies { + compile project(":spring-yarn:spring-yarn-boot") + compile "org.springframework.boot:spring-boot-cli:$springBootVersion" + runtime "org.springframework:spring-web:$springVersion" } } diff --git a/gradle.properties b/gradle.properties index e6f636b38..253388dda 100644 --- a/gradle.properties +++ b/gradle.properties @@ -67,6 +67,8 @@ commonsioVersion = 2.4 cglibVersion = 3.1 snakeYamlVersion = 1.13 kiteVersion = 0.14.0 +httpclientVersion = 4.3.3 +jsonpathVersion = 0.8.1 ## Common testing libraries junitVersion = 4.11 diff --git a/settings.gradle b/settings.gradle index bc3a70909..4f2cbfb15 100644 --- a/settings.gradle +++ b/settings.gradle @@ -60,5 +60,5 @@ include 'spring-hadoop-build-tests' rootProject.children.find{ it.name == 'spring-hadoop-build-tests' }.name = 'spring-data-hadoop-build-tests' if (mr2) { print "Based on selected distro (${hadoopDistro}) we are including spring-yarn\n" - include 'spring-yarn:spring-yarn-core','spring-yarn:spring-yarn-integration','spring-yarn:spring-yarn-batch','spring-yarn:spring-yarn-test','spring-yarn:spring-yarn-build-tests','spring-yarn:spring-yarn-boot','spring-yarn:spring-yarn-boot-test','spring-yarn:spring-yarn-boot-build-tests' + include 'spring-yarn:spring-yarn-core','spring-yarn:spring-yarn-integration','spring-yarn:spring-yarn-batch','spring-yarn:spring-yarn-test','spring-yarn:spring-yarn-build-tests','spring-yarn:spring-yarn-boot','spring-yarn:spring-yarn-boot-cli','spring-yarn:spring-yarn-boot-test','spring-yarn:spring-yarn-boot-build-tests' } diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractApplicationCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractApplicationCommand.java new file mode 100644 index 000000000..6552c07be --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractApplicationCommand.java @@ -0,0 +1,59 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.command.OptionParsingCommand; +import org.springframework.boot.cli.command.options.OptionHandler; +import org.springframework.boot.cli.command.status.ExitStatus; + +/** + * Base class for all commands implemented by + * this cli package. + * + * @author Janne Valkealahti + * + */ +public class AbstractApplicationCommand extends OptionParsingCommand { + + protected AbstractApplicationCommand(String name, String description, OptionHandler handler) { + super(name, description, handler); + } + + protected abstract static class ApplicationOptionHandler extends OptionHandler { + + @Override + protected final ExitStatus run(OptionSet options) throws Exception { + runApplication(options); + return ExitStatus.OK; + } + + protected abstract void runApplication(OptionSet options) throws Exception; + + protected boolean isFlagOn(OptionSet options, OptionSpec option) { + return options.has(option) ? options.valueOf(option) : false; + } + + } + + @Override + public String getUsageHelp() { + return "[options]"; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractCli.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractCli.java new file mode 100644 index 000000000..902b80b61 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/AbstractCli.java @@ -0,0 +1,89 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.CommandRunner; +import org.springframework.boot.cli.command.core.HelpCommand; +import org.springframework.boot.cli.command.core.VersionCommand; +import org.springframework.boot.loader.tools.LogbackInitializer; + +/** + * Base Spring YARN Cli implementation. + * + * @author Janne Valkealahti + * + */ +public abstract class AbstractCli { + + private final List commands = new ArrayList(); + + /** + * Register command. + * + * @param command the command + */ + protected void registerCommand(Command command) { + commands.add(command); + } + + /** + * Register commands. + * + * @param commands the commands + */ + protected void registerCommands(List commands) { + this.commands.addAll(commands); + } + + /** + * Main method which should be called from implementing class. + * + * @param args the program args + */ + protected void doMain(String[] args) { + System.setProperty("java.awt.headless", Boolean.toString(true)); + LogbackInitializer.initialize(); + + CommandRunner runner = new CommandRunner(getMainCommandName()); + runner.addCommand(new HelpCommand(runner)); + + for (Command command : commands) { + runner.addCommand(command); + } + runner.setOptionCommands(HelpCommand.class, VersionCommand.class); + + int exitCode = runner.runAndHandleErrors(args); + if (exitCode != 0) { + // If successful, leave it to run in case it's a server app + System.exit(exitCode); + } + } + + /** + * Get a main command name which can be overwritten + * if default is not suitable for user cli implementation. + * + * @return the main command name + */ + protected String getMainCommandName() { + return "java -jar "; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/CliSystemConstants.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/CliSystemConstants.java new file mode 100644 index 000000000..f331fd7db --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/CliSystemConstants.java @@ -0,0 +1,49 @@ +package org.springframework.yarn.boot.cli; + +import static java.util.Arrays.asList; + +import java.util.List; + +public abstract class CliSystemConstants { + + public final static List OPTIONS_APPLICATION_ID = asList("application-id", "a"); + + public final static List OPTIONS_APPLICATION_TYPE = asList("application-type", "t"); + + public final static List OPTIONS_APPLICATION_VERSION = asList("application-version", "v"); + + public final static List OPTIONS_CLUSTER_ID = asList("cluster-id", "c"); + + public final static List OPTIONS_CLUSTER_DEF = asList("cluster-def", "i"); + + public final static List OPTIONS_VERBOSE = asList("verbose", "v"); + + public final static List OPTIONS_PROJECTION_TYPE = asList("projection-type", "p"); + + public final static List OPTIONS_PROJECTION_ANY = asList("projection-any", "w"); + + public final static List OPTIONS_PROJECTION_HOSTS = asList("projection-hosts", "h"); + + public final static List OPTIONS_PROJECTION_RACKS = asList("projection-racks", "r"); + + public final static String DESC_APPLICATION_ID = "Specify YARN application id"; + + public final static String DESC_CLUSTER_ID = "Specify cluster id"; + + public final static String DESC_CLUSTER_DEF = "Specify cluster def id"; + + public final static String DESC_APPLICATION_TYPE = "Application type"; + + public final static String DESC_APPLICATION_VERSION = "Application version"; + + public final static String DESC_VERBOSE = "Verbose output"; + + public final static String DESC_PROJECTION_TYPE = "Projection type"; + + public final static String DESC_PROJECTION_ANY = "Projection any count"; + + public final static String DESC_PROJECTION_HOSTS = "Projection hosts counts"; + + public final static String DESC_PROJECTION_RACKS = "Projection racks counts"; + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterCreateCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterCreateCommand.java new file mode 100644 index 000000000..065597045 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterCreateCommand.java @@ -0,0 +1,172 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.command.options.OptionHandler; +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command creating a container cluster. + * + * @author Janne Valkealahti + * + */ +public class YarnClusterCreateCommand extends AbstractApplicationCommand { + + public YarnClusterCreateCommand() { + super("clustercreate", "Create cluster", new ClusterCreateOptionHandler()); + } + + public YarnClusterCreateCommand(OptionHandler handler) { + super("clustercreate", "Create cluster", handler); + } + + public YarnClusterCreateCommand(String name, String description, OptionHandler handler) { + super(name, description, handler); + } + + protected static class ClusterCreateOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + private OptionSpec clusterIdOption; + + private OptionSpec clusterDefOption; + + private OptionSpec projectionTypeOption; + + private OptionSpec projectionDataAnyOption; + + private OptionSpec projectionDataHostsOption; + + private OptionSpec projectionDataRacksOption; + + @Override + protected void options() { + this.applicationIdOption = option(CliSystemConstants.OPTIONS_APPLICATION_ID, + CliSystemConstants.DESC_APPLICATION_ID).withRequiredArg(); + this.clusterIdOption = option(CliSystemConstants.OPTIONS_CLUSTER_ID, CliSystemConstants.DESC_CLUSTER_ID) + .withRequiredArg(); + this.clusterDefOption = option(CliSystemConstants.OPTIONS_CLUSTER_DEF, CliSystemConstants.DESC_CLUSTER_DEF) + .withOptionalArg(); + this.projectionTypeOption = option(CliSystemConstants.OPTIONS_PROJECTION_TYPE, + CliSystemConstants.DESC_PROJECTION_TYPE).withRequiredArg(); + this.projectionDataAnyOption = option(CliSystemConstants.OPTIONS_PROJECTION_ANY, + CliSystemConstants.DESC_PROJECTION_ANY).withRequiredArg(); + this.projectionDataHostsOption = option(CliSystemConstants.OPTIONS_PROJECTION_HOSTS, + CliSystemConstants.DESC_PROJECTION_HOSTS).withOptionalArg().withValuesSeparatedBy(","); + this.projectionDataRacksOption = option(CliSystemConstants.OPTIONS_PROJECTION_RACKS, + CliSystemConstants.DESC_PROJECTION_RACKS).withOptionalArg().withValuesSeparatedBy(","); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + String clusterId = options.valueOf(clusterIdOption); + String clusterDef = options.valueOf(clusterDefOption); + String projectionType = options.valueOf(projectionTypeOption); + String projectionAny = options.valueOf(projectionDataAnyOption); + List projectionHosts = options.valuesOf(projectionDataHostsOption); + List projectionRacks = options.valuesOf(projectionDataRacksOption); + Assert.state(StringUtils.hasText(appId) && StringUtils.hasText(clusterId), "Cluster Id and Application Id must be defined"); + + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERCREATE"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterId", + clusterId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterDef", + clusterDef); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.projectionType", + projectionType); + + if (StringUtils.hasText(projectionAny)) { + appProperties + .setProperty("spring.yarn.internal.ContainerClusterApplication.projectionDataAny", projectionAny); + } + + for (Entry entry : getMapFromString(projectionHosts).entrySet()) { + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.projectionDataHosts." + + entry.getKey(), entry.getValue().toString()); + } + + for (Entry entry : getMapFromString(projectionRacks).entrySet()) { + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.projectionDataRacks." + + entry.getKey(), entry.getValue().toString()); + } + + Properties extraProperties = getExtraProperties(options); + if (extraProperties != null) { + for (String key : extraProperties.stringPropertyNames()) { + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.extraProperties." + + key, extraProperties.getProperty(key)); + } + } + + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + protected Properties getExtraProperties(OptionSet options) { + return null; + } + + private static Map getMapFromString(List sources) { + HashMap map = new HashMap(); + if (sources != null) { + String currentHost = null; + for (String source : sources) { + if (isNumber(source)) { + map.put(currentHost, Integer.parseInt(source)); + } else { + currentHost = source; + } + if (!map.containsKey(currentHost)) { + map.put(currentHost, 1); + } else { + } + } + } + return map; + } + + private static boolean isNumber(String source) { + try { + Integer.parseInt(source); + return true; + } catch (NumberFormatException e) { + } + return false; + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterDestroyCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterDestroyCommand.java new file mode 100644 index 000000000..efb2192bf --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterDestroyCommand.java @@ -0,0 +1,71 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command destroying a container cluster. + * + * @author Janne Valkealahti + * + */ +public class YarnClusterDestroyCommand extends AbstractApplicationCommand { + + public YarnClusterDestroyCommand() { + super("clusterdestroy", "Destroy cluster", new ClusterDestroyOptionHandler()); + } + + private static final class ClusterDestroyOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + private OptionSpec clusterIdOption; + + @Override + protected final void options() { + this.applicationIdOption = option("application-id", "Specify Yarn Application Id").withRequiredArg(); + this.clusterIdOption = option("cluster-id", "Specify Cluster Id").withRequiredArg(); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + String versionId = options.valueOf(clusterIdOption); + Assert.state(StringUtils.hasText(appId) && StringUtils.hasText(versionId), "Cluster Id and Application Id must be defined"); + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERDESTROY"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterId", + versionId); + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterInfoCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterInfoCommand.java new file mode 100644 index 000000000..8392135c1 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterInfoCommand.java @@ -0,0 +1,78 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command listing info about a container cluster. + * + * @author Janne Valkealahti + * + */ +public class YarnClusterInfoCommand extends AbstractApplicationCommand { + + public YarnClusterInfoCommand() { + super("clusterinfo", "List cluster info", new ClusterInfoOptionHandler()); + } + + private static final class ClusterInfoOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + private OptionSpec clusterIdOption; + + private OptionSpec verboseOption; + + @Override + protected final void options() { + applicationIdOption = option(CliSystemConstants.OPTIONS_APPLICATION_ID, CliSystemConstants.DESC_APPLICATION_ID).withRequiredArg(); + clusterIdOption = option(CliSystemConstants.OPTIONS_CLUSTER_ID, CliSystemConstants.DESC_CLUSTER_ID).withRequiredArg(); + verboseOption = option(CliSystemConstants.OPTIONS_VERBOSE, CliSystemConstants.DESC_VERBOSE).withOptionalArg().ofType(Boolean.class) + .defaultsTo(true); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + String clusterId = options.valueOf(clusterIdOption); + Assert.state(StringUtils.hasText(appId) && StringUtils.hasText(clusterId), "Cluster Id and Application Id must be defined"); + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERINFO"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterId", + clusterId); + if (isFlagOn(options, verboseOption)) { + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.verbose", "true"); + } + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterModifyCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterModifyCommand.java new file mode 100644 index 000000000..a9b8931db --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterModifyCommand.java @@ -0,0 +1,136 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Map.Entry; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command modifying a container cluster. + * + * @author Janne Valkealahti + * + */ +public class YarnClusterModifyCommand extends AbstractApplicationCommand { + + public YarnClusterModifyCommand() { + super("clustermodify", "Modify cluster", new ClusterModifyOptionHandler()); + } + + private static final class ClusterModifyOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + private OptionSpec clusterIdOption; + + private OptionSpec projectionDataAnyOption; + + private OptionSpec projectionDataHostsOption; + + private OptionSpec projectionDataRacksOption; + + @Override + protected final void options() { + this.applicationIdOption = option(CliSystemConstants.OPTIONS_APPLICATION_ID, + CliSystemConstants.DESC_APPLICATION_ID).withRequiredArg(); + this.clusterIdOption = option(CliSystemConstants.OPTIONS_CLUSTER_ID, CliSystemConstants.DESC_CLUSTER_ID) + .withRequiredArg(); + this.projectionDataAnyOption = option(CliSystemConstants.OPTIONS_PROJECTION_ANY, + CliSystemConstants.DESC_PROJECTION_ANY).withRequiredArg(); + this.projectionDataHostsOption = option(CliSystemConstants.OPTIONS_PROJECTION_HOSTS, + CliSystemConstants.DESC_PROJECTION_HOSTS).withOptionalArg().withValuesSeparatedBy(","); + this.projectionDataRacksOption = option(CliSystemConstants.OPTIONS_PROJECTION_RACKS, + CliSystemConstants.DESC_PROJECTION_RACKS).withOptionalArg().withValuesSeparatedBy(","); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + String clusterId = options.valueOf(clusterIdOption); + String projectionAny = options.valueOf(projectionDataAnyOption); + List projectionHosts = options.valuesOf(projectionDataHostsOption); + List projectionRacks = options.valuesOf(projectionDataRacksOption); + Assert.state(StringUtils.hasText(appId) && StringUtils.hasText(clusterId), "Cluster Id and Application Id must be defined"); + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERMODIFY"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterId", + clusterId); + + if (StringUtils.hasText(projectionAny)) { + appProperties + .setProperty("spring.yarn.internal.ContainerClusterApplication.projectionDataAny", projectionAny); + } + + for (Entry entry : getMapFromString(projectionHosts).entrySet()) { + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.projectionDataHosts." + + entry.getKey(), entry.getValue().toString()); + } + + for (Entry entry : getMapFromString(projectionRacks).entrySet()) { + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.projectionDataRacks." + + entry.getKey(), entry.getValue().toString()); + } + + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + private static Map getMapFromString(List sources) { + HashMap map = new HashMap(); + if (sources != null) { + String currentHost = null; + for (String source : sources) { + if (isNumber(source)) { + map.put(currentHost, Integer.parseInt(source)); + } else { + currentHost = source; + } + if (!map.containsKey(currentHost)) { + map.put(currentHost, 1); + } else { + } + } + } + return map; + } + + private static boolean isNumber(String source) { + try { + Integer.parseInt(source); + return true; + } catch (NumberFormatException e) { + } + return false; + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStartCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStartCommand.java new file mode 100644 index 000000000..789173b3b --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStartCommand.java @@ -0,0 +1,71 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command starting a container cluster. + * + * @author Janne Valkealahti + * + */ +public class YarnClusterStartCommand extends AbstractApplicationCommand { + + public YarnClusterStartCommand() { + super("clusterstart", "Start cluster", new ClusterStartOptionHandler()); + } + + private static final class ClusterStartOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + private OptionSpec clusterIdOption; + + @Override + protected final void options() { + this.applicationIdOption = option("application-id", "Specify Yarn Application Id").withRequiredArg(); + this.clusterIdOption = option("cluster-id", "Specify Cluster Id").withRequiredArg(); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + String clusterId = options.valueOf(clusterIdOption); + Assert.state(StringUtils.hasText(appId) && StringUtils.hasText(clusterId), "Cluster Id and Application Id must be defined"); + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERSTART"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterId", + clusterId); + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStopCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStopCommand.java new file mode 100644 index 000000000..d2ea8a9d1 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClusterStopCommand.java @@ -0,0 +1,71 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command stopping a container cluster. + * + * @author Janne Valkealahti + * + */ +public class YarnClusterStopCommand extends AbstractApplicationCommand { + + public YarnClusterStopCommand() { + super("clusterstop", "Stop cluster", new ClusterStopOptionHandler()); + } + + private static final class ClusterStopOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + private OptionSpec clusterIdOption; + + @Override + protected final void options() { + this.applicationIdOption = option("application-id", "Specify Yarn Application Id").withRequiredArg(); + this.clusterIdOption = option("cluster-id", "Specify Cluster Id").withRequiredArg(); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + String clusterId = options.valueOf(clusterIdOption); + Assert.state(StringUtils.hasText(appId) && StringUtils.hasText(clusterId), "Cluster Id and Application Id must be defined"); + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERSTOP"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.clusterId", + clusterId); + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClustersInfoCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClustersInfoCommand.java new file mode 100644 index 000000000..d1f324c77 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnClustersInfoCommand.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication; + +/** + * Command listing info about container clusters. + * + * @author Janne Valkealahti + * + */ +public class YarnClustersInfoCommand extends AbstractApplicationCommand { + + public YarnClustersInfoCommand() { + super("clustersinfo", "List clusters", new ClustersInfoOptionHandler()); + } + + private static final class ClustersInfoOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + @Override + protected final void options() { + this.applicationIdOption = option(CliSystemConstants.OPTIONS_APPLICATION_ID, CliSystemConstants.DESC_APPLICATION_ID).withRequiredArg(); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + Assert.hasText(appId, "Application Id must be defined"); + YarnContainerClusterApplication app = new YarnContainerClusterApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.operation", "CLUSTERSINFO"); + appProperties.setProperty("spring.yarn.internal.ContainerClusterApplication.applicationId", + appId); + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnKillCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnKillCommand.java new file mode 100644 index 000000000..d8db9de35 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnKillCommand.java @@ -0,0 +1,62 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.yarn.boot.app.YarnKillApplication; + +/** + * Command killing yarn applications. + * + * @author Janne Valkealahti + * + */ +public class YarnKillCommand extends AbstractApplicationCommand { + + public YarnKillCommand() { + super("kill", "Kill application", new KillOptionHandler()); + } + + private static final class KillOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationIdOption; + + @Override + protected final void options() { + this.applicationIdOption = option(CliSystemConstants.OPTIONS_APPLICATION_ID, CliSystemConstants.DESC_APPLICATION_ID).withRequiredArg(); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appId = options.valueOf(applicationIdOption); + Assert.hasText(appId, "Application Id must be defined"); + YarnKillApplication app = new YarnKillApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.YarnKillApplication.applicationId", appId); + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushCommand.java new file mode 100644 index 000000000..2152d6ba5 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushCommand.java @@ -0,0 +1,62 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.yarn.boot.app.YarnPushApplication; + +/** + * Command pushing an application package into hdfs. + * + * @author Janne Valkealahti + * + */ +public class YarnPushCommand extends AbstractApplicationCommand { + + public YarnPushCommand() { + super("push", "Push new application version", new PushOptionHandler()); + } + + private static final class PushOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationVersionOption; + + @Override + protected final void options() { + this.applicationVersionOption = option(CliSystemConstants.OPTIONS_APPLICATION_VERSION, + CliSystemConstants.DESC_APPLICATION_VERSION).withOptionalArg().defaultsTo("app"); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appVersion = options.valueOf(applicationVersionOption); + YarnPushApplication app = new YarnPushApplication(); + app.applicationVersion(appVersion); + Properties instanceProperties = new Properties(); + instanceProperties.setProperty("spring.yarn.applicationVersion", appVersion); + app.configFile("application.properties", instanceProperties); + app.run(); + Log.info("New instance " + appVersion + " installed"); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushedCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushedCommand.java new file mode 100644 index 000000000..d85c80400 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnPushedCommand.java @@ -0,0 +1,48 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import org.springframework.boot.cli.command.AbstractCommand; +import org.springframework.boot.cli.command.status.ExitStatus; +import org.springframework.boot.cli.util.Log; +import org.springframework.yarn.boot.app.YarnInfoApplication; + +/** + * Command listing pushed application from hdfs. + * + * @author Janne Valkealahti + * + */ +public class YarnPushedCommand extends AbstractCommand { + + public YarnPushedCommand() { + super("pushed", "List pushed applications"); + } + + @Override + public ExitStatus run(String... args) throws Exception { + YarnInfoApplication app = new YarnInfoApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.YarnInfoApplication.operation", "PUSHED"); + app.appProperties(appProperties); + String info = app.run(args); + Log.info(info); + return ExitStatus.OK; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmitCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmitCommand.java new file mode 100644 index 000000000..f654fafbd --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmitCommand.java @@ -0,0 +1,60 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.springframework.boot.cli.util.Log; +import org.springframework.util.Assert; +import org.springframework.yarn.boot.app.YarnSubmitApplication; + +/** + * Command submitting a new application instance. + * + * @author Janne Valkealahti + * + */ +public class YarnSubmitCommand extends AbstractApplicationCommand { + + public YarnSubmitCommand() { + super("submit", "Submit application", new SubmitOptionHandler()); + } + + private static final class SubmitOptionHandler extends ApplicationOptionHandler { + + private OptionSpec applicationVersionOption; + + @Override + protected final void options() { + this.applicationVersionOption = option(CliSystemConstants.OPTIONS_APPLICATION_VERSION, + CliSystemConstants.DESC_APPLICATION_VERSION).withOptionalArg().defaultsTo("app"); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + String appVersion = options.valueOf(applicationVersionOption); + Assert.hasText(appVersion, "Application version must be defined"); + YarnSubmitApplication app = new YarnSubmitApplication(); + app.applicationVersion(appVersion); + ApplicationId applicationId = app.run(new String[0]); + Log.info("New instance submitted with id " + applicationId); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmittedCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmittedCommand.java new file mode 100644 index 000000000..df5e801e3 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/YarnSubmittedCommand.java @@ -0,0 +1,80 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli; + +import java.util.Properties; + +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +import org.springframework.boot.cli.util.Log; +import org.springframework.util.StringUtils; +import org.springframework.yarn.boot.app.YarnInfoApplication; + +/** + * Command listing submitted applications. + * + * @author Janne Valkealahti + * + */ +public class YarnSubmittedCommand extends AbstractApplicationCommand { + + public YarnSubmittedCommand() { + this(null); + } + + public YarnSubmittedCommand(String defaultAppType) { + super("submitted", "List submitted applications", new SubmittedOptionHandler(defaultAppType)); + } + + private static final class SubmittedOptionHandler extends ApplicationOptionHandler { + + private String defaultAppType = "BOOT"; + + private OptionSpec typeOption; + + private OptionSpec verboseOption; + + private SubmittedOptionHandler(String defaultAppType) { + if (StringUtils.hasText(defaultAppType)) { + this.defaultAppType = defaultAppType; + } + } + + @Override + protected final void options() { + typeOption = option(CliSystemConstants.OPTIONS_APPLICATION_TYPE, CliSystemConstants.DESC_APPLICATION_TYPE).withOptionalArg().defaultsTo(defaultAppType); + verboseOption = option(CliSystemConstants.OPTIONS_VERBOSE, CliSystemConstants.DESC_VERBOSE).withOptionalArg().ofType(Boolean.class) + .defaultsTo(true); + } + + @Override + protected void runApplication(OptionSet options) throws Exception { + YarnInfoApplication app = new YarnInfoApplication(); + Properties appProperties = new Properties(); + appProperties.setProperty("spring.yarn.internal.YarnInfoApplication.operation", "SUBMITTED"); + if (isFlagOn(options, verboseOption)) { + appProperties.setProperty("spring.yarn.internal.YarnInfoApplication.verbose", "true"); + } + appProperties.setProperty("spring.yarn.internal.YarnInfoApplication.type", options.valueOf(typeOption)); + app.appProperties(appProperties); + String info = app.run(new String[0]); + Log.info(info); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/AnsiString.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/AnsiString.java new file mode 100644 index 000000000..0178cc451 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/AnsiString.java @@ -0,0 +1,80 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import jline.Terminal; + +import org.fusesource.jansi.Ansi; +import org.fusesource.jansi.AnsiRenderer.Code; + +/** + * Simple utitliy class to build an ANSI string when supported by the {@link Terminal}. + * + * @author Phillip Webb + */ +class AnsiString { + + private final Terminal terminal; + + private final StringBuilder value = new StringBuilder(); + + /** + * Create a new {@link AnsiString} for the given {@link Terminal}. + * @param terminal the terminal used to test if {@link Terminal#isAnsiSupported() ANSI + * is supported}. + */ + AnsiString(Terminal terminal) { + this.terminal = terminal; + } + + /** + * Append text with the given ANSI codes + * @param text the text to append + * @param codes the ANSI codes + * @return this string + */ + AnsiString append(String text, Code... codes) { + if (codes.length == 0 || !isAnsiSupported()) { + this.value.append(text); + return this; + } + Ansi ansi = Ansi.ansi(); + for (Code code : codes) { + ansi = applyCode(ansi, code); + } + this.value.append(ansi.a(text).reset().toString()); + return this; + } + + private Ansi applyCode(Ansi ansi, Code code) { + if (code.isColor()) { + if (code.isBackground()) { + return ansi.bg(code.getColor()); + } + return ansi.fg(code.getColor()); + } + return ansi.a(code.getAttribute()); + } + + private boolean isAnsiSupported() { + return this.terminal.isAnsiSupported(); + } + + @Override + public String toString() { + return this.value.toString(); + } +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ClearCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ClearCommand.java new file mode 100644 index 000000000..4fad03442 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ClearCommand.java @@ -0,0 +1,45 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import jline.console.ConsoleReader; + +import org.springframework.boot.cli.command.AbstractCommand; +import org.springframework.boot.cli.command.status.ExitStatus; + +/** + * Clear the {@link Shell} screen. + * + * @author Dave Syer + * @author Phillip Webb + */ +class ClearCommand extends AbstractCommand { + + private final ConsoleReader consoleReader; + + public ClearCommand(ConsoleReader consoleReader) { + super("clear", "Clear the screen"); + this.consoleReader = consoleReader; + } + + @Override + public ExitStatus run(String... args) throws Exception { + this.consoleReader.setPrompt(""); + this.consoleReader.clearScreen(); + return ExitStatus.OK; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/CommandCompleter.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/CommandCompleter.java new file mode 100644 index 000000000..0b2fbeb6c --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/CommandCompleter.java @@ -0,0 +1,146 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import jline.console.ConsoleReader; +import jline.console.completer.AggregateCompleter; +import jline.console.completer.ArgumentCompleter; +import jline.console.completer.ArgumentCompleter.ArgumentDelimiter; +import jline.console.completer.Completer; +import jline.console.completer.FileNameCompleter; +import jline.console.completer.StringsCompleter; + +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.options.OptionHelp; +import org.springframework.boot.cli.util.Log; + +/** + * JLine {@link Completer} for Spring Boot {@link Command}s. + * + * @author Jon Brisbin + * @author Phillip Webb + */ +public class CommandCompleter extends StringsCompleter { + + private final Map commandCompleters = new HashMap(); + + private final List commands = new ArrayList(); + + private final ConsoleReader console; + + public CommandCompleter(ConsoleReader consoleReader, + ArgumentDelimiter argumentDelimiter, Iterable commands) { + this.console = consoleReader; + List names = new ArrayList(); + for (Command command : commands) { + this.commands.add(command); + names.add(command.getName()); + List options = new ArrayList(); + for (OptionHelp optionHelp : command.getOptionsHelp()) { + options.addAll(optionHelp.getOptions()); + } + AggregateCompleter arguementCompleters = new AggregateCompleter( + new StringsCompleter(options), new FileNameCompleter()); + ArgumentCompleter argumentCompleter = new ArgumentCompleter( + argumentDelimiter, arguementCompleters); + argumentCompleter.setStrict(false); + this.commandCompleters.put(command.getName(), argumentCompleter); + } + getStrings().addAll(names); + } + + @Override + public int complete(String buffer, int cursor, List candidates) { + int completionIndex = super.complete(buffer, cursor, candidates); + int spaceIndex = buffer.indexOf(' '); + String commandName = (spaceIndex == -1) ? "" : buffer.substring(0, spaceIndex); + if (!"".equals(commandName.trim())) { + for (Command command : this.commands) { + if (command.getName().equals(commandName)) { + if (cursor == buffer.length() && buffer.endsWith(" ")) { + printUsage(command); + break; + } + Completer completer = this.commandCompleters.get(command.getName()); + if (completer != null) { + completionIndex = completer.complete(buffer, cursor, candidates); + break; + } + } + } + } + return completionIndex; + } + + private void printUsage(Command command) { + try { + int maxOptionsLength = 0; + List optionHelpLines = new ArrayList(); + for (OptionHelp optionHelp : command.getOptionsHelp()) { + OptionHelpLine optionHelpLine = new OptionHelpLine(optionHelp); + optionHelpLines.add(optionHelpLine); + maxOptionsLength = Math.max(maxOptionsLength, optionHelpLine.getOptions() + .length()); + } + + this.console.println(); + this.console.println("Usage:"); + this.console.println(command.getName() + " " + command.getUsageHelp()); + for (OptionHelpLine optionHelpLine : optionHelpLines) { + this.console.println(String.format("\t%" + maxOptionsLength + "s: %s", + optionHelpLine.getOptions(), optionHelpLine.getUsage())); + } + this.console.drawLine(); + } + catch (IOException ex) { + Log.error(ex.getMessage() + " (" + ex.getClass().getName() + ")"); + } + } + + /** + * Encapsulated options and usage help. + */ + private static class OptionHelpLine { + + private final String options; + + private final String usage; + + public OptionHelpLine(OptionHelp optionHelp) { + StringBuffer options = new StringBuffer(); + for (String option : optionHelp.getOptions()) { + options.append(options.length() == 0 ? "" : ", "); + options.append(option); + } + this.options = options.toString(); + this.usage = optionHelp.getUsageHelp(); + } + + public String getOptions() { + return this.options; + } + + public String getUsage() { + return this.usage; + } + } +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/EscapeAwareWhiteSpaceArgumentDelimiter.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/EscapeAwareWhiteSpaceArgumentDelimiter.java new file mode 100644 index 000000000..1d4ed69c5 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/EscapeAwareWhiteSpaceArgumentDelimiter.java @@ -0,0 +1,102 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import jline.console.completer.ArgumentCompleter.ArgumentList; +import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter; + +/** + * Escape aware variant of {@link WhitespaceArgumentDelimiter}. + * + * @author Phillip Webb + */ +class EscapeAwareWhiteSpaceArgumentDelimiter extends WhitespaceArgumentDelimiter { + + @Override + public boolean isEscaped(CharSequence buffer, int pos) { + return (isEscapeChar(buffer, pos - 1)); + } + + private boolean isEscapeChar(CharSequence buffer, int pos) { + if (pos >= 0) { + for (char c : getEscapeChars()) { + if (buffer.charAt(pos) == c) { + return !isEscapeChar(buffer, pos - 1); + } + } + } + return false; + } + + @Override + public boolean isQuoted(CharSequence buffer, int pos) { + int closingQuote = searchBackwards(buffer, pos - 1, getQuoteChars()); + if (closingQuote == -1) { + return false; + } + int openingQuote = searchBackwards(buffer, closingQuote - 1, + buffer.charAt(closingQuote)); + if (openingQuote == -1) { + return true; + } + return isQuoted(buffer, openingQuote - 1); + } + + private int searchBackwards(CharSequence buffer, int pos, char... chars) { + while (pos >= 0) { + for (char c : chars) { + if (buffer.charAt(pos) == c && !isEscaped(buffer, pos)) { + return pos; + } + } + pos--; + } + return -1; + } + + public String[] parseArguments(String line) { + ArgumentList delimit = delimit(line, 0); + return cleanArguments(delimit.getArguments()); + } + + private String[] cleanArguments(String[] arguments) { + String[] cleanArguments = new String[arguments.length]; + for (int i = 0; i < arguments.length; i++) { + cleanArguments[i] = cleanArgument(arguments[i]); + } + return cleanArguments; + } + + private String cleanArgument(String argument) { + for (char c : getQuoteChars()) { + String quote = String.valueOf(c); + if (argument.startsWith(quote) && argument.endsWith(quote)) { + return replaceEscapes(argument.substring(1, argument.length() - 1)); + } + } + return replaceEscapes(argument); + } + + private String replaceEscapes(String string) { + string = string.replace("\\ ", " "); + string = string.replace("\\\\", "\\"); + string = string.replace("\\t", "\t"); + string = string.replace("\\\"", "\""); + string = string.replace("\\\'", "\'"); + return string; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ExitCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ExitCommand.java new file mode 100644 index 000000000..4df04a385 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ExitCommand.java @@ -0,0 +1,38 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import org.springframework.boot.cli.command.AbstractCommand; +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.status.ExitStatus; + +/** + * {@link Command} to quit the {@link Shell}. + * + * @author Phillip Webb + */ +class ExitCommand extends AbstractCommand { + + public ExitCommand() { + super("exit", "Quit the embedded shell"); + } + + @Override + public ExitStatus run(String... args) throws Exception { + throw new ShellExitException(); + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ForkProcessCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ForkProcessCommand.java new file mode 100644 index 000000000..8afe72a25 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ForkProcessCommand.java @@ -0,0 +1,81 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.options.OptionHelp; +import org.springframework.boot.cli.command.status.ExitStatus; +import org.springframework.boot.loader.tools.JavaExecutable; + +/** + * Decorate an existing command to run it by forking the current java process. + * + * @author Phillip Webb + */ +class ForkProcessCommand extends RunProcessCommand { + + private static final String MAIN_CLASS = "org.springframework.boot.loader.JarLauncher"; + + private final Command command; + + public ForkProcessCommand(Command command) { + super(new JavaExecutable().toString()); + this.command = command; + } + + @Override + public String getName() { + return this.command.getName(); + } + + @Override + public String getDescription() { + return this.command.getDescription(); + } + + @Override + public String getUsageHelp() { + return this.command.getUsageHelp(); + } + + @Override + public String getHelp() { + return this.command.getHelp(); + } + + @Override + public Collection getOptionsHelp() { + return this.command.getOptionsHelp(); + } + + @Override + public ExitStatus run(String... args) throws Exception { + List fullArgs = new ArrayList(); + fullArgs.add("-cp"); + fullArgs.add(System.getProperty("java.class.path")); + fullArgs.add(MAIN_CLASS); + fullArgs.add(this.command.getName()); + fullArgs.addAll(Arrays.asList(args)); + run(fullArgs); + return ExitStatus.OK; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/PromptCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/PromptCommand.java new file mode 100644 index 000000000..0e944984c --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/PromptCommand.java @@ -0,0 +1,50 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import org.springframework.boot.cli.command.AbstractCommand; +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.status.ExitStatus; + +/** + * {@link Command} to change the {@link Shell} prompt. + * + * @author Dave Syer + */ +public class PromptCommand extends AbstractCommand { + + private final ShellPrompts prompts; + + public PromptCommand(ShellPrompts shellPrompts) { + super("prompt", "Change the prompt used with the current 'shell' command. " + + "Execute with no arguments to return to the previous value."); + this.prompts = shellPrompts; + } + + @Override + public ExitStatus run(String... strings) throws Exception { + if (strings.length > 0) { + for (String string : strings) { + this.prompts.pushPrompt(string + " "); + } + } + else { + this.prompts.popPrompt(); + } + return ExitStatus.OK; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/RunProcessCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/RunProcessCommand.java new file mode 100644 index 000000000..833506aaa --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/RunProcessCommand.java @@ -0,0 +1,63 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import org.springframework.boot.cli.command.AbstractCommand; +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.status.ExitStatus; +import org.springframework.boot.loader.tools.RunProcess; + +/** + * Special {@link Command} used to run a process from the shell. NOTE: this command is not + * directly installed into the shell. + * + * @author Phillip Webb + */ +class RunProcessCommand extends AbstractCommand { + + private final String[] command; + private volatile RunProcess process; + + public RunProcessCommand(String... command) { + super(null, null); + this.command = command; + } + + @Override + public ExitStatus run(String... args) throws Exception { + return run(Arrays.asList(args)); + } + + protected ExitStatus run(Collection args) throws IOException { + this.process = new RunProcess(this.command); + int code = this.process.run(args.toArray(new String[args.size()])); + if (code == 0) { + return ExitStatus.OK; + } + else { + return new ExitStatus(code, "EXTERNAL_ERROR"); + } + } + + public boolean handleSigInt() { + return this.process.handleSigInt(); + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/Shell.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/Shell.java new file mode 100644 index 000000000..d8da8ec66 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/Shell.java @@ -0,0 +1,204 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import jline.console.ConsoleReader; +import jline.console.completer.CandidateListCompletionHandler; + +import org.fusesource.jansi.AnsiRenderer.Code; +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.CommandRunner; +import org.springframework.boot.cli.command.core.HelpCommand; +import org.springframework.boot.loader.tools.SignalUtils; +import org.springframework.util.StringUtils; + +/** + * A shell for Spring YARN Cli. Drops the user into an event loop (REPL) where command line + * completion and history are available without relying on OS shell features. + * + * @author Jon Brisbin + * @author Dave Syer + * @author Phillip Webb + * @author Janne Valkealahti + */ +public class Shell { + + private final ShellCommandRunner commandRunner; + + private final ConsoleReader consoleReader; + + private final EscapeAwareWhiteSpaceArgumentDelimiter argumentDelimiter = new EscapeAwareWhiteSpaceArgumentDelimiter(); + + private final ShellPrompts prompts = new ShellPrompts(); + + /** + * Create a new {@link Shell} instance. + * @throws IOException + */ + public Shell(List commands) throws IOException { + attachSignalHandler(); + this.consoleReader = new ConsoleReader(); + this.commandRunner = createCommandRunner(commands); + initializeConsoleReader(); + } + + private ShellCommandRunner createCommandRunner(List commands) { + ShellCommandRunner runner = new ShellCommandRunner(); + runner.addCommand(new HelpCommand(runner)); + if (commands != null) { + runner.addCommands(commands); + } + runner.addCommand(new PromptCommand(this.prompts)); + runner.addCommand(new ClearCommand(this.consoleReader)); + runner.addCommand(new ExitCommand()); + runner.addAliases("exit", "quit"); + runner.addAliases("help", "?"); + runner.addAliases("clear", "cls"); + return runner; + } + + private void initializeConsoleReader() { + this.consoleReader.setHistoryEnabled(true); + this.consoleReader.setBellEnabled(false); + this.consoleReader.setExpandEvents(false); + this.consoleReader.addCompleter(new CommandCompleter(this.consoleReader, + this.argumentDelimiter, this.commandRunner)); + this.consoleReader.setCompletionHandler(new CandidateListCompletionHandler()); + } + + private void attachSignalHandler() { + SignalUtils.attachSignalHandler(new Runnable() { + @Override + public void run() { + handleSigInt(); + } + }); + } + + /** + * Run the shell until the user exists. + * @throws Exception on error + */ + public void run() throws Exception { + printBanner(); + try { + runInputLoop(); + } + catch (Exception ex) { + if (!(ex instanceof ShellExitException)) { + throw ex; + } + } + } + + private void printBanner() { + String version = getClass().getPackage().getImplementationVersion(); + version = (version == null ? "" : " (v" + version + ")"); + System.out.println(ansi("Spring YARN Cli", Code.BOLD).append(version, Code.FAINT)); + System.out.println(ansi("Hit TAB to complete. Type 'help' and hit " + + "RETURN for help, and 'exit' to quit.")); + } + + private void runInputLoop() throws Exception { + String line; + while ((line = this.consoleReader.readLine(getPrompt())) != null) { + while (line.endsWith("\\")) { + line = line.substring(0, line.length() - 1); + line += this.consoleReader.readLine("> "); + } + if (StringUtils.hasLength(line)) { + String[] args = this.argumentDelimiter.parseArguments(line); + this.commandRunner.runAndHandleErrors(args); + } + } + } + + private String getPrompt() { + String prompt = this.prompts.getPrompt(); + return ansi(prompt, Code.FG_BLUE).toString(); + } + + private AnsiString ansi(String text, Code... codes) { + return new AnsiString(this.consoleReader.getTerminal()).append(text, codes); + } + + /** + * Final handle an interrup signal (CTRL-C) + */ + protected void handleSigInt() { + if (this.commandRunner.handleSigInt()) { + return; + } + System.out.println("\nThanks for using Spring Boot"); + System.exit(1); + } + + /** + * Extension of {@link CommandRunner} to deal with {@link RunProcessCommand}s and + * aliases. + */ + private class ShellCommandRunner extends CommandRunner { + + private volatile Command lastCommand; + + private final Map aliases = new HashMap(); + + public ShellCommandRunner() { + super(null); + } + + public void addAliases(String command, String... aliases) { + for (String alias : aliases) { + this.aliases.put(alias, command); + } + } + + @Override + public Command findCommand(String name) { + if (name.startsWith("!")) { + return new RunProcessCommand(name.substring(1)); + } + if (this.aliases.containsKey(name)) { + name = this.aliases.get(name); + } + return super.findCommand(name); + } + + @Override + protected void beforeRun(Command command) { + this.lastCommand = command; + } + + @Override + protected void afterRun(Command command) { + } + + public boolean handleSigInt() { + Command command = this.lastCommand; + if (command != null && command instanceof RunProcessCommand) { + return ((RunProcessCommand) command).handleSigInt(); + } + return false; + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellCommand.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellCommand.java new file mode 100644 index 000000000..8281d7869 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellCommand.java @@ -0,0 +1,45 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import java.util.List; + +import org.springframework.boot.cli.command.AbstractCommand; +import org.springframework.boot.cli.command.Command; +import org.springframework.boot.cli.command.status.ExitStatus; + +/** + * {@link Command} to start a nested REPL shell. + * + * @author Phillip Webb + * @see Shell + */ +public class ShellCommand extends AbstractCommand { + + private List commands; + + public ShellCommand(List commands) { + super("shell", "Start a nested shell"); + this.commands = commands; + } + + @Override + public ExitStatus run(String... args) throws Exception { + new Shell(commands).run(); + return ExitStatus.OK; + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellExitException.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellExitException.java new file mode 100644 index 000000000..9a78e6064 --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellExitException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import org.springframework.boot.cli.command.CommandException; + +/** + * Exception used to stop the {@link Shell}. + * + * @author Phillip Webb + */ +public class ShellExitException extends CommandException { + + private static final long serialVersionUID = 1L; + + public ShellExitException() { + super(Option.RETHROW); + } + +} diff --git a/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellPrompts.java b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellPrompts.java new file mode 100644 index 000000000..514679b9f --- /dev/null +++ b/spring-yarn/spring-yarn-boot-cli/src/main/java/org/springframework/yarn/boot/cli/shell/ShellPrompts.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.cli.shell; + +import java.util.Stack; + +/** + * Abstraction to manage a stack of prompts. + * + * @author Phillip Webb + */ +public class ShellPrompts { + + private static final String DEFAULT_PROMPT = "$ "; + + private final Stack prompts = new Stack(); + + /** + * Push a new prompt to be used by the shell. + * @param prompt the prompt + * @see #popPrompt() + */ + public void pushPrompt(String prompt) { + this.prompts.push(prompt); + } + + /** + * Pop a previously pushed prompt, returning to the previous value. + * @see #pushPrompt(String) + */ + public void popPrompt() { + if (!this.prompts.isEmpty()) { + this.prompts.pop(); + } + } + + /** + * Returns the current prompt. + */ + public String getPrompt() { + return this.prompts.isEmpty() ? DEFAULT_PROMPT : this.prompts.peek(); + } +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/ContainerClusterAppmasterAutoConfiguration.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/ContainerClusterAppmasterAutoConfiguration.java new file mode 100644 index 000000000..b2c2d0cbc --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/ContainerClusterAppmasterAutoConfiguration.java @@ -0,0 +1,38 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot; + +import org.springframework.boot.actuate.autoconfigure.EndpointWebMvcAutoConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.cluster.ContainerClusterStateMachineConfiguration; +import org.springframework.yarn.boot.condition.ConditionalOnYarnAppmaster; +import org.springframework.yarn.config.annotation.EnableYarn; + +@Configuration +@ConditionalOnYarnAppmaster +@ConditionalOnClass(EnableYarn.class) +@ConditionalOnMissingBean(YarnAppmaster.class) +@ConditionalOnExpression("${spring.yarn.appmaster.containercluster.enabled:false}") +@AutoConfigureBefore({EndpointWebMvcAutoConfiguration.class, YarnAppmasterAutoConfiguration.class}) +@Import(ContainerClusterStateMachineConfiguration.class) +public class ContainerClusterAppmasterAutoConfiguration { +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/YarnAppmasterAutoConfiguration.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/YarnAppmasterAutoConfiguration.java index 10c45ccf8..8dabbdbc9 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/YarnAppmasterAutoConfiguration.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/YarnAppmasterAutoConfiguration.java @@ -15,6 +15,8 @@ */ package org.springframework.yarn.boot; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import org.apache.commons.logging.Log; @@ -38,12 +40,18 @@ import org.springframework.yarn.YarnSystemConstants; import org.springframework.yarn.am.AppmasterTrackService; import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.grid.GridProjectionFactory; +import org.springframework.yarn.am.grid.support.DefaultGridProjectionFactory; +import org.springframework.yarn.am.grid.support.ProjectionData; import org.springframework.yarn.batch.support.YarnJobLauncher; +import org.springframework.yarn.boot.actuate.endpoint.YarnContainerClusterEndpoint; +import org.springframework.yarn.boot.actuate.endpoint.mvc.YarnContainerClusterMvcEndpoint; import org.springframework.yarn.boot.condition.ConditionalOnYarnAppmaster; import org.springframework.yarn.boot.properties.SpringHadoopProperties; import org.springframework.yarn.boot.properties.SpringYarnAppmasterLaunchContextProperties; import org.springframework.yarn.boot.properties.SpringYarnAppmasterLocalizerProperties; import org.springframework.yarn.boot.properties.SpringYarnAppmasterProperties; +import org.springframework.yarn.boot.properties.SpringYarnAppmasterProperties.ContainerClustersProperties; import org.springframework.yarn.boot.properties.SpringYarnAppmasterResourceProperties; import org.springframework.yarn.boot.properties.SpringYarnBatchProperties; import org.springframework.yarn.boot.properties.SpringYarnEnvProperties; @@ -52,6 +60,7 @@ import org.springframework.yarn.boot.support.BootApplicationEventTransformer; import org.springframework.yarn.boot.support.BootLocalResourcesSelector; import org.springframework.yarn.boot.support.BootLocalResourcesSelector.Mode; +import org.springframework.yarn.boot.support.BootMultiLocalResourcesSelector; import org.springframework.yarn.boot.support.EmbeddedAppmasterTrackService; import org.springframework.yarn.boot.support.SpringYarnBootUtils; import org.springframework.yarn.config.annotation.EnableYarn; @@ -62,8 +71,10 @@ import org.springframework.yarn.config.annotation.builders.YarnEnvironmentConfigurer; import org.springframework.yarn.config.annotation.builders.YarnResourceLocalizerConfigurer; import org.springframework.yarn.config.annotation.configurers.LocalResourcesHdfsConfigurer; +import org.springframework.yarn.config.annotation.configurers.MasterContainerAllocatorConfigurer; import org.springframework.yarn.fs.LocalResourcesSelector; import org.springframework.yarn.fs.LocalResourcesSelector.Entry; +import org.springframework.yarn.fs.MultiLocalResourcesSelector; import org.springframework.yarn.launch.LaunchCommandsFactoryBean; import org.springframework.yarn.support.YarnContextUtils; @@ -95,15 +106,41 @@ public AppmasterTrackService appmasterTrackService() { } @Configuration - @EnableConfigurationProperties({ SpringYarnAppmasterLocalizerProperties.class }) + @EnableConfigurationProperties({ SpringYarnAppmasterProperties.class, SpringYarnAppmasterLocalizerProperties.class }) public static class LocalResourcesSelectorConfig { + @Autowired + private SpringYarnAppmasterProperties syap; + @Autowired private SpringYarnAppmasterLocalizerProperties syalp; @Bean @ConditionalOnMissingBean(LocalResourcesSelector.class) public LocalResourcesSelector localResourcesSelector() { + + Map selectors = new HashMap(); + if (syap.getContainercluster() != null && syap.getContainercluster().getClusters() != null) { + for (java.util.Map.Entry entry : syap.getContainercluster().getClusters().entrySet()) { + SpringYarnAppmasterLocalizerProperties props = entry.getValue().getLocalizer(); + if (props == null) { + continue; + } + BootLocalResourcesSelector selector = new BootLocalResourcesSelector(Mode.CONTAINER); + if (StringUtils.hasText(props.getZipPattern())) { + selector.setZipArchivePattern(props.getZipPattern()); + } + if (props.getPropertiesNames() != null) { + selector.setPropertiesNames(props.getPropertiesNames()); + } + if (props.getPropertiesSuffixes() != null) { + selector.setPropertiesSuffixes(props.getPropertiesSuffixes()); + } + selector.addPatterns(props.getPatterns()); + selectors.put(entry.getKey(), selector); + } + } + BootLocalResourcesSelector selector = new BootLocalResourcesSelector(Mode.CONTAINER); if (StringUtils.hasText(syalp.getZipPattern())) { selector.setZipArchivePattern(syalp.getZipPattern()); @@ -115,7 +152,7 @@ public LocalResourcesSelector localResourcesSelector() { selector.setPropertiesSuffixes(syalp.getPropertiesSuffixes()); } selector.addPatterns(syalp.getPatterns()); - return selector; + return new BootMultiLocalResourcesSelector(selector, selectors); } } @@ -177,6 +214,57 @@ public YarnJobLauncher yarnJobLauncher() { } + @Configuration + @EnableConfigurationProperties({ SpringYarnAppmasterProperties.class }) + @ConditionalOnExpression("${spring.yarn.appmaster.containercluster.enabled:false}") + public static class ContainerClusterConfig { + + @Autowired + private SpringYarnAppmasterProperties syap; + + @Bean + public GridProjectionFactory gridProjectionFactory() { + Map projections = new HashMap(); + Map clusterProps = syap.getContainercluster().getClusters(); + if (clusterProps != null) { + for (java.util.Map.Entry entry : clusterProps.entrySet()) { + ProjectionData data = new ProjectionData(entry.getValue().getProjectionAny(), entry.getValue() + .getProjectionHosts(), entry.getValue().getProjectionRacks()); + data.setType(entry.getValue().getProjectionType()); + SpringYarnAppmasterResourceProperties resource = entry.getValue().getResource(); + if (resource != null) { + data.setPriority(resource.getPriority()); + } + projections.put(entry.getKey(), data); + + } + } + return new DefaultGridProjectionFactory(projections); + } + + } + + @Configuration + @EnableConfigurationProperties({ SpringYarnAppmasterProperties.class }) + @ConditionalOnExpression("${spring.yarn.endpoints.containercluster.enabled:false}") + public static class EndPointConfig { + + @Autowired + private SpringYarnAppmasterProperties syap; + + @Bean + @ConditionalOnMissingBean + public YarnContainerClusterEndpoint yarnContainerClusterEndpoint() { + return new YarnContainerClusterEndpoint(); + } + + @Bean + @ConditionalOnBean(YarnContainerClusterEndpoint.class) + public YarnContainerClusterMvcEndpoint yarnContainerClusterMvcEndpoint(YarnContainerClusterEndpoint delegate) { + return new YarnContainerClusterMvcEndpoint(delegate); + } + + } @Configuration @EnableConfigurationProperties({ SpringHadoopProperties.class, SpringYarnProperties.class, @@ -239,6 +327,16 @@ public void configure(YarnResourceLocalizerConfigurer localizer) throws Exceptio for (Entry e : localResourcesSelector.select(applicationDir != null ? applicationDir : "/")) { withHdfs.hdfs(e.getPath(), e.getType(), applicationDir == null); } + + if (syap.getContainercluster() != null && localResourcesSelector instanceof MultiLocalResourcesSelector && syap.getContainercluster().getClusters() != null) { + MultiLocalResourcesSelector selector = ((MultiLocalResourcesSelector)localResourcesSelector); + for (java.util.Map.Entry entry : syap.getContainercluster().getClusters().entrySet()) { + withHdfs = localizer.withHdfs(entry.getKey()); + for (Entry e : selector.select(entry.getKey(), applicationDir != null ? applicationDir : "/")) { + withHdfs.hdfs(e.getPath(), e.getType(), applicationDir == null); + } + } + } } @Override @@ -254,18 +352,53 @@ public void configure(YarnEnvironmentConfigurer environment) throws Exception { .delimiter(syalcp.getPathSeparator()) .entries(syalcp.getContainerAppClasspath()) .entry(explodedEntryIfZip(syalcp)); + + if (syap.getContainercluster() != null && syap.getContainercluster().getClusters() != null) { + for (java.util.Map.Entry entry : syap.getContainercluster().getClusters().entrySet()) { + SpringYarnAppmasterLaunchContextProperties props = entry.getValue().getLaunchcontext(); + environment + .withClasspath(entry.getKey()) + .includeBaseDirectory(props.isIncludeBaseDirectory()) + .useYarnAppClasspath(props.isUseYarnAppClasspath()) + .useMapreduceAppClasspath(props.isUseMapreduceAppClasspath()) + .siteYarnAppClasspath(syp.getSiteYarnAppClasspath()) + .siteMapreduceAppClasspath(syp.getSiteMapreduceAppClasspath()) + .delimiter(props.getPathSeparator()) + .entries(props.getContainerAppClasspath()) + .entry(explodedEntryIfZip(props)); + } + } } @Override public void configure(YarnAppmasterConfigurer master) throws Exception { master .appmasterClass(syap.getAppmasterClass() != null ? syap.getAppmasterClass() : appmasterClass) - .containerCommands(createContainerCommands(syalcp)) - .withContainerAllocator() - .locality(syalcp.isLocality()) - .memory(syarp.getMemory()) - .priority(syarp.getPriority()) - .virtualCores(syarp.getVirtualCores()); + .containerCommands(createContainerCommands(syalcp)); + + MasterContainerAllocatorConfigurer containerAllocatorConfigurer = master.withContainerAllocator(); + containerAllocatorConfigurer + .locality(syalcp.isLocality()) + .memory(syarp.getMemory()) + .priority(syarp.getPriority()) + .virtualCores(syarp.getVirtualCores()); + + if (syap.getContainercluster() != null && syap.getContainercluster().getClusters() != null) { + for (java.util.Map.Entry entry : syap.getContainercluster().getClusters().entrySet()) { + SpringYarnAppmasterResourceProperties resource = entry.getValue().getResource(); + SpringYarnAppmasterLaunchContextProperties launchcontext = entry.getValue().getLaunchcontext(); + + master + .containerCommands(entry.getKey(), createContainerCommands(launchcontext)); + + containerAllocatorConfigurer + .withCollection(entry.getKey()) + .priority(resource != null ? resource.getPriority() : null) + .memory(resource != null ? resource.getMemory() : null) + .virtualCores(resource != null ? resource.getVirtualCores() : null) + .locality(launchcontext != null ? launchcontext.isLocality() : false); + } + } } } diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/YarnContainerClusterEndpoint.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/YarnContainerClusterEndpoint.java new file mode 100644 index 000000000..b101f50ca --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/YarnContainerClusterEndpoint.java @@ -0,0 +1,96 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint; + +import java.util.Map; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.endpoint.AbstractEndpoint; +import org.springframework.boot.actuate.endpoint.Endpoint; +import org.springframework.util.Assert; +import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.cluster.ContainerCluster; +import org.springframework.yarn.am.cluster.ContainerClusterAppmaster; +import org.springframework.yarn.am.grid.support.ProjectionData; + +/** + * {@link Endpoint} handling operations against {@link ContainerClusterAppmaster}. + * + * @author Janne Valkealahti + * + */ +public class YarnContainerClusterEndpoint extends AbstractEndpoint> implements InitializingBean { + + public final static String ENDPOINT_ID = "yarn_containercluster"; + + private ContainerClusterAppmaster appmaster; + + /** + * Instantiates a new yarn container cluster endpoint. + */ + public YarnContainerClusterEndpoint() { + super(ENDPOINT_ID); + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.notNull(appmaster, "ContainerClusterAppmaster must be set"); + } + + @Override + public Map invoke() { + return getClusters(); + } + + /** + * Sets {@link ContainerClusterAppmaster} if passed {@link YarnAppmaster} is + * instance of it. + * + * @param yarnAppmaster the yarn application master + */ + @Autowired + public void setYarnAppmaster(YarnAppmaster yarnAppmaster) { + if (yarnAppmaster instanceof ContainerClusterAppmaster) { + appmaster = ((ContainerClusterAppmaster)yarnAppmaster); + } + } + + public Map getClusters() { + return appmaster.getContainerClusters(); + } + + public ContainerCluster createCluster(String clusterId, String clusterDef, ProjectionData projectionData, Map extraProperties) { + return appmaster.createContainerCluster(clusterId, clusterDef, projectionData, extraProperties); + } + + public void startCluster(String clusterId) { + appmaster.startContainerCluster(clusterId); + } + + public void stopCluster(String clusterId) { + appmaster.stopContainerCluster(clusterId); + } + + public void destroyCluster(String clusterId) { + appmaster.destroyContainerCluster(clusterId); + } + + public void modifyCluster(String id, ProjectionData projectionData) { + appmaster.modifyContainerCluster(id, projectionData); + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/AbstractContainerClusterRequest.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/AbstractContainerClusterRequest.java new file mode 100644 index 000000000..04361d129 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/AbstractContainerClusterRequest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc; + +import java.util.Map; + +/** + * Base class for requests handling clusters. + * + * @author Janne Valkealahti + * + */ +public abstract class AbstractContainerClusterRequest { + + private ProjectionDataType projectionData; + + public ProjectionDataType getProjectionData() { + return projectionData; + } + + public void setProjectionData(ProjectionDataType projectionData) { + this.projectionData = projectionData; + } + + public static class ProjectionDataType { + Integer any; + Map hosts; + Map racks; + public Integer getAny() { + return any; + } + public void setAny(Integer any) { + this.any = any; + } + public Map getHosts() { + return hosts; + } + public void setHosts(Map hosts) { + this.hosts = hosts; + } + public Map getRacks() { + return racks; + } + public void setRacks(Map racks) { + this.racks = racks; + } + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterCreateRequest.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterCreateRequest.java new file mode 100644 index 000000000..60693af5b --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterCreateRequest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc; + +import java.util.Map; + +/** + * + * + * @author Janne Valkealahti + * + */ +public class ContainerClusterCreateRequest extends AbstractContainerClusterRequest { + + private String clusterId; + + private String clusterDef; + + private ProjectionType projection; + + private ProjectionDataType projectionData; + + private Map extraProperties; + + public String getClusterId() { + return clusterId; + } + + public void setClusterId(String clusterId) { + this.clusterId = clusterId; + } + + public String getClusterDef() { + return clusterDef; + } + + public void setClusterDef(String clusterDef) { + this.clusterDef = clusterDef; + } + + public ProjectionType getProjection() { + return projection; + } + + public void setProjection(ProjectionType projection) { + this.projection = projection; + } + + public ProjectionDataType getProjectionData() { + return projectionData; + } + + public void setProjectionData(ProjectionDataType projectionData) { + this.projectionData = projectionData; + } + + public Map getExtraProperties() { + return extraProperties; + } + + public void setExtraProperties(Map extraProperties) { + this.extraProperties = extraProperties; + } + + public enum ProjectionType { + ANY, + HOSTS, + RACKS + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterModifyRequest.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterModifyRequest.java new file mode 100644 index 000000000..7c90ce62d --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/ContainerClusterModifyRequest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc; + +import org.springframework.util.StringUtils; + +public class ContainerClusterModifyRequest extends AbstractContainerClusterRequest { + + private String action; + + public void setAction(String action) { + this.action = action; + } + + public String getAction() { + return action; + } + + public static ModifyAction getModifyAction(String action) { + if (StringUtils.hasText(action)) { + return ModifyAction.valueOf(action.toUpperCase()); + } + return null; + } + + public enum ModifyAction { + START, + STOP + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpoint.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpoint.java new file mode 100644 index 000000000..f9b9bf831 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpoint.java @@ -0,0 +1,243 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc; + +import static org.springframework.web.servlet.mvc.method.annotation.MvcUriComponentsBuilder.on; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +import org.springframework.boot.actuate.endpoint.mvc.EndpointMvcAdapter; +import org.springframework.boot.actuate.endpoint.mvc.MvcEndpoint; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.servlet.mvc.method.annotation.MvcUriComponentsBuilder; +import org.springframework.web.util.UriComponents; +import org.springframework.yarn.am.cluster.ContainerCluster; +import org.springframework.yarn.am.grid.support.ProjectionData; +import org.springframework.yarn.boot.actuate.endpoint.YarnContainerClusterEndpoint; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterModifyRequest.ModifyAction; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.ContainerClusterResource; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.YarnContainerClusterEndpointResource; + +/** + * A custom {@link MvcEndpoint} adding specific rest API used to + * control {@link YarnContainerClusterEndpoint}. + * + * @author Janne Valkealahti + * + */ +@RequestMapping("/" + YarnContainerClusterEndpoint.ENDPOINT_ID) +public class YarnContainerClusterMvcEndpoint extends EndpointMvcAdapter { + + private final YarnContainerClusterEndpoint delegate; + + /** + * Instantiates a new yarn container cluster mvc endpoint. + * + * @param delegate the delegate {@link YarnContainerClusterEndpoint} + */ + public YarnContainerClusterMvcEndpoint(YarnContainerClusterEndpoint delegate) { + super(delegate); + this.delegate = delegate; + } + + /** + * Main {@link EndpointMvcAdapter#invoke()} which returns information + * about existing container clusters. + */ + @RequestMapping(method = RequestMethod.GET) + @ResponseBody + public Object invoke() { + Collection clusters = delegate.getClusters().values(); + YarnContainerClusterEndpointResource response = new YarnContainerClusterEndpointResource(); + Collection clusterIds = new ArrayList(); + for (ContainerCluster cluster : clusters) { + clusterIds.add(cluster.getId()); + } + response.setClusters(clusterIds); + return response; + } + + /** + * Creates a new container cluster. + * + * @param request the container cluster create request + * @return the container cluster create response + */ + @RequestMapping(method = RequestMethod.POST) + public HttpEntity createCluster(@RequestBody ContainerClusterCreateRequest request) { + ProjectionData projectionData = new ProjectionData(); + if (request.getProjectionData().getAny() != null) { + projectionData.setAny(request.getProjectionData().getAny()); + } + if (request.getProjectionData().getHosts() != null) { + projectionData.setHosts(request.getProjectionData().getHosts()); + } + if (request.getProjectionData().getRacks() != null) { + projectionData.setRacks(request.getProjectionData().getRacks()); + } + + if (request.getProjection() == null) { + throw new InvalidInputException("Projection not defined"); + } + + projectionData.setType(request.getProjection().toString().toLowerCase()); + + Map extraProperties = request.getExtraProperties(); + + delegate.createCluster(request.getClusterId(), request.getClusterDef(), projectionData, extraProperties); + + HttpHeaders responseHeaders = new HttpHeaders(); + UriComponents uriComponents = MvcUriComponentsBuilder + .fromMethodCall(on(YarnContainerClusterMvcEndpoint.class).clusterInfo(request.getClusterId())).build(); + responseHeaders.setLocation(uriComponents.toUri()); + + return new ResponseEntity(responseHeaders, HttpStatus.CREATED); + } + + /** + * Gets a status of a specific container cluster. + * + * @param clusterId the container cluster identifier + * @return the container cluster status response + */ + @RequestMapping(value = "/{clusterId}", method = RequestMethod.GET) + public HttpEntity clusterInfo(@PathVariable("clusterId") String clusterId) { + ContainerCluster cluster = delegate.getClusters().get(clusterId); + if (cluster == null) { + throw new NoSuchClusterException("No such cluster: " + clusterId); + } + ContainerClusterResource response = new ContainerClusterResource(cluster); + return new ResponseEntity(response, HttpStatus.OK); + } + + /** + * Modifies a container cluster state. + * + * @param clusterId the container cluster identifier + * @param request Binding for modify request content + * @return the container cluster status response + */ + @RequestMapping(value = "/{clusterId}", method = RequestMethod.PUT) + public HttpEntity modifyCluster(@PathVariable("clusterId") String clusterId, + @RequestBody ContainerClusterModifyRequest request) { + ModifyAction action = ContainerClusterModifyRequest.getModifyAction(request.getAction()); + if (action == null) { + throw new NoSuchActionException("Action " + request.getAction() + " not supported"); + } + ContainerCluster cluster = getClusterMayThrow(clusterId); + if (ModifyAction.START.equals(action)) { + delegate.startCluster(clusterId); + } else if (ModifyAction.STOP.equals(action)) { + delegate.stopCluster(clusterId); + } + ContainerClusterResource response = new ContainerClusterResource(cluster); + return new ResponseEntity(response, + HttpStatus.OK); + } + + @RequestMapping(value = "/{clusterId}", method = RequestMethod.DELETE) + public ResponseEntity destroyCluster(@PathVariable("clusterId") String clusterId) { + ContainerCluster cluster = getClusterMayThrow(clusterId); + delegate.destroyCluster(cluster.getId()); + return new ResponseEntity(HttpStatus.OK); + } + + /** + * Modify a container cluster + * + * @param clusterId the container cluster identifier + * @return the container cluster modify response + */ + @RequestMapping(value = "/{clusterId}", method = RequestMethod.PATCH) + public HttpEntity updateCluster(@PathVariable("clusterId") String clusterId, @RequestBody ContainerClusterCreateRequest request) { + ContainerCluster cluster = delegate.getClusters().get(clusterId); + if (cluster == null) { + throw new NoSuchClusterException("No such cluster: " + clusterId); + } + ProjectionData data = new ProjectionData(); + if (request.getProjectionData().getAny() != null) { + data.setAny(request.getProjectionData().getAny()); + } + if (request.getProjectionData().getHosts() != null) { + data.setHosts(request.getProjectionData().getHosts()); + } + if (request.getProjectionData().getRacks() != null) { + data.setRacks(request.getProjectionData().getRacks()); + } + delegate.modifyCluster(clusterId, data); + + ContainerClusterResource response = new ContainerClusterResource(delegate.getClusters().get(clusterId)); + return new ResponseEntity(response, + HttpStatus.OK); + } + + /** + * Gets a cluster and automatically throws a {@link NoSuchClusterException} + * if cluster doesn't exist. + * + * @param clusterId the cluster id + * @return container cluster if found + */ + private ContainerCluster getClusterMayThrow(String clusterId) { + ContainerCluster cluster = delegate.getClusters().get(clusterId); + if (cluster == null) { + throw new NoSuchClusterException("No such cluster: " + clusterId); + } + return cluster; + } + + @SuppressWarnings("serial") + @ResponseStatus(value = HttpStatus.NOT_FOUND, reason = "No such cluster") + private static class NoSuchClusterException extends RuntimeException { + + public NoSuchClusterException(String string) { + super(string); + } + + } + + @SuppressWarnings("serial") + @ResponseStatus(value = HttpStatus.NOT_FOUND, reason = "No such action") + private static class NoSuchActionException extends RuntimeException { + + public NoSuchActionException(String string) { + super(string); + } + + } + + @SuppressWarnings("serial") + @ResponseStatus(value = HttpStatus.METHOD_NOT_ALLOWED, reason = "Invalid input") + private static class InvalidInputException extends RuntimeException { + + public InvalidInputException(String string) { + super(string); + } + + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerAllocateDataResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerAllocateDataResource.java new file mode 100644 index 000000000..11f294038 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerAllocateDataResource.java @@ -0,0 +1,53 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ContainerAllocateDataResource { + + private Integer any = 0; + + private Map hosts = new ConcurrentHashMap(); + + private Map racks = new ConcurrentHashMap(); + + public void setAny(Integer any) { + this.any = any; + } + + public Integer getAny() { + return any; + } + + public void setHosts(Map hosts) { + this.hosts = hosts; + } + + public Map getHosts() { + return hosts; + } + + public void setRacks(Map racks) { + this.racks = racks; + } + + public Map getRacks() { + return racks; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterResource.java new file mode 100644 index 000000000..ededc771e --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterResource.java @@ -0,0 +1,65 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +import org.springframework.yarn.am.cluster.ClusterState; +import org.springframework.yarn.am.cluster.ContainerCluster; + +/** + * Domain mapping for {@link ContainerCluster}. + * + * @author Janne Valkealahti + * + */ +public class ContainerClusterResource { + + private String id; + + private GridProjectionResource gridProjection; + + private ContainerClusterStateResource containerClusterState; + + public ContainerClusterResource() { + } + + public ContainerClusterResource(ContainerCluster cluster) { + this.id = cluster.getId(); + this.gridProjection = new GridProjectionResource(cluster.getGridProjection()); + ClusterState clusterState = cluster.getStateMachine().getState().getId(); + containerClusterState = new ContainerClusterStateResource(clusterState); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public GridProjectionResource getGridProjection() { + return gridProjection; + } + + public ContainerClusterStateResource getContainerClusterState() { + return containerClusterState; + } + + public void setContainerClusterState(ContainerClusterStateResource containerClusterState) { + this.containerClusterState = containerClusterState; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterStateResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterStateResource.java new file mode 100644 index 000000000..9b3617847 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/ContainerClusterStateResource.java @@ -0,0 +1,39 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +import org.springframework.yarn.am.cluster.ClusterState; + +public class ContainerClusterStateResource { + + private ClusterState clusterState; + + public ContainerClusterStateResource() { + } + + public ContainerClusterStateResource(ClusterState clusterState) { + this.clusterState = clusterState; + } + + public ClusterState getClusterState() { + return clusterState; + } + + public void setClusterState(ClusterState clusterState) { + this.clusterState = clusterState; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridMemberResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridMemberResource.java new file mode 100644 index 000000000..d3db600d5 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridMemberResource.java @@ -0,0 +1,37 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +public class GridMemberResource { + + private String id; + + public GridMemberResource() { + } + + public GridMemberResource(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridProjectionResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridProjectionResource.java new file mode 100644 index 000000000..f3faf9ca8 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/GridProjectionResource.java @@ -0,0 +1,65 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +import java.util.ArrayList; +import java.util.Collection; + +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.support.ProjectionData; + +public class GridProjectionResource { + + private Collection members; + + private ProjectionData projectionData; + + private SatisfyStateDataResource satisfyState; + + public GridProjectionResource() { + } + + public GridProjectionResource(GridProjection projection) { + this.members = new ArrayList(); + for (GridMember member : projection.getMembers()) { + this.members.add(new GridMemberResource(member.getId().toString())); + } + projectionData = projection.getProjectionData(); + satisfyState = new SatisfyStateDataResource(projection.getSatisfyState()); + } + + public Collection getMembers() { + return members; + } + + public void setMembers(Collection members) { + this.members = members; + } + + public ProjectionData getProjectionData() { + return projectionData; + } + + public SatisfyStateDataResource getSatisfyState() { + return satisfyState; + } + + public void setSatisfyState(SatisfyStateDataResource satisfyState) { + this.satisfyState = satisfyState; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/SatisfyStateDataResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/SatisfyStateDataResource.java new file mode 100644 index 000000000..301c06cc1 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/SatisfyStateDataResource.java @@ -0,0 +1,47 @@ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +import java.util.ArrayList; +import java.util.Collection; + +import org.springframework.yarn.am.allocate.ContainerAllocateData; +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.support.SatisfyStateData; + +public class SatisfyStateDataResource { + + private Collection removeData; + + private ContainerAllocateDataResource allocateData; + + public SatisfyStateDataResource() { + } + + public SatisfyStateDataResource(SatisfyStateData satisfyState) { + allocateData = new ContainerAllocateDataResource(); + ContainerAllocateData data = satisfyState.getAllocateData(); + allocateData.setAny(data.getAny()); + allocateData.setHosts(data.getHosts()); + allocateData.setRacks(data.getRacks()); + removeData = new ArrayList(); + for (GridMember member : satisfyState.getRemoveData()) { + removeData.add(new GridMemberResource(member.getId().toString())); + } + } + + public void setAllocateData(ContainerAllocateDataResource allocateData) { + this.allocateData = allocateData; + } + + public ContainerAllocateDataResource getAllocateData() { + return allocateData; + } + + public void setRemoveData(Collection removeData) { + this.removeData = removeData; + } + + public Collection getRemoveData() { + return removeData; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/YarnContainerClusterEndpointResource.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/YarnContainerClusterEndpointResource.java new file mode 100644 index 000000000..efb1a14a9 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/actuate/endpoint/mvc/domain/YarnContainerClusterEndpointResource.java @@ -0,0 +1,50 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc.domain; + +import java.util.ArrayList; +import java.util.Collection; + +import org.springframework.yarn.boot.actuate.endpoint.mvc.YarnContainerClusterMvcEndpoint; + +/** + * Response domain mapping for {@link YarnContainerClusterMvcEndpoint#invoke()}. + * + * @author Janne Valkealahti + * + */ +public class YarnContainerClusterEndpointResource { + + private Collection clusterIds; + + public YarnContainerClusterEndpointResource() { + this(new ArrayList()); + } + + public YarnContainerClusterEndpointResource(Collection clusters) { + super(); + this.clusterIds = clusters; + } + + public Collection getClusters() { + return clusterIds; + } + + public void setClusters(Collection clusters) { + this.clusterIds = clusters; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterApplication.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterApplication.java new file mode 100644 index 000000000..899a60f61 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterApplication.java @@ -0,0 +1,355 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.app; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.springframework.boot.actuate.autoconfigure.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.EndpointMBeanExportAutoConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; +import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration; +import org.springframework.boot.autoconfigure.web.EmbeddedServletContainerAutoConfiguration; +import org.springframework.boot.autoconfigure.web.WebMvcAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.yarn.am.cluster.ContainerCluster; +import org.springframework.yarn.boot.SpringApplicationCallback; +import org.springframework.yarn.boot.SpringApplicationTemplate; +import org.springframework.yarn.boot.actuate.endpoint.YarnContainerClusterEndpoint; +import org.springframework.yarn.boot.actuate.endpoint.mvc.AbstractContainerClusterRequest.ProjectionDataType; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterCreateRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterCreateRequest.ProjectionType; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterModifyRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.YarnContainerClusterMvcEndpoint; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.ContainerClusterResource; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.YarnContainerClusterEndpointResource; +import org.springframework.yarn.boot.support.SpringYarnBootUtils; +import org.springframework.yarn.client.YarnClient; +import org.springframework.yarn.support.console.ContainerClusterReport; +import org.springframework.yarn.support.console.ContainerClusterReport.ClusterInfoField; +import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoField; +import org.springframework.yarn.support.console.ContainerClusterReport.ClustersInfoReportData; + +/** + * A Boot application which is used to control Spring YARN {@link ContainerCluster}s + * via rest API offered by a {@link YarnContainerClusterMvcEndpoint}. + * + * @author Janne Valkealahti + * + */ +@Configuration +@EnableAutoConfiguration(exclude = { EmbeddedServletContainerAutoConfiguration.class, WebMvcAutoConfiguration.class, + JmxAutoConfiguration.class, BatchAutoConfiguration.class, JmxAutoConfiguration.class, + EndpointMBeanExportAutoConfiguration.class, EndpointAutoConfiguration.class }) +public class YarnContainerClusterApplication extends AbstractClientApplication { + + @Override + protected YarnContainerClusterApplication getThis() { + return this; + } + + public String run() { + return run(new String[0]); + } + + public String run(String... args) { + SpringApplicationBuilder builder = new SpringApplicationBuilder(); + builder.web(false); + builder.sources(YarnContainerClusterApplication.class, OperationProperties.class); + SpringYarnBootUtils.addSources(builder, sources.toArray(new Object[0])); + SpringYarnBootUtils.addProfiles(builder, profiles.toArray(new String[0])); + SpringYarnBootUtils.addApplicationListener(builder, appProperties); + + SpringApplicationTemplate template = new SpringApplicationTemplate(builder); + return template.execute(new SpringApplicationCallback() { + + @Override + public String runWithSpringApplication(ApplicationContext context) throws Exception { + OperationProperties operationProperties = context.getBean(OperationProperties.class); + YarnClient client = context.getBean(YarnClient.class); + ApplicationId applicationId = ConverterUtils.toApplicationId(operationProperties.getApplicationId()); + String clusterId = operationProperties.getClusterId(); + String clusterDef = operationProperties.getClusterDef(); + String projectionType = operationProperties.getProjectionType(); + Integer projectionDataAny = operationProperties.getProjectionDataAny(); + Map projectionDataHosts = operationProperties.getProjectionDataHosts(); + Map projectionDataRacks = operationProperties.getProjectionDataRacks(); + Map extraProperties = operationProperties.getExtraProperties(); + Operation operation = operationProperties.getOperation(); + boolean verbose = operationProperties.isVerbose(); + + if (Operation.CLUSTERSINFO == operation) { + return doClustersInfo(client, applicationId); + } else if (Operation.CLUSTERINFO == operation) { + return doClusterInfo(client, applicationId, clusterId, verbose); + } else if (Operation.CLUSTERCREATE == operation) { + return doClusterCreate(client, applicationId, clusterId, clusterDef, projectionType, + projectionDataAny, projectionDataHosts, projectionDataRacks, extraProperties); + } else if (Operation.CLUSTERDESTROY == operation) { + return doClusterDestroy(client, applicationId, clusterId); + } else if (Operation.CLUSTERMODIFY == operation) { + return doClusterModify(client, applicationId, clusterId, projectionDataAny, projectionDataHosts, + projectionDataRacks); + } else if (Operation.CLUSTERSTART == operation) { + return doClusterStart(client, applicationId, clusterId); + } else if (Operation.CLUSTERSTOP == operation) { + return doClusterStop(client, applicationId, clusterId); + } + return null; + } + + }, args); + + } + + private String doClustersInfo(YarnClient client, ApplicationId applicationId) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + YarnContainerClusterEndpointResource response = operations.getClusters(); + return ContainerClusterReport.clustersInfoReportBuilder() + .add(ClustersInfoField.ID) + .from(new ArrayList(response.getClusters())) + .build().toString(); + } + + private String doClusterInfo(YarnClient client, ApplicationId applicationId, String clusterId, boolean verbose) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + ContainerClusterResource response = operations.clusterInfo(clusterId); + + List data = new ArrayList(); + + Integer pany = response.getGridProjection().getProjectionData().getAny(); + Map phosts = response.getGridProjection().getProjectionData().getHosts(); + Map pracks = response.getGridProjection().getProjectionData().getRacks(); + Integer sany = response.getGridProjection().getSatisfyState().getAllocateData().getAny(); + Map shosts = response.getGridProjection().getSatisfyState().getAllocateData().getHosts(); + Map sracks = response.getGridProjection().getSatisfyState().getAllocateData().getRacks(); + + data.add(new ClustersInfoReportData(response.getContainerClusterState().getClusterState().toString(), response.getGridProjection().getMembers().size(), pany, phosts, pracks, sany, shosts, sracks)); + if (verbose) { + return ContainerClusterReport.clusterInfoReportBuilder() + .add(ClusterInfoField.STATE) + .add(ClusterInfoField.MEMBERS) + .add(ClusterInfoField.PROJECTIONANY) + .add(ClusterInfoField.PROJECTIONHOSTS) + .add(ClusterInfoField.PROJECTIONRACKS) + .add(ClusterInfoField.SATISFYANY) + .add(ClusterInfoField.SATISFYHOSTS) + .add(ClusterInfoField.SATISFYRACKS) + .from(data) + .build().toString(); + } else { + return ContainerClusterReport.clusterInfoReportBuilder() + .add(ClusterInfoField.STATE) + .add(ClusterInfoField.MEMBERS) + .from(data) + .build().toString(); + } + } + + private String doClusterCreate(YarnClient client, ApplicationId applicationId, String clusterId, String clusterDef, + String projectionType, Integer projectionDataAny, Map hosts, Map racks, + Map extraProperties) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + + ContainerClusterCreateRequest request = new ContainerClusterCreateRequest(); + request.setClusterId(clusterId); + request.setClusterDef(clusterDef); + request.setProjection(ProjectionType.valueOf(projectionType.toUpperCase())); + request.setExtraProperties(extraProperties); + + ProjectionDataType projectionData = new ProjectionDataType(); + projectionData.setAny(projectionDataAny); + projectionData.setHosts(hosts); + projectionData.setRacks(racks); + + request.setProjectionData(projectionData); + operations.clusterCreate(request); + return "Cluster " + clusterId + " created."; + } + + private String doClusterDestroy(YarnClient client, ApplicationId applicationId, String clusterId) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + operations.clusterDestroy(clusterId); + return "Cluster " + clusterId + " destroyed."; + } + + private String doClusterStart(YarnClient client, ApplicationId applicationId, String clusterId) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + ContainerClusterModifyRequest request = new ContainerClusterModifyRequest(); + request.setAction("start"); + operations.clusterStart(clusterId, request); + return "Cluster " + clusterId + " started."; + } + + private String doClusterStop(YarnClient client, ApplicationId applicationId, String clusterId) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + ContainerClusterModifyRequest request = new ContainerClusterModifyRequest(); + request.setAction("stop"); + operations.clusterStop(clusterId, request); + return "Cluster " + clusterId + " stopped."; + } + + private String doClusterModify(YarnClient client, ApplicationId applicationId, String clusterId, + Integer projectionDataAny, Map hosts, Map racks) { + YarnContainerClusterOperations operations = buildClusterOperations(client, applicationId); + + ContainerClusterCreateRequest request = new ContainerClusterCreateRequest(); + request.setClusterId(clusterId); + + ProjectionDataType projectionData = new ProjectionDataType(); + projectionData.setAny(projectionDataAny); + projectionData.setHosts(hosts); + projectionData.setRacks(racks); + + request.setProjectionData(projectionData); + + operations.clusterModify(clusterId, request); + return "Cluster " + clusterId + " modified."; + } + + private YarnContainerClusterOperations buildClusterOperations(YarnClient client, ApplicationId applicationId) { + ApplicationReport report = client.getApplicationReport(applicationId); + String trackingUrl = report.getOriginalTrackingUrl(); + return new YarnContainerClusterTemplate(trackingUrl + "/" + YarnContainerClusterEndpoint.ENDPOINT_ID); + } + + @ConfigurationProperties(value = "spring.yarn.internal.ContainerClusterApplication") + public static class OperationProperties { + + private Operation operation; + + private String applicationId; + + private String clusterId; + + private String clusterDef; + + private String projectionType; + + private Integer projectionDataAny; + + private Map projectionDataHosts; + + private Map projectionDataRacks; + + private Map extraProperties; + + private boolean verbose; + + public void setOperation(Operation operation) { + this.operation = operation; + } + + public Operation getOperation() { + return operation; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public String getApplicationId() { + return applicationId; + } + + public void setClusterId(String clusterId) { + this.clusterId = clusterId; + } + + public String getClusterId() { + return clusterId; + } + + public void setClusterDef(String clusterDef) { + this.clusterDef = clusterDef; + } + + public String getClusterDef() { + return clusterDef; + } + + public void setProjectionType(String projectionType) { + this.projectionType = projectionType; + } + + public String getProjectionType() { + return projectionType; + } + + public void setProjectionDataAny(Integer projectionDataAny) { + this.projectionDataAny = projectionDataAny; + } + + public Integer getProjectionDataAny() { + return projectionDataAny; + } + + public void setProjectionDataHosts(Map projectionDataHosts) { + this.projectionDataHosts = projectionDataHosts; + } + + public Map getProjectionDataHosts() { + return projectionDataHosts; + } + + public void setProjectionDataRacks(Map projectionDataRacks) { + this.projectionDataRacks = projectionDataRacks; + } + + public Map getProjectionDataRacks() { + return projectionDataRacks; + } + + public void setExtraProperties(Map extraProperties) { + this.extraProperties = extraProperties; + } + + public Map getExtraProperties() { + return extraProperties; + } + + public void setVerbose(boolean verbose) { + this.verbose = verbose; + } + + public boolean isVerbose() { + return verbose; + } + + } + + /** + * Operations supported by this application. + */ + private enum Operation { + CLUSTERSINFO, + CLUSTERINFO, + CLUSTERCREATE, + CLUSTERDESTROY, + CLUSTERMODIFY, + CLUSTERSTART, + CLUSTERSTOP + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterClientException.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterClientException.java new file mode 100644 index 000000000..635ec50e8 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterClientException.java @@ -0,0 +1,52 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.app; + +import org.springframework.core.NestedRuntimeException; + +/** + * Base class for exceptions thrown by {@link YarnContainerClusterApplication} + * whenever it encounters client-side errors. + * + * @author Janne Valkealahti + * + */ +public class YarnContainerClusterClientException extends NestedRuntimeException { + + private static final long serialVersionUID = 849031924295533758L; + + /** + * Construct a new instance of {@code YarnContainerClusterClientException} + * with the given message. + * + * @param msg the message + */ + public YarnContainerClusterClientException(String msg) { + super(msg); + } + + /** + * Construct a new instance of {@code YarnContainerClusterClientException} + * with the given message and exception. + * + * @param msg the message + * @param cause the exception + */ + public YarnContainerClusterClientException(String msg, Throwable cause) { + super(msg, cause); + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterOperations.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterOperations.java new file mode 100644 index 000000000..6a0bb041d --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterOperations.java @@ -0,0 +1,99 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.app; + +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterCreateRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterModifyRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.ContainerClusterResource; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.YarnContainerClusterEndpointResource; + +/** + * An operations interface for {@link YarnContainerClusterTemplate}. + * + * @author Janne Valkealahti + * + */ +public interface YarnContainerClusterOperations { + + /** + * Get a clusters info. + * + * @return the {@link YarnContainerClusterEndpointResource} + * @throws YarnContainerClusterClientException if client experienced an error + */ + YarnContainerClusterEndpointResource getClusters() throws YarnContainerClusterClientException; + + /** + * Create a container cluster. + * + * @param request the create request + * @return the {@link ContainerClusterResource} + * @throws YarnContainerClusterClientException if client experienced an error + */ + ContainerClusterResource clusterCreate(ContainerClusterCreateRequest request) + throws YarnContainerClusterClientException; + + /** + * Get a container cluster info. + * + * @param clusterId the cluster identifier + * @return the {@link ContainerClusterResource} + * @throws YarnContainerClusterClientException if client experienced an error + */ + ContainerClusterResource clusterInfo(String clusterId) throws YarnContainerClusterClientException; + + /** + * Start a container cluster. + * + * @param clusterId the cluster identifier + * @param request the modify request. + * @return the {@link ContainerClusterResource} + * @throws YarnContainerClusterClientException if client experienced an error + */ + ContainerClusterResource clusterStart(String clusterId, ContainerClusterModifyRequest request) + throws YarnContainerClusterClientException; + + /** + * Stop a container cluster. + * + * @param clusterId the cluster identifier + * @param request the modify request. + * @return the {@link ContainerClusterResource} + * @throws YarnContainerClusterClientException if client experienced an error + */ + ContainerClusterResource clusterStop(String clusterId, ContainerClusterModifyRequest request) + throws YarnContainerClusterClientException; + + /** + * Modify a container cluster. + * + * @param clusterId the cluster identifier + * @param request the create request. + * @return the {@link ContainerClusterResource} + * @throws YarnContainerClusterClientException if client experienced an error + */ + ContainerClusterResource clusterModify(String clusterId, ContainerClusterCreateRequest request) + throws YarnContainerClusterClientException; + + /** + * Destroy a container cluster. + * + * @param clusterId the cluster identifier + * @throws YarnContainerClusterClientException if client experienced an error + */ + void clusterDestroy(String clusterId) throws YarnContainerClusterClientException; + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplate.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplate.java new file mode 100644 index 000000000..82b3ecb07 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplate.java @@ -0,0 +1,140 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.app; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterCreateRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterModifyRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.ContainerClusterResource; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.YarnContainerClusterEndpointResource; + +/** + * Template accessing boot mvc endpoint controlling container clusters. + * + * @author Janne Valkealahti + * + */ +public class YarnContainerClusterTemplate implements YarnContainerClusterOperations { + + protected final RestTemplate restTemplate; + + private final String baseUri; + + /** + * Constructs a {@link YarnContainerClusterTemplate} using a {@link RestTemplate} instantiated + * with {@link HttpComponentsClientHttpRequestFactory}. + * + * @param baseUri the base uri + */ + public YarnContainerClusterTemplate(String baseUri) { + this.restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory()); + this.baseUri = baseUri; + } + + /** + * Constructs a {@link YarnContainerClusterTemplate} using a given {@link RestTemplate}. + * + * @param baseUri the base uri + * @param restTemplate the rest template + */ + public YarnContainerClusterTemplate(String baseUri, RestTemplate restTemplate) { + this.restTemplate = restTemplate; + this.baseUri = baseUri; + } + + @Override + public YarnContainerClusterEndpointResource getClusters() throws YarnContainerClusterClientException { + return restTemplate.getForObject(baseUri, YarnContainerClusterEndpointResource.class); + } + + @Override + public ContainerClusterResource clusterCreate(ContainerClusterCreateRequest request) throws YarnContainerClusterClientException { + try { + return restTemplate.postForObject(baseUri, request, ContainerClusterResource.class); + } catch (RestClientException e) { + throw convertException(e); + } + } + + @Override + public ContainerClusterResource clusterInfo(String clusterId) throws YarnContainerClusterClientException { + try { + return restTemplate.getForObject(baseUri + "/" + clusterId, ContainerClusterResource.class); + } catch (RestClientException e) { + throw convertException(e); + } + } + + @Override + public ContainerClusterResource clusterStart(String clusterId, ContainerClusterModifyRequest request) throws YarnContainerClusterClientException { + try { + HttpEntity requestEntity = new HttpEntity(request); + ResponseEntity exchange = restTemplate.exchange(baseUri + "/" + clusterId, + HttpMethod.PUT, requestEntity, ContainerClusterResource.class); + return exchange.getBody(); + } catch (RestClientException e) { + throw convertException(e); + } + } + + @Override + public ContainerClusterResource clusterStop(String clusterId, ContainerClusterModifyRequest request) throws YarnContainerClusterClientException { + try { + HttpEntity requestEntity = new HttpEntity(request); + ResponseEntity exchange = restTemplate.exchange(baseUri + "/" + clusterId, + HttpMethod.PUT, requestEntity, ContainerClusterResource.class); + return exchange.getBody(); + } catch (RestClientException e) { + throw convertException(e); + } + } + + @Override + public ContainerClusterResource clusterModify(String clusterId, ContainerClusterCreateRequest request) throws YarnContainerClusterClientException { + try { + HttpEntity requestEntity = new HttpEntity(request); + ResponseEntity exchange = restTemplate.exchange(baseUri + "/" + clusterId, + HttpMethod.PATCH, requestEntity, ContainerClusterResource.class); + return exchange.getBody(); + } catch (RestClientException e) { + throw convertException(e); + } + } + + @Override + public void clusterDestroy(String clusterId) throws YarnContainerClusterClientException { + try { + Map uriVariables = new HashMap(); + uriVariables.put("clusterId", clusterId); + restTemplate.delete(baseUri + "/{clusterId}", uriVariables); + } catch (RestClientException e) { + throw convertException(e); + } + } + + private YarnContainerClusterClientException convertException(Exception e) { + return new YarnContainerClusterClientException("Error communicating with rest endpoint",e); + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnInfoApplication.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnInfoApplication.java index 109c7b568..b46091826 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnInfoApplication.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnInfoApplication.java @@ -23,6 +23,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.EndpointMBeanExportAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration; @@ -51,7 +53,8 @@ */ @Configuration @EnableAutoConfiguration(exclude = { EmbeddedServletContainerAutoConfiguration.class, WebMvcAutoConfiguration.class, - JmxAutoConfiguration.class, BatchAutoConfiguration.class }) + JmxAutoConfiguration.class, BatchAutoConfiguration.class, JmxAutoConfiguration.class, + EndpointMBeanExportAutoConfiguration.class, EndpointAutoConfiguration.class }) public class YarnInfoApplication extends AbstractClientApplication { /** diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnKillApplication.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnKillApplication.java index 9bfc7ccbf..87aea6a6e 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnKillApplication.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnKillApplication.java @@ -19,6 +19,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.springframework.boot.actuate.autoconfigure.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.EndpointMBeanExportAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration; @@ -42,7 +44,8 @@ */ @Configuration @EnableAutoConfiguration(exclude = { EmbeddedServletContainerAutoConfiguration.class, WebMvcAutoConfiguration.class, - JmxAutoConfiguration.class, BatchAutoConfiguration.class }) + JmxAutoConfiguration.class, BatchAutoConfiguration.class, JmxAutoConfiguration.class, + EndpointMBeanExportAutoConfiguration.class, EndpointAutoConfiguration.class }) public class YarnKillApplication extends AbstractClientApplication { public String run() { diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnPushApplication.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnPushApplication.java index ed016691a..8dd20a7c0 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnPushApplication.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnPushApplication.java @@ -20,6 +20,8 @@ import java.util.Properties; import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.EndpointMBeanExportAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration; @@ -50,7 +52,8 @@ */ @Configuration @EnableAutoConfiguration(exclude = { EmbeddedServletContainerAutoConfiguration.class, WebMvcAutoConfiguration.class, - JmxAutoConfiguration.class, BatchAutoConfiguration.class }) + JmxAutoConfiguration.class, BatchAutoConfiguration.class, JmxAutoConfiguration.class, + EndpointMBeanExportAutoConfiguration.class, EndpointAutoConfiguration.class }) public class YarnPushApplication extends AbstractClientApplication { private Map configFilesContents = new HashMap(); diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnSubmitApplication.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnSubmitApplication.java index 31c45e5eb..2ee37485f 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnSubmitApplication.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/app/YarnSubmitApplication.java @@ -17,6 +17,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.EndpointAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.EndpointMBeanExportAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration; import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration; @@ -43,7 +45,8 @@ */ @Configuration @EnableAutoConfiguration(exclude = { EmbeddedServletContainerAutoConfiguration.class, WebMvcAutoConfiguration.class, - JmxAutoConfiguration.class, BatchAutoConfiguration.class }) + JmxAutoConfiguration.class, BatchAutoConfiguration.class, JmxAutoConfiguration.class, + EndpointMBeanExportAutoConfiguration.class, EndpointAutoConfiguration.class }) public class YarnSubmitApplication extends AbstractClientApplication { /** diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/condition/OnYarnAppmasterCondition.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/condition/OnYarnAppmasterCondition.java index 3617f0f4c..d8f9e8f44 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/condition/OnYarnAppmasterCondition.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/condition/OnYarnAppmasterCondition.java @@ -20,6 +20,7 @@ import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.util.StringUtils; /** * {@link Condition} that checks for the presence of Hadoop Yarn appmaster. @@ -34,6 +35,10 @@ class OnYarnAppmasterCondition extends SpringBootCondition { @Override public ConditionOutcome getMatchOutcome(ConditionContext context, AnnotatedTypeMetadata metadata) { String tokenEnv = System.getenv(TOKEN_ENV); + // TODO: check that this is ok + if (!StringUtils.hasText(tokenEnv)) { + tokenEnv = System.getProperty(TOKEN_ENV); + } if (tokenEnv != null) { String[] split = tokenEnv.split("/"); diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterProperties.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterProperties.java index 1069a28d5..d8e7c3ac2 100644 --- a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterProperties.java +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterProperties.java @@ -1,5 +1,7 @@ package org.springframework.yarn.boot.properties; +import java.util.Map; + import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -15,6 +17,8 @@ public class SpringYarnAppmasterProperties { private boolean keepContextAlive = true; private int containerCount = 1; private String appmasterClass; + private ContainerClusterProperties containercluster; +// private Map containercluster; public String getAppmasterClass() { return appmasterClass; @@ -40,4 +44,118 @@ public void setContainerCount(int containerCount) { this.containerCount = containerCount; } + public ContainerClusterProperties getContainercluster() { + return containercluster; + } + + public void setContainercluster(ContainerClusterProperties containercluster) { + this.containercluster = containercluster; + } + +// public Map getContainercluster() { +// return containercluster; +// } +// +// public void setContainercluster(Map containercluster) { +// this.containercluster = containercluster; +// } + + public static class ContainerClusterProperties { + + private boolean enabled; + + private Map clusters; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public Map getClusters() { + return clusters; + } + + public void setClusters(Map clusters) { + this.clusters = clusters; + } + + } + + public static class ContainerClustersProperties { + + private SpringYarnAppmasterResourceProperties resource; + + private SpringYarnAppmasterLaunchContextProperties launchcontext; + + private SpringYarnAppmasterLocalizerProperties localizer; + + private String projectionType; + + private Integer projectionAny; + + private Map projectionHosts; + + private Map projectionRacks; + + public SpringYarnAppmasterResourceProperties getResource() { + return resource; + } + + public void setResource(SpringYarnAppmasterResourceProperties resource) { + this.resource = resource; + } + + public SpringYarnAppmasterLaunchContextProperties getLaunchcontext() { + return launchcontext; + } + + public void setLaunchcontext(SpringYarnAppmasterLaunchContextProperties launchcontext) { + this.launchcontext = launchcontext; + } + + public SpringYarnAppmasterLocalizerProperties getLocalizer() { + return localizer; + } + + public void setLocalizer(SpringYarnAppmasterLocalizerProperties localizer) { + this.localizer = localizer; + } + + public String getProjectionType() { + return projectionType; + } + + public void setProjectionType(String projectionType) { + this.projectionType = projectionType; + } + + public Integer getProjectionAny() { + return projectionAny; + } + + public void setProjectionAny(Integer projectionAny) { + this.projectionAny = projectionAny; + } + + public Map getProjectionHosts() { + return projectionHosts; + } + + public void setProjectionHosts(Map projectionHosts) { + this.projectionHosts = projectionHosts; + } + + public Map getProjectionRacks() { + return projectionRacks; + } + + public void setProjectionRacks(Map projectionRacks) { + this.projectionRacks = projectionRacks; + } + + } + } diff --git a/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/support/BootMultiLocalResourcesSelector.java b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/support/BootMultiLocalResourcesSelector.java new file mode 100644 index 000000000..6ebf33e27 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/support/BootMultiLocalResourcesSelector.java @@ -0,0 +1,61 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.support; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.yarn.fs.LocalResourcesSelector; +import org.springframework.yarn.fs.MultiLocalResourcesSelector; + +/** + * Default implementation of a {@link MultiLocalResourcesSelector}. + * + * @author Janne Valkealahti + * + */ +public class BootMultiLocalResourcesSelector implements MultiLocalResourcesSelector { + + private final HashMap selectors = new HashMap(); + + /** + * Instantiates a new boot multi local resources selector. + * + * @param selector the default selector + * @param selectors the custom selectors + */ + public BootMultiLocalResourcesSelector(LocalResourcesSelector selector, Map selectors) { + this.selectors.put(null, selector); + this.selectors.putAll(selectors); + } + + @Override + public List select(String dir) { + return selectors.get(null).select(dir); + } + + @Override + public List select(String id, String dir) { + LocalResourcesSelector selector = selectors.get(id); + if (selector != null) { + return selector.select(dir); + } else { + return selectors.get(null).select(dir); + } + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/main/resources/META-INF/spring.factories b/spring-yarn/spring-yarn-boot/src/main/resources/META-INF/spring.factories index 55d6ec064..ff4f99e44 100644 --- a/spring-yarn/spring-yarn-boot/src/main/resources/META-INF/spring.factories +++ b/spring-yarn/spring-yarn-boot/src/main/resources/META-INF/spring.factories @@ -2,4 +2,5 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ org.springframework.yarn.boot.YarnClientAutoConfiguration,\ org.springframework.yarn.boot.YarnContainerAutoConfiguration,\ -org.springframework.yarn.boot.YarnAppmasterAutoConfiguration \ No newline at end of file +org.springframework.yarn.boot.YarnAppmasterAutoConfiguration,\ +org.springframework.yarn.boot.ContainerClusterAppmasterAutoConfiguration diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/MockUtils.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/MockUtils.java new file mode 100644 index 000000000..c447ea574 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/MockUtils.java @@ -0,0 +1,90 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; + +public abstract class MockUtils { + + public static ContainerStatus getMockContainerStatus(ContainerId containerId, ContainerState containerState, int exitStatus) { + ContainerStatus status = mock(ContainerStatus.class); + when(status.getContainerId()).thenReturn(containerId); + when(status.getState()).thenReturn(containerState); + when(status.getExitStatus()).thenReturn(exitStatus); + return status; + } + + public static ApplicationId getMockApplicationId(int appId) { + ApplicationId applicationId = mock(ApplicationId.class); + when(applicationId.getClusterTimestamp()).thenReturn(0L); + when(applicationId.getId()).thenReturn(appId); + return applicationId; + } + + public static Container getMockContainer(ContainerId containerId, NodeId nodeId, Resource resource, + Priority priority) { + Container container = mock(Container.class); + when(container.getId()).thenReturn(containerId); + when(container.getNodeId()).thenReturn(nodeId); + when(container.getResource()).thenReturn(resource); + when(container.getPriority()).thenReturn(priority); + return container; + } + + public static ContainerId getMockContainerId(ApplicationAttemptId applicationAttemptId, int id) { + ContainerId containerId = mock(ContainerId.class); + doReturn(applicationAttemptId).when(containerId).getApplicationAttemptId(); + doReturn(id).when(containerId).getId(); + doReturn(Integer.toString(id)).when(containerId).toString(); + return containerId; + } + + public static NodeId getMockNodeId(String host, int port) { + NodeId nodeId = mock(NodeId.class); + doReturn(host).when(nodeId).getHost(); + doReturn(port).when(nodeId).getPort(); + return nodeId; + } + + public static Priority getMockPriority(int priority) { + Priority pri = mock(Priority.class); + when(pri.getPriority()).thenReturn(priority); + return pri; + } + + public static ApplicationAttemptId getMockApplicationAttemptId(int appId, int attemptId) { + ApplicationId applicationId = mock(ApplicationId.class); + when(applicationId.getClusterTimestamp()).thenReturn(0L); + when(applicationId.getId()).thenReturn(appId); + ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class); + when(applicationAttemptId.getApplicationId()).thenReturn(applicationId); + when(applicationAttemptId.getAttemptId()).thenReturn(attemptId); + return applicationAttemptId; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/TestUtils.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/TestUtils.java new file mode 100644 index 000000000..b9a3c5b70 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/TestUtils.java @@ -0,0 +1,94 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; + +import org.springframework.util.ReflectionUtils; + +/** + * Testing utilities. + * + * @author Janne Valkealahti + * + */ +public abstract class TestUtils { + + @SuppressWarnings("unchecked") + public static T readField(String name, Object target) throws Exception { + Field field = null; + Class clazz = target.getClass(); + do { + try { + field = clazz.getDeclaredField(name); + } catch (Exception ex) { + } + + clazz = clazz.getSuperclass(); + } while (field == null && !clazz.equals(Object.class)); + + if (field == null) + throw new IllegalArgumentException("Cannot find field '" + name + "' in the class hierarchy of " + + target.getClass()); + field.setAccessible(true); + return (T) field.get(target); + } + + @SuppressWarnings("unchecked") + public static T callMethod(String name, Object target) throws Exception { + Class clazz = target.getClass(); + Method method = ReflectionUtils.findMethod(clazz, name); + + if (method == null) + throw new IllegalArgumentException("Cannot find method '" + method + "' in the class hierarchy of " + + target.getClass()); + method.setAccessible(true); + return (T) ReflectionUtils.invokeMethod(method, target); + } + + @SuppressWarnings("unchecked") + public static T callMethod(String name, Object target, Object[] args, Class[] argsTypes) throws Exception { + Class clazz = target.getClass(); + Method method = ReflectionUtils.findMethod(clazz, name, argsTypes); + + if (method == null) + throw new IllegalArgumentException("Cannot find method '" + method + "' in the class hierarchy of " + + target.getClass()); + method.setAccessible(true); + return (T) ReflectionUtils.invokeMethod(method, target, args); + } + + public static void setField(String name, Object target, Object value) throws Exception { + Field field = null; + Class clazz = target.getClass(); + do { + try { + field = clazz.getDeclaredField(name); + } catch (Exception ex) { + } + + clazz = clazz.getSuperclass(); + } while (field == null && !clazz.equals(Object.class)); + + if (field == null) + throw new IllegalArgumentException("Cannot find field '" + name + "' in the class hierarchy of " + + target.getClass()); + field.setAccessible(true); + field.set(target, value); + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/YarnAppmasterAutoConfigurationTests.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/YarnAppmasterAutoConfigurationTests.java new file mode 100644 index 000000000..56300c88c --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/YarnAppmasterAutoConfigurationTests.java @@ -0,0 +1,80 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot; + +import static org.junit.Assert.assertNotNull; + +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.springframework.boot.autoconfigure.PropertyPlaceholderAutoConfiguration; +import org.springframework.boot.test.EnvironmentTestUtils; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.cluster.ManagedContainerClusterAppmaster; +import org.springframework.yarn.am.cluster.ContainerClusterStateMachineConfiguration; + +/** + * Tests for {@link YarnAppmasterAutoConfiguration}. + * + * @author Janne Valkealahti + * + */ +public class YarnAppmasterAutoConfigurationTests { + + private AnnotationConfigApplicationContext context; + + @Rule + public ExpectedException expected = ExpectedException.none(); + + @After + public void close() { + if (context != null) { + context.close(); + } + System.clearProperty("HADOOP_TOKEN_FILE_LOCATION"); + } + + @Test + public void testDefaultContext() throws Exception { + System.setProperty("HADOOP_TOKEN_FILE_LOCATION", "xx/xx/xx/00001/container_tokens"); + context = new AnnotationConfigApplicationContext(); + EnvironmentTestUtils + .addEnvironment( + this.context, + "spring.yarn.appmaster.appmasterClass:org.springframework.yarn.boot.YarnAppmasterAutoConfigurationTests$TestManagedContainerClusterAppmaster", + "spring.yarn.appmaster.containercluster.enabled:true"); + context.register(ContainerClusterStateMachineConfiguration.class, TestConfigWithClass.class, + YarnAppmasterAutoConfiguration.class, PropertyPlaceholderAutoConfiguration.class); + context.refresh(); + assertNotNull(context.getBean(YarnAppmaster.class)); + } + + @Configuration + public static class TestConfigWithClass { + + } + + public static class TestManagedContainerClusterAppmaster extends ManagedContainerClusterAppmaster { + + @Override + protected void doStop() { + } + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpointTests.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpointTests.java new file mode 100644 index 000000000..65b821df7 --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/actuate/endpoint/mvc/YarnContainerClusterMvcEndpointTests.java @@ -0,0 +1,411 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.actuate.endpoint.mvc; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.autoconfigure.EndpointWebMvcAutoConfiguration; +import org.springframework.boot.actuate.autoconfigure.ManagementServerPropertiesAutoConfiguration; +import org.springframework.boot.autoconfigure.hateoas.HypermediaAutoConfiguration; +import org.springframework.boot.test.SpringApplicationConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.http.MediaType; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.annotation.DirtiesContext.ClassMode; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.web.WebAppConfiguration; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.allocate.ContainerAllocateData; +import org.springframework.yarn.am.allocate.ContainerAllocator; +import org.springframework.yarn.am.cluster.ClusterState; +import org.springframework.yarn.am.cluster.ContainerCluster; +import org.springframework.yarn.am.cluster.ContainerClusterStateMachineConfiguration; +import org.springframework.yarn.am.cluster.ManagedContainerClusterAppmaster; +import org.springframework.yarn.am.container.ContainerLauncher; +import org.springframework.yarn.am.grid.GridProjectionFactory; +import org.springframework.yarn.am.grid.support.DefaultGridProjectionFactory; +import org.springframework.yarn.am.grid.support.HostsGridProjection; +import org.springframework.yarn.am.grid.support.ProjectionData; +import org.springframework.yarn.boot.MockUtils; +import org.springframework.yarn.boot.TestUtils; +import org.springframework.yarn.boot.actuate.endpoint.YarnContainerClusterEndpoint; +import org.springframework.yarn.boot.actuate.endpoint.mvc.YarnContainerClusterMvcEndpointTests.TestConfiguration; +import org.springframework.yarn.listener.ContainerAllocatorListener; + +@RunWith(SpringJUnit4ClassRunner.class) +@SpringApplicationConfiguration(classes = { TestConfiguration.class }) +@WebAppConfiguration +@DirtiesContext(classMode=ClassMode.AFTER_EACH_TEST_METHOD) +public class YarnContainerClusterMvcEndpointTests { + + private final static String BASE = "/" + YarnContainerClusterEndpoint.ENDPOINT_ID; + + @Autowired + private WebApplicationContext context; + + private MockMvc mvc; + + private TestYarnAppmaster appmaster; + + @Before + public void setUp() { + mvc = MockMvcBuilders.webAppContextSetup(context).build(); + appmaster = (TestYarnAppmaster) context.getBean(YarnAppmaster.class); + } + + /** + * Test home GET when no clusters has been defined. + */ + @Test + public void testHomeEmpty() throws Exception { + mvc. + perform(get(BASE)). + andExpect(status().isOk()). +// andExpect(content().string(is("foo"))). + andExpect(content().string(not(isEmptyString()))). + andExpect(jsonPath("$.*", hasSize(1))). + andExpect(jsonPath("$.clusters", hasSize(0))); + } + + /** + * Test cluster create using any projection. + */ + @Test + public void testClusterCreateAnyProjection() throws Exception { + String content = "{\"clusterId\":\"cluster1\",\"clusterDef\":\"cluster1\",\"projection\":\"ANY\",\"projectionData\":{\"any\":1}},\"extraProperties\":{\"key1\":\"value1\"}}"; + mvc. + perform(post(BASE).content(content).contentType(MediaType.APPLICATION_JSON)). + andExpect(status().isCreated()). + andExpect(content().string(is(""))). + andExpect(header().string("Location", endsWith(BASE + "/cluster1"))); + Map clusters = TestUtils.readField("clusters", appmaster); + assertThat(clusters.size(), is(1)); + assertThat(clusters.containsKey("cluster1"), is(true)); + assertThat(clusters.get("cluster1").getGridProjection().getProjectionData().getAny(), is(1)); + } + + /** + * Test cluster create using hosts projection. + */ + @Test + public void testClusterCreateHostsProjection() throws Exception { + String content = "{\"clusterId\":\"cluster1\",\"projection\":\"HOSTS\",\"projectionData\":{\"any\":1,\"hosts\":{\"host1\":11,\"host2\":22}}}"; + mvc. + perform(post(BASE).content(content).contentType(MediaType.APPLICATION_JSON)). + andExpect(status().isCreated()). + andExpect(content().string(is(""))). + andExpect(header().string("Location", endsWith(BASE + "/cluster1"))); + + Map clusters = TestUtils.readField("clusters", appmaster); + assertThat(clusters.size(), is(1)); + assertThat(clusters.containsKey("cluster1"), is(true)); + assertThat(clusters.get("cluster1").getGridProjection(), instanceOf(HostsGridProjection.class)); + assertThat(clusters.get("cluster1").getGridProjection().getSatisfyState().getAllocateData().getAny(), is(0)); + assertThat(clusters.get("cluster1").getGridProjection().getSatisfyState().getAllocateData().getHosts().size(), is(2)); + assertThat(clusters.get("cluster1").getGridProjection().getSatisfyState().getAllocateData().getHosts().get("host1"), is(11)); + assertThat(clusters.get("cluster1").getGridProjection().getSatisfyState().getAllocateData().getHosts().get("host2"), is(22)); + } + + + /** + * Test home GET when one any cluster has been created. + */ + @Test + public void testHomeWithAnyCluster() throws Exception { + testClusterCreateAnyProjection(); + + mvc. + perform(get(BASE)). + andExpect(status().isOk()). + andExpect(content().string(not(isEmptyString()))). + andExpect(jsonPath("$.*", hasSize(1))). + andExpect(jsonPath("$.clusters", hasSize(1))). + andExpect(jsonPath("$.clusters[0]", is("cluster1"))); + } + + /** + * Test home GET when one hosts cluster has been created. + */ + @Test + public void testHomeWithHostsCluster() throws Exception { + testClusterCreateHostsProjection(); + + mvc. + perform(get(BASE)). + andExpect(status().isOk()). + andExpect(content().string(not(isEmptyString()))). + andExpect(jsonPath("$.*", hasSize(1))). + andExpect(jsonPath("$.clusters", hasSize(1))). + andExpect(jsonPath("$.clusters[0]", is("cluster1"))); + } + + @Test + public void testClusterWithMember() throws Exception { + testHomeWithAnyCluster(); + TestUtils.callMethod("doTask", appmaster); + allocateContainer(appmaster, 1); + + mvc. + perform(get(BASE + "/cluster1")). + andExpect(status().isOk()). + andExpect(content().string(not(isEmptyString()))). + andExpect(jsonPath("$.*", hasSize(3))). + andExpect(jsonPath("$.id", is("cluster1"))). + andExpect(jsonPath("$.gridProjection.*", hasSize(3))). + andExpect(jsonPath("$.gridProjection.members", hasSize(1))). + andExpect(jsonPath("$.gridProjection.projectionData.*", hasSize(5))). + andExpect(jsonPath("$.gridProjection.projectionData.type", is("any"))). + andExpect(jsonPath("$.gridProjection.projectionData.priority", nullValue())). + andExpect(jsonPath("$.gridProjection.projectionData.any", is(1))). + andExpect(jsonPath("$.gridProjection.projectionData.hosts.*", hasSize(0))). + andExpect(jsonPath("$.gridProjection.projectionData.racks.*", hasSize(0))). + andExpect(jsonPath("$.gridProjection.satisfyState.*", hasSize(2))). + andExpect(jsonPath("$.gridProjection.satisfyState.allocateData.*", hasSize(3))). + andExpect(jsonPath("$.gridProjection.satisfyState.allocateData.racks.*", hasSize(0))). + andExpect(jsonPath("$.gridProjection.satisfyState.allocateData.any", is(0))). + andExpect(jsonPath("$.gridProjection.satisfyState.allocateData.hosts.*", hasSize(0))). + andExpect(jsonPath("$.gridProjection.satisfyState.removeData", hasSize(0))). + andExpect(jsonPath("$.containerClusterState.clusterState", is(ClusterState.INITIAL.toString()))); + } + + @Test + public void testStartCluster() throws Exception { + String content = "{\"clusterId\":\"foo\",\"projection\":\"ANY\",\"projectionData\":{\"any\":1}}"; + mvc. + perform(post(BASE).content(content).contentType(MediaType.APPLICATION_JSON)); + + content = "{\"action\":\"start\"}"; + mvc. + perform(put(BASE + "/foo").content(content).contentType(MediaType.APPLICATION_JSON)). + andExpect(status().isOk()). + andExpect(content().string(not(isEmptyString()))); + + Map clusters = TestUtils.readField("clusters", appmaster); + assertThat(clusters.size(), is(1)); + assertThat(clusters.containsKey("foo"), is(true)); + assertThat(clusters.get("foo").getStateMachine().getState().getId(), is(ClusterState.RUNNING)); + } + + @Test + public void testStartClusterDoesNotExist() throws Exception { + String content = "{\"action\":\"start\"}"; + mvc. + perform(put(BASE + "/foo").content(content).contentType(MediaType.APPLICATION_JSON)). + andExpect(status().isNotFound()). + andExpect(status().reason("No such cluster")); + } + + @Test + public void testStopCluster() throws Exception { + testStartCluster(); + String content = "{\"action\":\"stop\"}"; + mvc. + perform(put(BASE + "/foo").content(content).contentType(MediaType.APPLICATION_JSON)). + andExpect(status().isOk()). + andExpect(content().string(not(isEmptyString()))); + Map clusters = TestUtils.readField("clusters", appmaster); + assertThat(clusters.size(), is(1)); + assertThat(clusters.containsKey("foo"), is(true)); + assertThat(clusters.get("foo").getStateMachine().getState().getId(), is(ClusterState.STOPPED)); + } + + @Test + public void testClusterStatus() throws Exception { + testClusterCreateHostsProjection(); + mvc. + perform(get(BASE + "/cluster1")). + andExpect(status().isOk()). + andExpect(content().string(containsString("cluster1"))); + + } + + @Test + public void testClusterModify() throws Exception { + testStartCluster(); + String content = "{\"clusterId\":\"foo\",\"projectionData\":{\"any\":2}}}"; + mvc. + perform(patch(BASE + "/foo").content(content).contentType(MediaType.APPLICATION_JSON)). + andExpect(status().isOk()); + Map clusters = TestUtils.readField("clusters", appmaster); + assertThat(clusters.size(), is(1)); + assertThat(clusters.containsKey("foo"), is(true)); + assertThat(clusters.get("foo").getGridProjection().getProjectionData().getAny(), is(2)); + } + + @Import({ ContainerClusterStateMachineConfiguration.class, EndpointWebMvcAutoConfiguration.class, ManagementServerPropertiesAutoConfiguration.class, + HypermediaAutoConfiguration.class }) + @EnableWebMvc + @Configuration + public static class TestConfiguration { + + @Bean + public YarnContainerClusterEndpoint endpoint() { + return new YarnContainerClusterEndpoint(); + } + + @Bean + public YarnContainerClusterMvcEndpoint mvcEndpoint() { + return new YarnContainerClusterMvcEndpoint(endpoint()); + } + + @Bean + public YarnAppmaster appMaster() { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestYarnAppmaster appmaster = new TestYarnAppmaster(); + appmaster.setAllocator(allocator); + appmaster.setLauncher(launcher); + return appmaster; + } + + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + + @Bean + public GridProjectionFactory gridProjectionFactory() { + Map defaults = new HashMap(); + defaults.put("cluster1", new ProjectionData(null, null, null, "any", 0)); + return new DefaultGridProjectionFactory(defaults); + } + + } + + protected static class TestYarnAppmaster extends ManagedContainerClusterAppmaster { + @Override + protected void onInit() throws Exception { + setConfiguration(new org.apache.hadoop.conf.Configuration()); + super.onInit(); + } + @Override + protected void doStart() { + } + @Override + protected void doStop() { + } + } + + protected static class TestContainerAllocator implements ContainerAllocator { + + ArrayList containerAllocateData; + ArrayList releaseContainers; + + public void resetTestData() { + containerAllocateData = null; + releaseContainers = null; + } + + @Override + public void allocateContainers(int count) { + } + + @Override + public void allocateContainers(ContainerAllocateData containerAllocateData) { + if (this.containerAllocateData == null) { + this.containerAllocateData = new ArrayList(); + } + this.containerAllocateData.add(containerAllocateData); + } + + @Override + public void releaseContainers(List containers) { + if (this.releaseContainers == null) { + this.releaseContainers = new ArrayList(); + } + this.releaseContainers.addAll(containers); + } + + @Override + public void releaseContainer(ContainerId containerId) { + } + + @Override + public void addListener(ContainerAllocatorListener listener) { + } + + @Override + public void setProgress(float progress) { + } + + } + + protected static class TestContainerLauncher implements ContainerLauncher { + + Container container; + + public void resetTestData() { + container = null; + } + + @Override + public void launchContainer(Container container, List commands) { + this.container = container; + } + + } + + protected Container allocateContainer(Object appmaster, int id) throws Exception { + return allocateContainer(appmaster, id, null); + } + + protected Container allocateContainer(Object appmaster, int id, String host) throws Exception { + ContainerId containerId = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 0); + NodeId nodeId = MockUtils.getMockNodeId(host, 0); + Priority priority = MockUtils.getMockPriority(0); + Container container = MockUtils.getMockContainer(containerId, nodeId, null, priority); + TestUtils.callMethod("onContainerAllocated", appmaster, new Object[]{container}, new Class[]{Container.class}); + return container; + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterApplicationOperationPropertiesTests.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterApplicationOperationPropertiesTests.java new file mode 100644 index 000000000..78a914c6d --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterApplicationOperationPropertiesTests.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.app; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import org.junit.Test; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.yarn.boot.app.YarnContainerClusterApplication.OperationProperties; + +public class YarnContainerClusterApplicationOperationPropertiesTests { + + @Test + public void testProjectionDataProperties() { + SpringApplication app = new SpringApplication(TestConfiguration.class); + app.setWebEnvironment(false); + ConfigurableApplicationContext context = app.run(new String[] { + "--spring.yarn.internal.ContainerClusterApplication.projectionDataAny=1", + "--spring.yarn.internal.ContainerClusterApplication.projectionDataHosts.host1=1", + "--spring.yarn.internal.ContainerClusterApplication.projectionDataRacks.rack1=1", + "--spring.yarn.internal.ContainerClusterApplication.projectionDataHosts.host2=2", + "--spring.yarn.internal.ContainerClusterApplication.projectionDataRacks.rack2=2" }); + OperationProperties properties = context.getBean(OperationProperties.class); + assertThat(properties, notNullValue()); + assertThat(properties.getProjectionDataAny(), is(1)); + assertThat(properties.getProjectionDataHosts().get("host1"), is(1)); + assertThat(properties.getProjectionDataHosts().get("host2"), is(2)); + assertThat(properties.getProjectionDataRacks().get("rack1"), is(1)); + assertThat(properties.getProjectionDataRacks().get("rack2"), is(2)); + context.close(); + } + + @Configuration + @EnableConfigurationProperties({ OperationProperties.class }) + protected static class TestConfiguration { + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplateTests.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplateTests.java new file mode 100644 index 000000000..f852c5b0c --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/app/YarnContainerClusterTemplateTests.java @@ -0,0 +1,131 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.app; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThat; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.jsonPath; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; +import org.springframework.yarn.boot.actuate.endpoint.YarnContainerClusterEndpoint; +import org.springframework.yarn.boot.actuate.endpoint.mvc.AbstractContainerClusterRequest.ProjectionDataType; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterCreateRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterCreateRequest.ProjectionType; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterModifyRequest; +import org.springframework.yarn.boot.actuate.endpoint.mvc.ContainerClusterModifyRequest.ModifyAction; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.ContainerClusterResource; +import org.springframework.yarn.boot.actuate.endpoint.mvc.domain.YarnContainerClusterEndpointResource; + +/** + * Tests for {@link YarnContainerClusterTemplate} and {@link YarnContainerClusterOperations}. + * + * @author Janne Valkealahti + * + */ +public class YarnContainerClusterTemplateTests { + + private MockRestServiceServer mockServer; + + private RestTemplate restTemplate; + + private YarnContainerClusterOperations operations; + + @Before + public void setUp() { + restTemplate = new RestTemplate(); + mockServer = MockRestServiceServer.createServer(restTemplate); + operations = new YarnContainerClusterTemplate("/" + YarnContainerClusterEndpoint.ENDPOINT_ID, restTemplate); + } + + @Test + public void testHome() { + String responseBody = "{\"clusters\":[]}"; + mockServer. + expect(requestTo("/" + YarnContainerClusterEndpoint.ENDPOINT_ID)). + andExpect(method(HttpMethod.GET)). + andRespond(withSuccess(responseBody, MediaType.APPLICATION_JSON)); + YarnContainerClusterEndpointResource response = operations.getClusters(); + assertThat(response, notNullValue()); + } + + @Test + public void testStart() { + String responseBody = "{" + "\"id\":\"cluster1\"," + + "\"gridProjection\":{\"members\":[],\"projectionData\":{\"any\":1,\"hosts\":{},\"racks\":{}}," + + "\"satisfyState\":null},\"containerClusterState\":{\"clusterState\":\"RUNNING\"}" + "}"; + mockServer. + expect(requestTo("/" + YarnContainerClusterEndpoint.ENDPOINT_ID + "/cluster1")). + andExpect(method(HttpMethod.PUT)). + andExpect(jsonPath("$.*", hasSize(2))). + andExpect(jsonPath("$.projectionData", nullValue())). + andExpect(jsonPath("$.action", is(ModifyAction.START.toString().toLowerCase()))). + andRespond(withSuccess(responseBody, MediaType.APPLICATION_JSON)); + ContainerClusterModifyRequest request = new ContainerClusterModifyRequest(); + request.setAction("start"); + ContainerClusterResource response = operations.clusterStart("cluster1", request); + assertThat(response, notNullValue()); + } + + @Test + public void testStop() { + String responseBody = "{" + "\"id\":\"cluster1\"," + + "\"gridProjection\":{\"members\":[],\"projectionData\":{\"any\":1,\"hosts\":{},\"racks\":{}}," + + "\"satisfyState\":null},\"containerClusterState\":{\"clusterState\":\"STOPPING\"}" + "}"; + mockServer. + expect(requestTo("/" + YarnContainerClusterEndpoint.ENDPOINT_ID + "/cluster1")). + andExpect(method(HttpMethod.PUT)). + andExpect(jsonPath("$.*", hasSize(2))). + andExpect(jsonPath("$.projectionData", nullValue())). + andExpect(jsonPath("$.action", is(ModifyAction.STOP.toString().toLowerCase()))). + andRespond(withSuccess(responseBody, MediaType.APPLICATION_JSON)); + ContainerClusterModifyRequest request = new ContainerClusterModifyRequest(); + request.setAction("stop"); + ContainerClusterResource response = operations.clusterStop("cluster1", request); + assertThat(response, notNullValue()); + } + + @Test + public void testCreateCluster() { + String responseBody = "{" + "\"id\":\"cluster1\"," + + "\"gridProjection\":{\"members\":[],\"projectionData\":{\"any\":1,\"hosts\":{},\"racks\":{}}," + + "\"satisfyState\":null},\"containerClusterState\":null" + "}"; + + mockServer.expect(requestTo("/" + YarnContainerClusterEndpoint.ENDPOINT_ID)).andExpect(method(HttpMethod.POST)) + .andRespond(withSuccess(responseBody, MediaType.APPLICATION_JSON)); + + ContainerClusterCreateRequest request = new ContainerClusterCreateRequest(); + request.setClusterId("cluster1"); + request.setProjection(ProjectionType.ANY); + ProjectionDataType projectionData = new ProjectionDataType(); + projectionData.setAny(1); + request.setProjectionData(projectionData); + + ContainerClusterResource response = operations.clusterCreate(request); + assertThat(response, notNullValue()); + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterContainerClusterPropertiesTests.java b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterContainerClusterPropertiesTests.java new file mode 100644 index 000000000..f42ad7bba --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/properties/SpringYarnAppmasterContainerClusterPropertiesTests.java @@ -0,0 +1,282 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.boot.properties; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.util.List; +import java.util.Map; + +import org.junit.Test; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; + +/** + * Tests for {@link SpringYarnAppmasterContainerClusterProperties} bindings. + * + * @author Janne Valkealahti + * + */ +public class SpringYarnAppmasterContainerClusterPropertiesTests { + + @Test + public void testAllPropertiesSet() { + SpringApplication app = new SpringApplication(TestConfiguration.class); + app.setWebEnvironment(false); + ConfigurableApplicationContext context = app + .run(new String[] { "--spring.config.name=SpringYarnAppmasterContainerClusterPropertiesTests1" }); + SpringYarnAppmasterProperties properties = context.getBean(SpringYarnAppmasterProperties.class); + assertThat(properties, notNullValue()); + + assertThat(properties.getContainercluster().getClusters().size(), is(2)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionType(), is("any")); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionAny(), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().size(), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().get("host1"), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().get("host2"), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().size(), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().get("rack1"), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().get("rack2"), is(2)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource(), notNullValue()); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getPriority(), is(234)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getMemory(), is("memoryFoo")); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getVirtualCores(), is(123)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getLaunchcontext(), notNullValue()); + SpringYarnAppmasterLaunchContextProperties properties1 = properties.getContainercluster().getClusters().get("cluster1").getLaunchcontext(); + assertThat(properties1, notNullValue()); + assertThat(properties1.getArchiveFile(), is("archiveFileFoo")); + Map arguments = properties1.getArguments(); + assertThat(arguments, notNullValue()); + assertThat(arguments.size(), is(2)); + assertThat(arguments.get("argumentsKeyFoo1"), is("argumentsValFoo1")); + assertThat(arguments.get("argumentsKeyFoo2"), is("argumentsValFoo2")); + List classpath = properties1.getContainerAppClasspath(); + assertThat(classpath, notNullValue()); + assertThat(classpath.size(), is(2)); + assertThat(classpath.get(0), is("classpath1Foo")); + assertThat(classpath.get(1), is("classpath2Foo")); + assertThat(properties1.getRunnerClass(), is("runnerClassFoo")); + List options = properties1.getOptions(); + assertThat(options, notNullValue()); + assertThat(options.size(), is(2)); + assertThat(options.get(0), is("options1Foo")); + assertThat(options.get(1), is("options2Foo")); + assertThat(properties1.isLocality(), is(true)); + assertThat(properties1.isUseYarnAppClasspath(), is(false)); + assertThat(properties1.isIncludeBaseDirectory(), is(false)); + assertThat(properties1.isIncludeLocalSystemEnv(), is(true)); + assertThat(properties1.getPathSeparator(), is(":")); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getLocalizer(), notNullValue()); + SpringYarnAppmasterLocalizerProperties properties2 = properties.getContainercluster().getClusters().get("cluster1").getLocalizer(); + assertThat(properties2, notNullValue()); + List pattern = properties2.getPatterns(); + assertThat(pattern, notNullValue()); + assertThat(pattern.size(), is(2)); + assertThat(pattern.get(0), is("patterns1Foo")); + assertThat(pattern.get(1), is("patterns2Foo")); + List names = properties2.getPropertiesNames(); + assertThat(names, notNullValue()); + assertThat(names.size(), is(2)); + assertThat(names.get(0), is("name1Foo")); + assertThat(names.get(1), is("name2Foo")); + List suffixes = properties2.getPropertiesSuffixes(); + assertThat(suffixes, notNullValue()); + assertThat(suffixes.size(), is(2)); + assertThat(suffixes.get(0), is("suffix1Foo")); + assertThat(suffixes.get(1), is("suffix2Foo")); + assertThat(properties2.getZipPattern(), is("zipPatternFoo")); + + assertThat(properties.getContainercluster().getClusters().get("cluster2").getResource(), notNullValue()); + assertThat(properties.getContainercluster().getClusters().get("cluster2").getLaunchcontext(), nullValue()); + assertThat(properties.getContainercluster().getClusters().get("cluster2").getLocalizer(), nullValue()); + assertThat(properties.getContainercluster().getClusters().get("cluster2").getResource().getPriority(), is(2344)); + assertThat(properties.getContainercluster().getClusters().get("cluster2").getResource().getMemory(), is("memoryFooo")); + assertThat(properties.getContainercluster().getClusters().get("cluster2").getResource().getVirtualCores(), is(1233)); + context.close(); + } + + @Test + public void testAllPropertiesSet2() { + SpringApplication app = new SpringApplication(TestConfiguration.class); + app.setWebEnvironment(false); + ConfigurableApplicationContext context = app + .run(new String[] { "--spring.config.name=SpringYarnAppmasterContainerClusterPropertiesTests2" }); + SpringYarnAppmasterProperties properties = context.getBean(SpringYarnAppmasterProperties.class); + assertThat(properties, notNullValue()); + + assertThat(properties.getContainercluster().isEnabled(), is(true)); + + assertThat(properties.getContainercluster().getClusters().size(), is(1)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionType(), is("any")); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionAny(), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().size(), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().get("host1"), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().get("host2"), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().size(), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().get("rack1"), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().get("rack2"), is(2)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource(), notNullValue()); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getPriority(), is(234)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getMemory(), is("memoryFoo")); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getVirtualCores(), is(123)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getLaunchcontext(), notNullValue()); + SpringYarnAppmasterLaunchContextProperties properties1 = properties.getContainercluster().getClusters().get("cluster1").getLaunchcontext(); + assertThat(properties1, notNullValue()); + assertThat(properties1.getArchiveFile(), is("archiveFileFoo")); + Map arguments = properties1.getArguments(); + assertThat(arguments, notNullValue()); + assertThat(arguments.size(), is(2)); + assertThat(arguments.get("argumentsKeyFoo1"), is("argumentsValFoo1")); + assertThat(arguments.get("argumentsKeyFoo2"), is("argumentsValFoo2")); + List classpath = properties1.getContainerAppClasspath(); + assertThat(classpath, notNullValue()); + assertThat(classpath.size(), is(2)); + assertThat(classpath.get(0), is("classpath1Foo")); + assertThat(classpath.get(1), is("classpath2Foo")); + assertThat(properties1.getRunnerClass(), is("runnerClassFoo")); + List options = properties1.getOptions(); + assertThat(options, notNullValue()); + assertThat(options.size(), is(2)); + assertThat(options.get(0), is("options1Foo")); + assertThat(options.get(1), is("options2Foo")); + assertThat(properties1.isLocality(), is(true)); + assertThat(properties1.isUseYarnAppClasspath(), is(false)); + assertThat(properties1.isIncludeBaseDirectory(), is(false)); + assertThat(properties1.isIncludeLocalSystemEnv(), is(true)); + assertThat(properties1.getPathSeparator(), is(":")); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getLocalizer(), notNullValue()); + SpringYarnAppmasterLocalizerProperties properties2 = properties.getContainercluster().getClusters().get("cluster1").getLocalizer(); + assertThat(properties2, notNullValue()); + List pattern = properties2.getPatterns(); + assertThat(pattern, notNullValue()); + assertThat(pattern.size(), is(2)); + assertThat(pattern.get(0), is("patterns1Foo")); + assertThat(pattern.get(1), is("patterns2Foo")); + List names = properties2.getPropertiesNames(); + assertThat(names, notNullValue()); + assertThat(names.size(), is(2)); + assertThat(names.get(0), is("name1Foo")); + assertThat(names.get(1), is("name2Foo")); + List suffixes = properties2.getPropertiesSuffixes(); + assertThat(suffixes, notNullValue()); + assertThat(suffixes.size(), is(2)); + assertThat(suffixes.get(0), is("suffix1Foo")); + assertThat(suffixes.get(1), is("suffix2Foo")); + assertThat(properties2.getZipPattern(), is("zipPatternFoo")); + + context.close(); + } + + @Test + public void testAllMultiYaml() { + SpringApplication app = new SpringApplication(TestConfiguration.class); + app.setWebEnvironment(false); + ConfigurableApplicationContext context = app + .run(new String[] { "--spring.config.name=SpringYarnAppmasterContainerClusterPropertiesTests2", + "--spring.config.location=classpath:/SpringYarnAppmasterContainerClusterPropertiesTests2-2.yml"}); + SpringYarnAppmasterProperties properties = context.getBean(SpringYarnAppmasterProperties.class); + assertThat(properties, notNullValue()); + + assertThat(properties.getContainercluster().isEnabled(), is(true)); + + assertThat(properties.getContainercluster().getClusters().size(), is(2)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionType(), is("any")); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionAny(), is(1)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().size(), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().get("host1"), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionHosts().get("host2"), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().size(), is(2)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().get("rack1"), is(1)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getProjectionRacks().get("rack2"), is(2)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource(), notNullValue()); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getPriority(), is(234)); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getMemory(), is("memoryFoo")); + assertThat(properties.getContainercluster().getClusters().get("cluster1").getResource().getVirtualCores(), is(123)); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getLaunchcontext(), notNullValue()); + SpringYarnAppmasterLaunchContextProperties properties1 = properties.getContainercluster().getClusters().get("cluster1").getLaunchcontext(); + assertThat(properties1, notNullValue()); + assertThat(properties1.getArchiveFile(), is("archiveFileFoo")); + Map arguments = properties1.getArguments(); + assertThat(arguments, notNullValue()); + assertThat(arguments.size(), is(2)); + assertThat(arguments.get("argumentsKeyFoo1"), is("argumentsValFoo1")); + assertThat(arguments.get("argumentsKeyFoo2"), is("argumentsValFoo2")); + List classpath = properties1.getContainerAppClasspath(); + assertThat(classpath, notNullValue()); + assertThat(classpath.size(), is(2)); + assertThat(classpath.get(0), is("classpath1Foo")); + assertThat(classpath.get(1), is("classpath2Foo")); + assertThat(properties1.getRunnerClass(), is("runnerClassFoo")); + List options = properties1.getOptions(); + assertThat(options, notNullValue()); + assertThat(options.size(), is(2)); + assertThat(options.get(0), is("options1Foo")); + assertThat(options.get(1), is("options2Foo")); + assertThat(properties1.isLocality(), is(true)); + assertThat(properties1.isUseYarnAppClasspath(), is(false)); + assertThat(properties1.isIncludeBaseDirectory(), is(false)); + assertThat(properties1.isIncludeLocalSystemEnv(), is(true)); + assertThat(properties1.getPathSeparator(), is(":")); + + assertThat(properties.getContainercluster().getClusters().get("cluster1").getLocalizer(), notNullValue()); + SpringYarnAppmasterLocalizerProperties properties2 = properties.getContainercluster().getClusters().get("cluster1").getLocalizer(); + assertThat(properties2, notNullValue()); + List pattern = properties2.getPatterns(); + assertThat(pattern, notNullValue()); + assertThat(pattern.size(), is(2)); + assertThat(pattern.get(0), is("patterns1Foo")); + assertThat(pattern.get(1), is("patterns2Foo")); + List names = properties2.getPropertiesNames(); + assertThat(names, notNullValue()); + assertThat(names.size(), is(2)); + assertThat(names.get(0), is("name1Foo")); + assertThat(names.get(1), is("name2Foo")); + List suffixes = properties2.getPropertiesSuffixes(); + assertThat(suffixes, notNullValue()); + assertThat(suffixes.size(), is(2)); + assertThat(suffixes.get(0), is("suffix1Foo")); + assertThat(suffixes.get(1), is("suffix2Foo")); + assertThat(properties2.getZipPattern(), is("zipPatternFoo")); + + assertThat(properties.getContainercluster().getClusters().get("cluster2").getProjectionType(), is("any")); + assertThat(properties.getContainercluster().getClusters().get("cluster2").getProjectionAny(), is(2)); + + context.close(); + } + + @Configuration + @EnableConfigurationProperties({SpringYarnAppmasterProperties.class}) + protected static class TestConfiguration { + } + +} diff --git a/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests1.yml b/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests1.yml new file mode 100644 index 000000000..593f949ec --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests1.yml @@ -0,0 +1,49 @@ +spring: + yarn: + appmaster: + containercluster: + clusters: + ? cluster1 + : projectionType: any + projectionAny: 1 + projectionHosts: + host1: 1 + host2: 2 + projectionRacks: + rack1: 1 + rack2: 2 + ? resource + : priority: 234 + memory: memoryFoo + virtualCores: 123 + ? launchcontext + : archiveFile: archiveFileFoo + runnerClass: runnerClassFoo + arguments: + ? argumentsKeyFoo1 + : argumentsValFoo1 + ? argumentsKeyFoo2 + : argumentsValFoo2 + containerAppClasspath: + - "classpath1Foo" + - "classpath2Foo" + options: + - "options1Foo" + - "options2Foo" + locality: true + includeLocalSystemEnv: true + useYarnAppClasspath: false + includeBaseDirectory: false + pathSeparator: ":" + ? localizer + : patterns: + - "patterns1Foo" + - "patterns2Foo" + zipPattern: zipPatternFoo + propertiesNames: [name1Foo, name2Foo] + propertiesSuffixes: [suffix1Foo, suffix2Foo] + ? cluster2 + : ? resource + : priority: 2344 + memory: memoryFooo + virtualCores: 1233 diff --git a/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2-2.yml b/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2-2.yml new file mode 100644 index 000000000..9b06f9c4a --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2-2.yml @@ -0,0 +1,8 @@ +spring: + yarn: + appmaster: + containercluster: + clusters: + cluster2: + projectionType: any + projectionAny: 2 diff --git a/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2.yml b/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2.yml new file mode 100644 index 000000000..b8f10253d --- /dev/null +++ b/spring-yarn/spring-yarn-boot/src/test/resources/SpringYarnAppmasterContainerClusterPropertiesTests2.yml @@ -0,0 +1,43 @@ +spring: + yarn: + appmaster: + containercluster: + enabled: true + clusters: + cluster1: + projectionType: any + projectionAny: 1 + projectionHosts: + host1: 1 + host2: 2 + projectionRacks: + rack1: 1 + rack2: 2 + resource: + priority: 234 + memory: memoryFoo + virtualCores: 123 + launchcontext: + archiveFile: archiveFileFoo + runnerClass: runnerClassFoo + arguments: + argumentsKeyFoo1: argumentsValFoo1 + argumentsKeyFoo2: argumentsValFoo2 + containerAppClasspath: + - "classpath1Foo" + - "classpath2Foo" + options: + - "options1Foo" + - "options2Foo" + locality: true + includeLocalSystemEnv: true + useYarnAppClasspath: false + includeBaseDirectory: false + pathSeparator: ":" + localizer: + patterns: + - "patterns1Foo" + - "patterns2Foo" + zipPattern: zipPatternFoo + propertiesNames: [name1Foo, name2Foo] + propertiesSuffixes: [suffix1Foo, suffix2Foo] diff --git a/spring-yarn/spring-yarn-build-tests/src/test/java/org/springframework/yarn/fs/DefaultResourceLocalizerTests.java b/spring-yarn/spring-yarn-build-tests/src/test/java/org/springframework/yarn/fs/DefaultResourceLocalizerTests.java index a188f9fcf..0eb324d8f 100644 --- a/spring-yarn/spring-yarn-build-tests/src/test/java/org/springframework/yarn/fs/DefaultResourceLocalizerTests.java +++ b/spring-yarn/spring-yarn-build-tests/src/test/java/org/springframework/yarn/fs/DefaultResourceLocalizerTests.java @@ -24,8 +24,6 @@ import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -34,7 +32,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.hadoop.fs.FsShell; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ContextConfiguration; @@ -57,8 +54,6 @@ @DirtiesContext(classMode=ClassMode.AFTER_CLASS) public class DefaultResourceLocalizerTests { - private final static Log log = LogFactory.getLog(DefaultResourceLocalizerTests.class); - @Autowired private Configuration configuration; @@ -252,8 +247,6 @@ public void testRawEntries() throws Exception { SmartResourceLocalizer localizer = (SmartResourceLocalizer) factory.getObject(); localizer.copy(); - listFiles(); - FileSystem fs = FileSystem.get(configuration); FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/rawContent1")); assertThat(fileStatus.isFile(), is(true)); @@ -280,22 +273,12 @@ public void testRawEntriesStaging() throws Exception { localizer.setStagingId("foo-id"); localizer.copy(); - listFiles(); - FileSystem fs = FileSystem.get(configuration); FileStatus fileStatus = fs.getFileStatus(new Path(dir + "/foo-id/rawContent1")); assertThat(fileStatus.isFile(), is(true)); assertThat(fileStatus.getLen(), is(10l)); } - private void listFiles() { - @SuppressWarnings("resource") - FsShell shell = new FsShell(configuration); - for (FileStatus s : shell.ls(true, "/")) { - log.info("XXX: " + s); - } - } - @org.springframework.context.annotation.Configuration public static class Config { diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/AbstractAppmaster.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/AbstractAppmaster.java index 397d4f493..ca0b9c469 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/AbstractAppmaster.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/AbstractAppmaster.java @@ -16,6 +16,7 @@ package org.springframework.yarn.am; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -53,13 +54,14 @@ public abstract class AbstractAppmaster extends LifecycleObjectSupport { private static final Log log = LogFactory.getLog(AbstractAppmaster.class); /** Environment variables for the process */ - private Map environment; +// private Map environment; + private final HashMap> environments = new HashMap>(); /** Yarn configuration */ private Configuration configuration; - /** Commands for container start */ - private List commands; + /** Commands for container start where value mapped with null key indicates defaults */ + private final Map> commands = new HashMap>(); /** Template operations talking to resource manager */ private AppmasterRmOperations rmTemplate; @@ -137,7 +139,12 @@ public void setTemplate(AppmasterRmOperations template) { * @return the environment variables */ public Map getEnvironment() { - return environment; +// return environment; + return getEnvironment(null); + } + + public Map getEnvironment(String id) { + return environments.get(id); } /** @@ -146,7 +153,12 @@ public Map getEnvironment() { * @param environment the environment variables */ public void setEnvironment(Map environment) { - this.environment = environment; +// this.environment = environment; + setEnvironment(null, environment); + } + + public void setEnvironment(String id, Map environment) { + environments.put(id, environment); } /** @@ -191,7 +203,17 @@ public void setConfiguration(Configuration configuration) { * @return the commands */ public List getCommands() { - return commands; + return commands.get(null); + } + + /** + * Gets the commands. + * + * @param id the commands identifier + * @return the commands + */ + public List getCommands(String id) { + return commands.get(id); } /** @@ -200,7 +222,17 @@ public List getCommands() { * @param commands the new commands */ public void setCommands(List commands) { - this.commands = commands; + this.commands.put(null, commands); + } + + /** + * Sets the commands with an identifier. + * + * @param id the commands identifier + * @param commands the new commands + */ + public void setCommands(String id, List commands) { + this.commands.put(id, commands); } /** @@ -209,7 +241,17 @@ public void setCommands(List commands) { * @param commands the new commands */ public void setCommands(String[] commands) { - this.commands = Arrays.asList(commands); + setCommands(Arrays.asList(commands)); + } + + /** + * Sets the commands with an identifier. + * + * @param id the commands identifier + * @param commands the new commands + */ + public void setCommands(String id, String[] commands) { + setCommands(id, Arrays.asList(commands)); } /** diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/ContainerAllocateData.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/ContainerAllocateData.java index 68ecb3bfc..d3a9b61d2 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/ContainerAllocateData.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/ContainerAllocateData.java @@ -28,6 +28,7 @@ */ public class ContainerAllocateData { + private String id; private Integer anyData = 0; private Map hostData = new ConcurrentHashMap(); private Map rackData = new ConcurrentHashMap(); @@ -110,6 +111,14 @@ public int getAny() { return anyData; } + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + @Override public String toString() { StringBuilder buf = new StringBuilder(); @@ -121,7 +130,7 @@ public String toString() { for (Entry entry : hostData.entrySet()) { buf.append(entry.getKey() + "=" + entry.getValue() + " "); } - buf.append("}]"); + buf.append("}], "); buf.append("rackData=[size=" + rackData.size() + ", "); buf.append("{"); diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultAllocateCountTracker.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultAllocateCountTracker.java index 6c84d913c..723c8af4a 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultAllocateCountTracker.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultAllocateCountTracker.java @@ -61,7 +61,13 @@ public class DefaultAllocateCountTracker { /** Counts for anys requested and not yet received */ private AtomicInteger requestedAny = new AtomicInteger(); - private final Configuration configuration; + private Configuration configuration; + + private String id; + + public String getId() { + return id; + } /** * Instantiates a new default allocate count tracker. @@ -72,6 +78,15 @@ public DefaultAllocateCountTracker(Configuration configuration) { this.configuration = configuration; } + public DefaultAllocateCountTracker(String id, Configuration configuration) { + this.configuration = configuration; + this.id = id; + } + + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + /** * Adds new count of containers into 'any' * pending requests. diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultContainerAllocator.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultContainerAllocator.java index 210b5a9da..67fd417e2 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultContainerAllocator.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/allocate/DefaultContainerAllocator.java @@ -18,9 +18,11 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import java.util.Queue; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -36,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.util.Records; +import org.springframework.util.StringUtils; import org.springframework.yarn.am.allocate.DefaultAllocateCountTracker.AllocateCountInfo; import org.springframework.yarn.listener.CompositeContainerAllocatorListener; import org.springframework.yarn.listener.ContainerAllocatorListener; @@ -67,6 +70,12 @@ public class DefaultContainerAllocator extends AbstractPollingAllocator implemen /** Locality relaxing */ private boolean locality = false; + private Map allocationParams = new ConcurrentHashMap(); + + private Map allocateCountTrackers = new ConcurrentHashMap(); + + private Map idToPriority = new ConcurrentHashMap(); + /** Increasing counter for rpc request id*/ private AtomicInteger requestId = new AtomicInteger(); @@ -79,9 +88,6 @@ public class DefaultContainerAllocator extends AbstractPollingAllocator implemen /** Internal set of containers marked as garbage by allocate tracker */ private Set garbageContainers = new HashSet(); - /** Tracker for request counts */ - private DefaultAllocateCountTracker allocateCountTracker; - /** Flag helping to avoid allocation garbage */ private AtomicBoolean allocationDirty = new AtomicBoolean(); @@ -91,7 +97,7 @@ public class DefaultContainerAllocator extends AbstractPollingAllocator implemen @Override protected void onInit() throws Exception { super.onInit(); - allocateCountTracker = new DefaultAllocateCountTracker(getConfiguration()); + internalInit(); } @Override @@ -99,7 +105,8 @@ public void allocateContainers(int count) { if (log.isDebugEnabled()) { log.debug("Incoming count: " + count); } - allocateCountTracker.addContainers(count); + DefaultAllocateCountTracker tracker = allocateCountTrackers.get(priority); + tracker.addContainers(count); allocationDirty.set(true); } @@ -111,10 +118,11 @@ public void addListener(ContainerAllocatorListener listener) { @Override public void allocateContainers(ContainerAllocateData containerAllocateData) { log.info("Incoming containerAllocateData: " + containerAllocateData); - log.info("State allocateCountTracker before adding allocation data: " + allocateCountTracker); - allocateCountTracker.addContainers(containerAllocateData); + DefaultAllocateCountTracker tracker = getAllocateCountTracker(containerAllocateData.getId()); + log.info("State allocateCountTracker before adding allocation data: " + tracker); + tracker.addContainers(containerAllocateData); allocationDirty.set(true); - log.info("State allocateCountTracker after adding allocation data: " + allocateCountTracker); + log.info("State allocateCountTracker after adding allocation data: " + tracker); } @Override @@ -130,23 +138,64 @@ public void releaseContainer(ContainerId containerId) { releaseContainers.add(containerId); } + public void setAllocationValues(String id, Integer priority, Integer virtualcores, Integer memory, Boolean locality) { + + if (log.isTraceEnabled()) { + log.trace("setAllocationValues 1: id=" + id + " priority=" + priority + " cores=" + virtualcores + " memory=" + memory + " locality=" + locality); + } + + id = StringUtils.hasText(id) ? id : ""; + if (!idToPriority.containsKey(id) && idToPriority.containsValue(priority)) { + throw new IllegalArgumentException("Key " + id + " is already mapped to priority " + priority); + } else { + idToPriority.put(id, priority); + } + allocationParams.put(priority, new ContainerAllocationValues(priority, virtualcores, memory, locality)); + allocateCountTrackers.put(priority, new DefaultAllocateCountTracker(id, getConfiguration())); + } + + private ContainerAllocationValues getAllocationValues(String id) { + return allocationParams.get(idToPriority.get(id)); + } + + private DefaultAllocateCountTracker getAllocateCountTracker(String id) { + id = StringUtils.hasText(id) ? id : ""; + return allocateCountTrackers.get(idToPriority.containsKey(id) ? idToPriority.get(id) : idToPriority.get("")); + } + private List createRequests() { List requestedContainers = new ArrayList(); - AllocateCountInfo allocateCounts = allocateCountTracker.getAllocateCounts(); - boolean hostsAdded = false; - for (Entry entry : allocateCounts.hostsInfo.entrySet()) { - requestedContainers.add(getContainerResourceRequest(entry.getValue(), entry.getKey(), true)); - hostsAdded = true; - } + for (DefaultAllocateCountTracker tracker : allocateCountTrackers.values()) { + AllocateCountInfo allocateCounts = tracker.getAllocateCounts(); + ContainerAllocationValues allocationValues = getAllocationValues(tracker.getId()); + if (log.isTraceEnabled()) { + log.trace("trace 1 " + allocationValues.locality); + log.trace("trace 2 tracker id:" + tracker.getId()); + } + boolean hostsAdded = false; + for (Entry entry : allocateCounts.hostsInfo.entrySet()) { + if (log.isTraceEnabled()) { + log.trace("trace 3 entry key=" + entry.getKey() + " value=" + entry.getValue()); + } + requestedContainers.add(getContainerResourceRequest(tracker.getId(), entry.getValue(), entry.getKey(), true)); + hostsAdded = true; + } + for (Entry entry : allocateCounts.racksInfo.entrySet()) { + if (log.isTraceEnabled()) { + log.trace("trace 4 entry key=" + entry.getKey() + " value=" + entry.getValue()); + } + requestedContainers.add(getContainerResourceRequest(tracker.getId(), entry.getValue(), entry.getKey(), (hostsAdded && allocationValues.locality) ? false : true)); + } + for (Entry entry : allocateCounts.anysInfo.entrySet()) { + if (log.isTraceEnabled()) { + log.trace("trace 5 entry key=" + entry.getKey() + " value=" + entry.getValue()); + } + requestedContainers.add(getContainerResourceRequest(tracker.getId(), entry.getValue(), entry.getKey(), !allocationValues.locality)); + } - for (Entry entry : allocateCounts.racksInfo.entrySet()) { - requestedContainers.add(getContainerResourceRequest(entry.getValue(), entry.getKey(), (hostsAdded && locality) ? false : true)); } - for (Entry entry : allocateCounts.anysInfo.entrySet()) { - requestedContainers.add(getContainerResourceRequest(entry.getValue(), entry.getKey(), !locality)); - } return requestedContainers; } @@ -201,26 +250,27 @@ protected List preProcessAllocatedContainers(List containe // processed by the listener and eventually send back // to us as a released container. - if (log.isDebugEnabled()) { - log.debug("State allocateCountTracker before handling allocated container: " + allocateCountTracker); - } - - List preProcessed = new ArrayList(); - for (Container container : containers) { - Container processed = allocateCountTracker.processAllocatedContainer(container); - if (processed != null) { - preProcessed.add(processed); - } else { - garbageContainers.add(container.getId()); - releaseContainers.add(container.getId()); - } - } - - if (log.isDebugEnabled()) { - log.debug("State allocateCountTracker after handling allocated container: " + allocateCountTracker); - } - - return preProcessed; + List preProcessed = new ArrayList(); + for (Container container : containers) { + + DefaultAllocateCountTracker tracker = allocateCountTrackers.get(container.getPriority().getPriority()); + + if (log.isDebugEnabled()) { + log.debug("State allocateCountTracker before handling allocated container: " + tracker); + } + Container processed = allocateCountTrackers.get(container.getPriority().getPriority()) + .processAllocatedContainer(container); + if (processed != null) { + preProcessed.add(processed); + } else { + garbageContainers.add(container.getId()); + releaseContainers.add(container.getId()); + } + if (log.isDebugEnabled()) { + log.debug("State allocateCountTracker after handling allocated container: " + tracker); + } + } + return preProcessed; } @Override @@ -327,25 +377,66 @@ public void setLocality(boolean locality) { this.locality = locality; } + /** + * Internal init which should only configure this class and + * should not touch parent classes. + */ + private void internalInit() { + setAllocationValues(null, priority, virtualcores, memory, locality); + for (DefaultAllocateCountTracker tracker : allocateCountTrackers.values()) { + tracker.setConfiguration(getConfiguration()); + } + } + /** * Utility method creating a {@link ResourceRequest}. * * @param numContainers number of containers to request * @return request to be sent to resource manager */ - private ResourceRequest getContainerResourceRequest(int numContainers, String hostName, boolean relaxLocality) { + private ResourceRequest getContainerResourceRequest(String id, int numContainers, String hostName, boolean relaxLocality) { + + ContainerAllocationValues allocationValues = getAllocationValues(id); + ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setRelaxLocality(relaxLocality); request.setResourceName(hostName); request.setNumContainers(numContainers); Priority pri = Records.newRecord(Priority.class); - pri.setPriority(priority); + pri.setPriority(allocationValues.priority); request.setPriority(pri); Resource capability = Records.newRecord(Resource.class); - capability.setMemory(memory); - ResourceCompat.setVirtualCores(capability, virtualcores); + capability.setMemory(allocationValues.memory); + ResourceCompat.setVirtualCores(capability, allocationValues.virtualcores); request.setCapability(capability); return request; } + private static class ContainerAllocationValues { + int priority = 0; + int virtualcores = 1; + int memory = 64; + boolean locality = false; + public ContainerAllocationValues(Integer priority, Integer virtualcores, Integer memory, Boolean locality) { + if (priority != null) { + this.priority = priority; + } + if (virtualcores != null) { + this.virtualcores = virtualcores; + } + if (memory != null) { + this.memory = memory; + } + if (locality != null) { + this.locality = locality; + } + } + @Override + public String toString() { + return "ContainerAllocationValues [priority=" + priority + ", virtualcores=" + virtualcores + ", memory=" + + memory + ", locality=" + locality + "]"; + } + + } + } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/AbstractContainerClusterAppmaster.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/AbstractContainerClusterAppmaster.java new file mode 100644 index 000000000..be3998ca4 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/AbstractContainerClusterAppmaster.java @@ -0,0 +1,434 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.yarn.YarnSystemException; +import org.springframework.yarn.am.AbstractEventingAppmaster; +import org.springframework.yarn.am.AppmasterCmOperations; +import org.springframework.yarn.am.AppmasterCmTemplate; +import org.springframework.yarn.am.ContainerLauncherInterceptor; +import org.springframework.yarn.am.allocate.AbstractAllocator; +import org.springframework.yarn.am.allocate.ContainerAllocateData; +import org.springframework.yarn.am.container.AbstractLauncher; +import org.springframework.yarn.am.grid.Grid; +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.GridProjectionFactory; +import org.springframework.yarn.am.grid.ProjectedGrid; +import org.springframework.yarn.am.grid.listener.ProjectedGridListenerAdapter; +import org.springframework.yarn.am.grid.support.AbstractGridProjection; +import org.springframework.yarn.am.grid.support.DefaultGrid; +import org.springframework.yarn.am.grid.support.DefaultGridMember; +import org.springframework.yarn.am.grid.support.DefaultProjectedGrid; +import org.springframework.yarn.am.grid.support.ProjectionData; +import org.springframework.yarn.am.grid.support.SatisfyStateData; +import org.springframework.yarn.am.monitor.ContainerAware; +import org.springframework.yarn.fs.MultiResourceLocalizer; +import org.springframework.yarn.support.PollingTaskSupport; +import org.springframework.yarn.support.statemachine.StateMachine; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineFactory; +import org.springframework.yarn.support.statemachine.state.State; + +/** + * Base implementation of a {@link ContainerClusterAppmaster}. + * + * @author Janne Valkealahti + * + */ +public abstract class AbstractContainerClusterAppmaster extends AbstractEventingAppmaster implements ContainerClusterAppmaster { + + private static final Log log = LogFactory.getLog(AbstractContainerClusterAppmaster.class); + + /** ClusterId to ContainerCluster map */ + final Map clusters = new HashMap(); + + /** ClusterId to Cluster definition id mapping */ + final Map clusterIdToRef = new HashMap(); + + /** Containers scheduled to be killed */ + private final Queue killQueue = new LinkedList(); + + /** Grid tracking generic grid members */ + private Grid grid; + + /** Projected grid tracking container clusters */ + ProjectedGrid projectedGrid; + + /** Poller for handling periodic tasks */ + private ClusterTaskPoller clusterTaskPoller; + + /** Factory for building on-demand state machines */ + private EnumStateMachineFactory stateMachineFactory; + + /** Factory creating projections */ + private GridProjectionFactory gridProjectionFactory; + + @Override + protected void onInit() throws Exception { + super.onInit(); + grid = doCreateGrid(); + Assert.notNull(grid, "Grid must be set"); + projectedGrid = doCreateProjectedGrid(grid); + Assert.notNull(projectedGrid, "ProjectedGrid must be set"); + projectedGrid.addProjectedGridListener(new CommandDispatchListener()); + + if (getLauncher() instanceof AbstractLauncher) { + ((AbstractLauncher)getLauncher()).addInterceptor(new ContainerLaunchContextModifyInterceptor()); + } + } + + @Override + protected void doStart() { + super.doStart(); + if (gridProjectionFactory != null) { + Map defaults = gridProjectionFactory.getProjectionDatas(); + for (Entry entry : defaults.entrySet()) { + // no type means it's missing projection settings + // so assume it's a blueprint and don't try to create + // cluster automatically + if (StringUtils.hasText(entry.getValue().getType())) { + createContainerCluster(entry.getKey(), entry.getValue()); + startContainerCluster(entry.getKey()); + } + } + } + } + + @Override + protected void doStop() { + if (clusterTaskPoller != null) { + clusterTaskPoller.stop(); + clusterTaskPoller = null; + } + super.doStop(); + } + + @Override + public void submitApplication() { + log.info("Submitting application"); + registerAppmaster(); + start(); + if(getAllocator() instanceof AbstractAllocator) { + ((AbstractAllocator)getAllocator()).setApplicationAttemptId(getApplicationAttemptId()); + } + clusterTaskPoller = new ClusterTaskPoller(getTaskScheduler(), getTaskExecutor()); + clusterTaskPoller.init(); + clusterTaskPoller.start(); + } + + @Override + protected void onContainerAllocated(Container container) { + if (getMonitor() instanceof ContainerAware) { + ((ContainerAware)getMonitor()).onContainer(Arrays.asList(container)); + } + DefaultGridMember member = new DefaultGridMember(container); + if (grid.addMember(member)) { + ContainerCluster cluster = findContainerClusterByContainer(container); + if (cluster != null) { + getLauncher().launchContainer(container, onContainerLaunchCommands(container, cluster, getCommands(clusterIdToRef.get(cluster.getId())))); + } else { + getLauncher().launchContainer(container, getCommands()); + } + } else { + getAllocator().releaseContainers(Arrays.asList(container)); + } + } + + @Override + protected void onContainerLaunched(Container container) { + if (getMonitor() instanceof ContainerAware) { + ((ContainerAware)getMonitor()).onContainer(Arrays.asList(container)); + } + } + + @Override + protected void onContainerCompleted(ContainerStatus status) { + super.onContainerCompleted(status); + + grid.removeMember(status.getContainerId()); + + if (getMonitor() instanceof ContainerAware) { + ((ContainerAware)getMonitor()).onContainerStatus(Arrays.asList(status)); + } + } + + @Override + public Map getContainerClusters() { + return clusters; + } + + @Override + public ContainerCluster createContainerCluster(String clusterId, ProjectionData projectionData) { + return createContainerCluster(clusterId, clusterId, projectionData, null); + } + + @Override + public ContainerCluster createContainerCluster(String clusterId, String clusterDef, ProjectionData projectionData, Map extraProperties) { + + if (clusterDef == null) { + clusterDef = clusterId; + } + + GridProjection projection = gridProjectionFactory.getGridProjection(projectionData); + + ProjectionData p = gridProjectionFactory.getProjectionDatas().get(clusterDef); + + if (projection instanceof AbstractGridProjection) { + ((AbstractGridProjection) projection).setConfiguration(getConfiguration()); + if (p != null && ((AbstractGridProjection) projection).getPriority() == null) { + ((AbstractGridProjection) projection).setPriority(p.getPriority()); + } + } + + StateMachine, ClusterEvent> stateMachine = stateMachineFactory + .getStateMachine(); + DefaultContainerCluster cluster = new DefaultContainerCluster(clusterId, projection, stateMachine, extraProperties); + clusters.put(cluster.getId(), cluster); + projectedGrid.addProjection(cluster.getGridProjection()); + clusterIdToRef.put(clusterId, clusterDef); + return cluster; + } + + @Override + public void startContainerCluster(String id) { + ContainerCluster cluster = clusters.get(id); + if (cluster != null) { + StateMachine, ClusterEvent> stateMachine = cluster.getStateMachine(); + stateMachine.sendEvent(ClusterEvent.START); + stateMachine.sendEvent(MessageBuilder.withPayload(ClusterEvent.CONFIGURE) + .setHeader("containercluster", cluster).setHeader("appmaster", AbstractContainerClusterAppmaster.this).build()); + } + } + + @Override + public void stopContainerCluster(String id) { + ContainerCluster cluster = clusters.get(id); + if (cluster != null) { + StateMachine, ClusterEvent> stateMachine = cluster.getStateMachine(); + stateMachine.sendEvent(MessageBuilder.withPayload(ClusterEvent.STOP).setHeader("containercluster", cluster) + .setHeader("appmaster", AbstractContainerClusterAppmaster.this).build()); + } + } + + @Override + public void destroyContainerCluster(String id) { + ContainerCluster cluster = clusters.get(id); + if (cluster != null) { + StateMachine, ClusterEvent> stateMachine = cluster.getStateMachine(); + stateMachine.sendEvent(MessageBuilder.withPayload(ClusterEvent.DESTROY) + .setHeader("containercluster", cluster) + .setHeader("appmaster", AbstractContainerClusterAppmaster.this).build()); + } + } + + @Override + public void modifyContainerCluster(String id, ProjectionData data) { + ContainerCluster cluster = clusters.get(id); + if (cluster != null) { + StateMachine, ClusterEvent> stateMachine = cluster.getStateMachine(); + stateMachine.sendEvent(MessageBuilder.withPayload(ClusterEvent.CONFIGURE).setHeader("projectiondata", data) + .setHeader("containercluster", cluster).setHeader("appmaster", AbstractContainerClusterAppmaster.this).build()); + } + } + + @Autowired + public void setStateMachineFactory(EnumStateMachineFactory stateMachineFactory) { + this.stateMachineFactory = stateMachineFactory; + } + + /** + * Sets the {@link GridProjectionFactory} used to create an instances + * of {@link GridProjection}s. + * + * @param factory the grid projection factory + */ + @Autowired + public void setGridProjectionFactory(GridProjectionFactory gridProjectionFactory) { + log.info("Setting gridProjectionFactory=" + gridProjectionFactory); + this.gridProjectionFactory = gridProjectionFactory; + } + + protected Grid doCreateGrid() { + return new DefaultGrid(); + } + + protected ProjectedGrid doCreateProjectedGrid(Grid grid) { + return new DefaultProjectedGrid(grid); + } + + protected void handleSatisfyStateData(ContainerCluster cluster, SatisfyStateData satisfyData) { + if (satisfyData.getAllocateData() != null) { + // who set the allocator settings matching id from allocate data??? + // if not configures, fall back to defaults??? + ContainerAllocateData allocateData = satisfyData.getAllocateData(); + // should not set id here!!! + allocateData.setId(clusterIdToRef.get(cluster.getId())); + getAllocator().allocateContainers(allocateData); + } + for (GridMember member : satisfyData.getRemoveData()) { + log.info("Queued container to be killed: " + member.getContainer().getId()); + killContainer(member.getContainer()); + } + } + + protected void killContainer(Container container) { + killQueue.add(container); + } + + protected AppmasterCmOperations getCmTemplate(Container container) { + try { + AppmasterCmTemplate template = new AppmasterCmTemplate(getConfiguration(), container); + template.afterPropertiesSet(); + return template; + } catch (Exception e) { + throw new YarnSystemException("Unable to create AppmasterCmTemplate", e); + } + } + + /** + * Called when a container is launched for sub classes to do + * a final modifications in these commands. Default implementation + * returns commands as is. + * + * @param cluster the cluster id + * @param commands the original commands + * @return modified commands + */ + protected List onContainerLaunchCommands(Container container, ContainerCluster cluster, List commands) { + return commands; + } + + /** + * Periodic task callback called by a {@link ClusterTaskPoller}. + */ + private void doTask() { + // kill containers not needed + handleKillQueue(); + } + + /** + * Kill all containers from queue. + */ + private void handleKillQueue() { + Container toKill = null; + while ((toKill = killQueue.poll()) != null) { + log.info("Killing container: " + toKill); + getCmTemplate(toKill).stopContainers(); + } + } + + private ContainerCluster findContainerClusterByContainer(Container container) { + // TODO: make finding cluster more clever + for (Entry entry : clusters.entrySet()) { + for (GridMember member : entry.getValue().getGridProjection().getMembers()) { + if (member.getContainer().equals(container)) { + return entry.getValue(); + } + } + } + return null; + } + + /** + * Builds current satisfy state data for all clusters. This data + * contains information what kind of containers clusters will need + * to satisfy its target state. Additionally data may also contain + * containers which it doesn't need which should be killed. + * + * @return the satisfy state data for clusters + */ + + /** + * Internal poller handling cluster allocation request scheduling. + */ + private class ClusterTaskPoller extends PollingTaskSupport { + + public ClusterTaskPoller(TaskScheduler taskScheduler, TaskExecutor taskExecutor) { + super(taskScheduler, taskExecutor); + } + + @Override + protected Void doPoll() { + doTask(); + return null; + } + + } + + private class CommandDispatchListener extends ProjectedGridListenerAdapter { + + @Override + public void memberRemoved(GridProjection projection, GridMember member) { + log.info("memberRemoved projection=" + projection + " member=" + member); + + for (ContainerCluster cluster : clusters.values()) { + if (cluster.getGridProjection().equals(projection)) { + StateMachine, ClusterEvent> stateMachine = cluster.getStateMachine(); + stateMachine.sendEvent(MessageBuilder.withPayload(ClusterEvent.CONFIGURE) + .setHeader("containercluster", cluster).setHeader("appmaster", AbstractContainerClusterAppmaster.this).build()); + } + } + + } + } + + private class ContainerLaunchContextModifyInterceptor implements ContainerLauncherInterceptor { + + @Override + public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) { + ContainerCluster cluster = findContainerClusterByContainer(container); + if (getResourceLocalizer() instanceof MultiResourceLocalizer) { + if (cluster != null) { + MultiResourceLocalizer loc = (MultiResourceLocalizer) getResourceLocalizer(); + Map resources = loc.getResources(clusterIdToRef.get(cluster.getId())); + context.setLocalResources(resources); + } + } else { + log.warn("Can't use container specific local resources because MultiResourceLocalizer expected instead of " + + getResourceLocalizer()); + } + + if (cluster != null) { + Map environment = new HashMap(context.getEnvironment()); + environment.putAll(getEnvironment(clusterIdToRef.get(cluster.getId()))); + context.setEnvironment(environment); + } + + return context; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterAllocatingAction.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterAllocatingAction.java new file mode 100644 index 000000000..89bd58662 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterAllocatingAction.java @@ -0,0 +1,45 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.support.ProjectionData; +import org.springframework.yarn.am.grid.support.SatisfyStateData; +import org.springframework.yarn.support.statemachine.action.Action; + +public class ClusterAllocatingAction implements Action { + + @Override + public void execute(MessageHeaders headers) { + ContainerCluster cluster = headers.get("containercluster", ContainerCluster.class); + ProjectionData projectionData = headers.get("projectiondata", ProjectionData.class); + AbstractContainerClusterAppmaster appmaster = headers.get("appmaster", AbstractContainerClusterAppmaster.class); + + if(projectionData != null) { + GridProjection gridProjection = cluster.getGridProjection(); + gridProjection.setProjectionData(projectionData); + } + + if (cluster != null) { + SatisfyStateData satisfyData = cluster.getGridProjection().getSatisfyState(); + if (satisfyData != null) { + appmaster.handleSatisfyStateData(cluster, satisfyData); + } + } + + } +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterDestroyingAction.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterDestroyingAction.java new file mode 100644 index 000000000..3fe25e5b4 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterDestroyingAction.java @@ -0,0 +1,31 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.yarn.support.statemachine.action.Action; + +public class ClusterDestroyingAction implements Action { + + @Override + public void execute(MessageHeaders headers) { + ContainerCluster cluster = headers.get("containercluster", ContainerCluster.class); + AbstractContainerClusterAppmaster appmaster = headers.get("appmaster", AbstractContainerClusterAppmaster.class); + ContainerCluster removed = appmaster.clusters.remove(cluster.getId()); + appmaster.projectedGrid.removeProjection(removed.getGridProjection()); + } + +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterEvent.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterEvent.java new file mode 100644 index 000000000..4a557439c --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterEvent.java @@ -0,0 +1,35 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import org.springframework.yarn.support.statemachine.state.State; + +/** + * Possible events controlling this machine. + */ +public enum ClusterEvent { + + /** + * Command requesting {@link State#RUNNING} state. + */ + START, + + CONFIGURE, + CONTINUE, + STOP, + END, + DESTROY +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterState.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterState.java new file mode 100644 index 000000000..71aafb3ef --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterState.java @@ -0,0 +1,69 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +/** + * Possible states of this machine. + */ +public enum ClusterState { + + /** + * Initial state when cluster has been created. Once we enter away from + * this state, we never get back from any other state. + */ + INITIAL, + + /** + * State where cluster is ready to go allocation mode or to be + * stopped or destroyed. + */ + RUNNING, + + /** + * State where cluster is allocating and configuring itself. + */ + ALLOCATING, + + /** + * State where cluster has been requested to be stopped. + */ + STOPPING, + + /** + * State where cluster is stopped. Cluster can either be started + * or destroyed. + */ + STOPPED, + + /** + * State when cluster is entering into a mode to be destroyed. This is + * similar when cluster is stopped but instead ending into a stopped state, + * we end into a destroyed state. + */ + DESTROYING, + + /** + * State for cluster when it has been destroyed and no further + * state changes are allowed except transition to final state. + */ + DESTROYED, + + /** + * Final state for cluster. Cluster object can be safely removed + * and possibly recreated. + */ + FINAL +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterStoppingAction.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterStoppingAction.java new file mode 100644 index 000000000..1327cf7b0 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ClusterStoppingAction.java @@ -0,0 +1,37 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import java.util.ArrayList; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.support.SatisfyStateData; +import org.springframework.yarn.support.statemachine.action.Action; + +public class ClusterStoppingAction implements Action { + + @Override + public void execute(MessageHeaders headers) { + ContainerCluster cluster = headers.get("containercluster", ContainerCluster.class); + AbstractContainerClusterAppmaster appmaster = headers.get("appmaster", AbstractContainerClusterAppmaster.class); + SatisfyStateData satisfyData = new SatisfyStateData(new ArrayList(cluster.getGridProjection().getMembers())); + appmaster.handleSatisfyStateData(cluster, satisfyData); +// ContainerCluster removed = appmaster.clusters.remove(cluster.getId()); +// appmaster.projectedGrid.removeProjection(removed.getGridProjection()); + } + +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerCluster.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerCluster.java new file mode 100644 index 000000000..974bd5cbb --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerCluster.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import java.util.Map; + +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.support.statemachine.StateMachine; +import org.springframework.yarn.support.statemachine.state.State; + +public interface ContainerCluster { + + String getId(); + + GridProjection getGridProjection(); + + StateMachine, ClusterEvent> getStateMachine(); + + Map getExtraProperties(); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterAppmaster.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterAppmaster.java new file mode 100644 index 000000000..d72c8ea88 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterAppmaster.java @@ -0,0 +1,46 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import java.util.Map; + +import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.grid.support.ProjectionData; + +/** + * Interface for {@link YarnAppmaster} which is controlling + * container clusters. + * + * @author Janne Valkealahti + * + */ +public interface ContainerClusterAppmaster extends YarnAppmaster { + + Map getContainerClusters(); + + ContainerCluster createContainerCluster(String clusterId, ProjectionData projection); + + ContainerCluster createContainerCluster(String clusterId, String clusterDef, ProjectionData projection, Map extraProperties); + + void startContainerCluster(String id); + + void stopContainerCluster(String id); + + void destroyContainerCluster(String id); + + void modifyContainerCluster(String id, ProjectionData data); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterInfo.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterInfo.java new file mode 100644 index 000000000..d5179e807 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterInfo.java @@ -0,0 +1,38 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import org.springframework.yarn.am.grid.GridProjection; + +public class ContainerClusterInfo { + + private String clusterId; + private GridProjection projection; + + public ContainerClusterInfo(String clusterId, GridProjection projection) { + this.clusterId = clusterId; + this.projection = projection; + } + + public String getClusterId() { + return clusterId; + } + + public GridProjection getProjection() { + return projection; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterStateMachineConfiguration.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterStateMachineConfiguration.java new file mode 100644 index 000000000..2aa65be93 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ContainerClusterStateMachineConfiguration.java @@ -0,0 +1,115 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.yarn.support.statemachine.config.EnableStateMachineFactory; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +/** + * Configuration for state machine build used by {@link AbstractContainerClusterAppmaster}. + * + * @author Janne Valkealahti + * + */ +@Configuration +@EnableStateMachineFactory +public class ContainerClusterStateMachineConfiguration extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(ClusterState.INITIAL) + .state(ClusterState.INITIAL) + .state(ClusterState.RUNNING) + .state(ClusterState.ALLOCATING, ClusterEvent.CONFIGURE) + .state(ClusterState.STOPPING) + .state(ClusterState.STOPPED) + .state(ClusterState.DESTROYING) + .state(ClusterState.DESTROYED) + .state(ClusterState.FINAL); + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + transitions + .withExternal() + .source(ClusterState.INITIAL) + .target(ClusterState.RUNNING) + .event(ClusterEvent.START) + .and() + .withExternal() + .source(ClusterState.RUNNING) + .target(ClusterState.ALLOCATING) + .event(ClusterEvent.CONFIGURE) + .action(clusterAllocatingAction()) + .and() + .withExternal() + .source(ClusterState.STOPPED) + .target(ClusterState.RUNNING) + .event(ClusterEvent.START) + .and() + .withExternal() + .source(ClusterState.ALLOCATING) + .target(ClusterState.RUNNING) + .and() + .withExternal() + .source(ClusterState.RUNNING) + .target(ClusterState.STOPPING) + .event(ClusterEvent.STOP) + .action(clusterStoppingAction()) + .and() + .withExternal() + .source(ClusterState.STOPPING) + .target(ClusterState.STOPPED) + .and() + .withExternal() + .source(ClusterState.STOPPED) + .target(ClusterState.DESTROYING) + .event(ClusterEvent.DESTROY) + .action(clusterDestroyingAction()) + .and() + .withExternal() + .source(ClusterState.INITIAL) + .target(ClusterState.DESTROYING) + .event(ClusterEvent.DESTROY) + .action(clusterDestroyingAction()) + .and() + .withExternal() + .source(ClusterState.DESTROYING) + .target(ClusterState.DESTROYED); + } + + @Bean + public ClusterAllocatingAction clusterAllocatingAction() { + return new ClusterAllocatingAction(); + } + + @Bean + public ClusterStoppingAction clusterStoppingAction() { + return new ClusterStoppingAction(); + } + + @Bean + public ClusterDestroyingAction clusterDestroyingAction() { + return new ClusterDestroyingAction(); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/DefaultContainerCluster.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/DefaultContainerCluster.java new file mode 100644 index 000000000..d727403a2 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/DefaultContainerCluster.java @@ -0,0 +1,65 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import java.util.Map; + +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.support.statemachine.StateMachine; +import org.springframework.yarn.support.statemachine.state.State; + +public class DefaultContainerCluster implements ContainerCluster { + + private final String id; + + private final GridProjection projection; + + private StateMachine, ClusterEvent> stateMachine; + + private Map extraProperties; + + public DefaultContainerCluster(String id, GridProjection projection, StateMachine, ClusterEvent> stateMachine) { + this(id, projection, stateMachine, null); + } + + public DefaultContainerCluster(String id, GridProjection projection, StateMachine, ClusterEvent> stateMachine, Map extraProperties) { + this.id = id; + this.projection = projection; + this.stateMachine = stateMachine; + this.extraProperties = extraProperties; + } + + @Override + public String getId() { + return id; + } + + @Override + public GridProjection getGridProjection() { + return projection; + } + + @Override + public StateMachine, ClusterEvent> getStateMachine() { + return stateMachine; + } + + @Override + public Map getExtraProperties() { + return extraProperties; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmaster.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmaster.java new file mode 100644 index 000000000..f9e9c1cac --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmaster.java @@ -0,0 +1,27 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +/** + * Default {@link ContainerClusterAppmaster} implementation using functionality + * from {@link AbstractContainerClusterAppmaster}. + * + * @author Janne Valkealahti + * + */ +public class ManagedContainerClusterAppmaster extends AbstractContainerClusterAppmaster { + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/Grid.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/Grid.java new file mode 100644 index 000000000..b5d83bb9c --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/Grid.java @@ -0,0 +1,87 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid; + +import java.util.Collection; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.springframework.yarn.am.grid.listener.GridListener; +import org.springframework.yarn.am.grid.support.GridMemberInterceptor; + +/** + * Grid allows to track members in a grid. + * + * @author Janne Valkealahti + * + */ +public interface Grid { + + /** + * Gets collection of grid members know to the grid system or + * empty collection if there are no known members. + * + * @return Collection of grid members + */ + Collection getMembers(); + + /** + * Gets a grid member. + * + * @param id the container id identifier + * @return Grid member or NULL if member doesn't exist + */ + GridMember getMember(ContainerId id); + + /** + * Adds a new grid member. + *

+ * If a grid refuses to add a particular member for any reason + * other than that it already contains the member, it must throw + * an exception (rather than returning false). This preserves + * the invariant that a grid always contains the specified node + * after this call returns. + * + * @param member the grid member + * @return true if this grid changed as a result of the call + */ + boolean addMember(GridMember member); + + /** + * Removes a grid member. + *

+ * Removes a single instance of the specified member from this + * grid, if it is present. + * + * @param id the container id identifier + * @return true if a member was removed as a result of this call + */ + boolean removeMember(ContainerId id); + + /** + * Adds a listener to be notified of grid members events. + * + * @param listener the grid listener + */ + void addGridListener(GridListener listener); + + /** + * Adds a grid member interceptor. + * + * @param interceptor the interceptor + */ + void addInterceptor(GridMemberInterceptor interceptor); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridMember.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridMember.java new file mode 100644 index 000000000..a44c3a7a2 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridMember.java @@ -0,0 +1,43 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; + +/** + * Interface representing a grid member. + * + * @author Janne Valkealahti + * + */ +public interface GridMember { + + /** + * Gets a container member identifier. + * + * @return the container member identifier + */ + ContainerId getId(); + + /** + * Gets the container. + * + * @return the container + */ + Container getContainer(); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjection.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjection.java new file mode 100644 index 000000000..6a7edc5e2 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjection.java @@ -0,0 +1,69 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid; + +import java.util.Collection; + +import org.springframework.yarn.am.grid.support.ProjectionData; +import org.springframework.yarn.am.grid.support.SatisfyStateData; + +/** + * Interface representing a grid projection. + * + * @author Janne Valkealahti + * + */ +public interface GridProjection { + + boolean acceptMember(GridMember member); + + /** + * Removes the member. + * + * @param member the member + * @return the grid member + */ + GridMember removeMember(GridMember member); + + /** + * Gets the members of this projection as {@code Collection}. + * + * @return the projection members + */ + Collection getMembers(); + + /** + * Gets the satisfy state. + * + * @return the satisfy state + */ + SatisfyStateData getSatisfyState(); + + /** + * Sets the projection data. + * + * @param data the new projection data + */ + void setProjectionData(ProjectionData data); + + /** + * Gets the projection data. + * + * @return the projection data + */ + ProjectionData getProjectionData(); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjectionFactory.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjectionFactory.java new file mode 100644 index 000000000..9722368a1 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/GridProjectionFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid; + +import java.util.Map; + +import org.springframework.yarn.am.grid.support.ProjectionData; + +/** + * A {@link GridProjectionFactory} is used from an application master + * responsible handling creation of {@link GridProjection}s. + * + * @author Janne Valkealahti + * + */ +public interface GridProjectionFactory { + + /** + * Gets all projection datas mapped with an identifier known + * to this factory. User of this factory can then use this + * information to create a default {@link GridProjection}s + * at startup. + * + * @return the mapping from identifier to projection data + */ + Map getProjectionDatas(); + + /** + * Builds a {@link GridProjection} using a {@link ProjectionData}. + * + * @param projectionData the projection data + * @return the grid projection + */ + GridProjection getGridProjection(ProjectionData projectionData); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/ProjectedGrid.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/ProjectedGrid.java new file mode 100644 index 000000000..52b9f6469 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/ProjectedGrid.java @@ -0,0 +1,55 @@ +package org.springframework.yarn.am.grid; + +import java.util.Collection; + +import org.springframework.yarn.am.grid.listener.ProjectedGridListener; + +/** + * ProjectedGrid + * + * @author Janne Valkealahti + * @see Grid + */ +public interface ProjectedGrid { + + /** + * Adds a new grid projection. + *

+ * If a projected grid refuses to add a particular grid projection for any reason + * other than that it already contains the grid projection, it must throw + * an exception (rather than returning false). This preserves + * the invariant that a projected grid always contains the specified grid projection + * after this call returns. + * + * @param projection the grid projection + * @return true if this projected grid changed as a result of the call + */ + boolean addProjection(GridProjection projection); + + /** + * Removes a grid projection. + *

+ * Removes a single instance of the specified grid projection from this + * projected grid, if it is present. + * + * @param projection the grid projection + * @return true if a grid projection was removed as a result of this call + */ + boolean removeProjection(GridProjection projection); + + /** + * Gets a collection of grid projections or empty collection of there + * are no projections defined. + * + * @return the collection of grid projections + */ + Collection getProjections(); + + /** + * Adds a listener to be notified of projected grid events. + * + * @param listener the projected grid listener + */ + void addProjectedGridListener(ProjectedGridListener listener); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultGridListener.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultGridListener.java new file mode 100644 index 000000000..4c7b955a0 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultGridListener.java @@ -0,0 +1,46 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.listener; + +import java.util.Iterator; + +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.listener.AbstractCompositeListener; + +/** + * Composite listener for handling grid member events. + * + * @author Janne Valkealahti + * + */ +public class DefaultGridListener extends + AbstractCompositeListener implements GridListener { + + @Override + public void memberAdded(GridMember node) { + for (Iterator iterator = getListeners().reverse(); iterator.hasNext();) { + iterator.next().memberAdded(node); + } + } + + @Override + public void memberRemoved(GridMember node) { + for (Iterator iterator = getListeners().reverse(); iterator.hasNext();) { + iterator.next().memberRemoved(node); + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultProjectedGridListener.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultProjectedGridListener.java new file mode 100644 index 000000000..1abf6ea52 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/DefaultProjectedGridListener.java @@ -0,0 +1,61 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.listener; + +import java.util.Iterator; + +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.listener.AbstractCompositeListener; + +/** + * Composite listener for handling container group events. + * + * @author Janne Valkealahti + * + */ +public class DefaultProjectedGridListener extends AbstractCompositeListener + implements ProjectedGridListener { + + @Override + public void projectionAdded(GridProjection group) { + for (Iterator iterator = getListeners().reverse(); iterator.hasNext();) { + iterator.next().projectionAdded(group); + } + } + + @Override + public void projectionRemoved(GridProjection group) { + for (Iterator iterator = getListeners().reverse(); iterator.hasNext();) { + iterator.next().projectionRemoved(group); + } + } + + @Override + public void memberAdded(GridProjection group, GridMember node) { + for (Iterator iterator = getListeners().reverse(); iterator.hasNext();) { + iterator.next().memberAdded(group, node); + } + } + + @Override + public void memberRemoved(GridProjection group, GridMember node) { + for (Iterator iterator = getListeners().reverse(); iterator.hasNext();) { + iterator.next().memberRemoved(group, node); + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/GridListener.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/GridListener.java new file mode 100644 index 000000000..7728f413c --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/GridListener.java @@ -0,0 +1,42 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.listener; + +import org.springframework.yarn.am.grid.GridMember; + +/** + * Listener for grid container node events. + * + * @author Janne Valkealahti + * + */ +public interface GridListener { + + /** + * Invoked when member is added into a grid. + * + * @param member the {@link GridMember} + */ + void memberAdded(GridMember member); + + /** + * Invoked when member is removed from a grid. + * + * @param member the {@link GridMember} + */ + void memberRemoved(GridMember member); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListener.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListener.java new file mode 100644 index 000000000..97543fff4 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListener.java @@ -0,0 +1,59 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.listener; + +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; + +/** + * Listener for grid container group events. + * + * @author Janne Valkealahti + * + */ +public interface ProjectedGridListener { + + /** + * Invoked when a new projection is added. + * + * @param projection the {@link GridProjection} + */ + void projectionAdded(GridProjection projection); + + /** + * Invoked when projection is removed. + * + * @param projection the {@link GridProjection} + */ + void projectionRemoved(GridProjection projection); + + /** + * Invoked when a member is added into a projection. + * + * @param projection the {@link GridProjection} a member belongs to + * @param member the {@link GridMember} + */ + void memberAdded(GridProjection projection, GridMember member); + + /** + * Invoked when a member is removed from a projection. + * + * @param projection the {@link GridProjection} a member belongs to + * @param member the {@link GridMember} + */ + void memberRemoved(GridProjection projection, GridMember member); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListenerAdapter.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListenerAdapter.java new file mode 100644 index 000000000..1420667cc --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/listener/ProjectedGridListenerAdapter.java @@ -0,0 +1,39 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.listener; + +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; + +public class ProjectedGridListenerAdapter implements ProjectedGridListener { + + @Override + public void projectionAdded(GridProjection projection) { + } + + @Override + public void projectionRemoved(GridProjection projection) { + } + + @Override + public void memberAdded(GridProjection projection, GridMember member) { + } + + @Override + public void memberRemoved(GridProjection projection, GridMember member) { + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGrid.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGrid.java new file mode 100644 index 000000000..0cdaf1d5f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGrid.java @@ -0,0 +1,134 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.springframework.util.Assert; +import org.springframework.yarn.am.grid.Grid; +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.listener.DefaultGridListener; +import org.springframework.yarn.am.grid.listener.GridListener; + +/** + * Simple {@code Grid} base implementation. + * + * @author Janne Valkealahti + * + */ +public abstract class AbstractGrid implements Grid { + + /** Listener dispatcher for grid events */ + private final DefaultGridListener gridListeners = new DefaultGridListener(); + + private final GridMemberInterceptorChain interceptorChain = new GridMemberInterceptorChain(); + + private ConcurrentHashMap members = new ConcurrentHashMap(); + + @Override + public Collection getMembers() { + return members.values(); + } + + @Override + public GridMember getMember(ContainerId id) { + return members.get(id); + } + + @Override + public boolean addMember(GridMember node) { + Assert.notNull(node, "Node must not be null"); + + node = interceptorChain.preAdd(node, this); + + if (node != null && members.putIfAbsent(node.getId(), node) == null) { + notifyMemberAdded(node); + return true; + } else { + return false; + } + } + + @Override + public boolean removeMember(ContainerId id) { + Assert.notNull(id, "Node identifier must not be null"); + GridMember removed = members.remove(id); + if (removed != null) { + notifyMemberRemoved(removed); + return true; + } else { + return false; + } + } + + @Override + public void addGridListener(GridListener listener) { + gridListeners.register(listener); + } + + /** + * Set the list of channel interceptors. This will clear any existing interceptors. + */ + public void setInterceptors(List interceptors) { + this.interceptorChain.set(interceptors); + } + + /** + * Add a channel interceptor to the end of the list. + */ + @Override + public void addInterceptor(GridMemberInterceptor interceptor) { + this.interceptorChain.add(interceptor); + } + + /** + * Return a read-only list of the configured interceptors. + */ + public List getInterceptors() { + return this.interceptorChain.getInterceptors(); + } + + /** + * Exposes the interceptor list for subclasses. + */ + protected GridMemberInterceptorChain getInterceptorChain() { + return this.interceptorChain; + } + + /** + * Notifies registered {@code ContainerGridListener}s that + * a {@code ContainerNode} has been added to a {@code ContainerGrid}. + * + * @param member the node + */ + protected void notifyMemberAdded(GridMember member) { + gridListeners.memberAdded(member); + } + + /** + * Notifies registered {@code ContainerGridListener}s that + * a {@code ContainerNode} has been removed from a {@code ContainerGrid}. + * + * @param member the node + */ + protected void notifyMemberRemoved(GridMember member) { + gridListeners.memberRemoved(member); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridMember.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridMember.java new file mode 100644 index 000000000..cba591acf --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridMember.java @@ -0,0 +1,40 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.springframework.yarn.am.grid.GridMember; + +public abstract class AbstractGridMember implements GridMember { + + private Container container; + + public AbstractGridMember(Container container) { + this.container = container; + } + + @Override + public ContainerId getId() { + return container.getId(); + } + + @Override + public Container getContainer() { + return container; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridProjection.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridProjection.java new file mode 100644 index 000000000..b9f3298ae --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractGridProjection.java @@ -0,0 +1,235 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.util.RackResolver; +import org.springframework.util.Assert; +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; + +/** + * Base implementation of a {@link GridProjection}. + * + * @author Janne Valkealahti + * + */ +public abstract class AbstractGridProjection implements GridProjection { + + private static final Log log = LogFactory.getLog(AbstractGridProjection.class); + + /** Member mapping from container id to grid member */ + private final ConcurrentHashMap members = new ConcurrentHashMap(); + + /** Tracking counts of mapped hosts */ + private final ConcurrentHashMap hostCounts = new ConcurrentHashMap(); + + /** Tracking counts of mapped racks */ + private final ConcurrentHashMap rackCounts = new ConcurrentHashMap(); + + /** Projection data which is a base for calculating a satisfy state for allocation */ + private ProjectionData projectionData; + + /** Hadoop configuration needed for hadoop's rack resolver */ + private Configuration configuration; + + private Integer priority; + + /** + * Instantiates a new abstract grid projection. + */ + public AbstractGridProjection() { + } + + public AbstractGridProjection(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public Collection getMembers() { + return members.values(); + } + + @Override + public void setProjectionData(ProjectionData data) { + projectionData = data; + } + + @Override + public ProjectionData getProjectionData() { + return projectionData; + } + + @Override + public abstract boolean acceptMember(GridMember member); + + @Override + public abstract SatisfyStateData getSatisfyState(); + + public void setPriority(Integer priority) { + this.priority = priority; + } + + public Integer getPriority() { + return priority; + } + + protected boolean addMember(GridMember member) { + Assert.notNull(member, "Node must not be null"); + if (members.putIfAbsent(member.getId(), member) == null) { + incrementHostCount(member); + incrementRackCount(member); + return true; + } else { + return false; + } + } + + @Override + public GridMember removeMember(GridMember member) { + GridMember removed = members.remove(member.getContainer().getId()); + if (removed != null) { + decrementHostCount(removed); + decrementRackCount(removed); + } + return removed; + } + + protected int getHostCount(String host) { + HostCountHolder holder = hostCounts.get(host); + return holder != null ? holder.count : 0; + } + + protected int getRackCount(String rack) { + HostCountHolder holder = rackCounts.get(rack); + return holder != null ? holder.count : 0; + } + + protected Collection getHostCountMembers(String host) { + HostCountHolder holder = hostCounts.get(host); + return holder != null ? holder.members : Collections.emptyList(); + } + + protected Collection getRackCountMembers(String host) { + HostCountHolder holder = rackCounts.get(host); + return holder != null ? holder.members : Collections.emptyList(); + } + + protected Set getHostCountHosts() { + return hostCounts.keySet(); + } + + protected Set getRackCountHosts() { + return rackCounts.keySet(); + } + + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + public Configuration getConfiguration() { + return configuration; + } + + protected boolean isSamePriority(GridMember member) { + Priority pri = member.getContainer().getPriority(); + if (pri != null && getPriority() != null) { + return pri.getPriority() == getPriority().intValue(); + } + return false; + } + + private void incrementHostCount(GridMember member) { + String host = null; + if (member.getContainer().getNodeId() != null) { + host = member.getContainer().getNodeId().getHost(); + } + if (host != null) { + if (!hostCounts.containsKey(host)) { + hostCounts.put(host, new HostCountHolder()); + } + hostCounts.get(host).add(member); + } + } + + private void incrementRackCount(GridMember member) { + String rack = resolveRack(member); + if (rack != null) { + if (!rackCounts.containsKey(rack)) { + rackCounts.put(rack, new HostCountHolder()); + } + rackCounts.get(rack).add(member); + } + } + + private void decrementRackCount(GridMember member) { + String rack = resolveRack(member); + if (rack != null) { + rackCounts.get(rack).remove(member); + } + } + + private void decrementHostCount(GridMember member) { + String host = null; + if (member.getContainer().getNodeId() != null) { + host = member.getContainer().getNodeId().getHost(); + } + if (host != null) { + hostCounts.get(host).remove(member); + } + } + + private String resolveRack(GridMember member) { + if (getConfiguration() != null) { + String host = member.getContainer().getNodeId().getHost(); + String rack = RackResolver.resolve(getConfiguration(), host).getNetworkLocation(); + if (rack == null) { + log.warn("Failed to resolve rack for node " + host + "."); + } else { + log.info("Resolve rack for node " + host + " into " + rack); + } + return rack; + } else { + return null; + } + } + + private static class HostCountHolder { + Integer count = 0; + Collection members = new HashSet(); + void add(GridMember member) { + if (members.add(member)) { + count++; + } + } + void remove(GridMember member) { + if (members.remove(member)) { + count--; + } + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractProjectedGrid.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractProjectedGrid.java new file mode 100644 index 000000000..c198a86b9 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AbstractProjectedGrid.java @@ -0,0 +1,159 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.Collection; +import java.util.HashSet; + +import org.springframework.util.Assert; +import org.springframework.yarn.am.grid.Grid; +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.ProjectedGrid; +import org.springframework.yarn.am.grid.listener.DefaultProjectedGridListener; +import org.springframework.yarn.am.grid.listener.GridListener; +import org.springframework.yarn.am.grid.listener.ProjectedGridListener; + +/** + * Simple {@code ProjectedGrid} base implementation. + * + * @author Janne Valkealahti + * + */ +public abstract class AbstractProjectedGrid implements ProjectedGrid { + + private final Collection projections = new HashSet(); + + /** Listener dispatcher for projected grid events */ + private final DefaultProjectedGridListener listeners = new DefaultProjectedGridListener(); + + private final Grid grid; + + /** + * Instantiates a new abstract projected grid. + * + * @param grid the grid + */ + public AbstractProjectedGrid(Grid grid) { + Assert.notNull(grid, "Grid must not be null"); + this.grid = grid; + this.grid.addInterceptor(new ProjectionAcceptInterceptor()); + this.grid.addGridListener(new ProjectionHandlingGridListener()); + } + + @Override + public boolean addProjection(GridProjection projection) { + Assert.notNull(projection, "Projection must not be null"); + return projections.add(projection); + } + + @Override + public boolean removeProjection(GridProjection projection) { + Assert.notNull(projection, "Projection must not be null"); + boolean removed = projections.remove(projection); + if (removed) { + notifyGridProjectionRemoved(projection); + } + return removed; + } + + @Override + public Collection getProjections() { + return projections; + } + + @Override + public void addProjectedGridListener(ProjectedGridListener listener) { + listeners.register(listener); + } + + /** + * Notifies registered {@code ProjectedGridListener}s that + * a {@code GridProjection} has been added to a {@code ProjectedGrid}. + * + * @param projection the grid projection + */ + protected void notifyGridProjectionAdded(GridProjection projection) { + listeners.projectionAdded(projection); + } + + /** + * Notifies registered {@code ProjectedGridListener}s that + * a {@code GridProjection} has been removed from a {@code ProjectedGrid}. + * + * @param projection the grid projection + */ + protected void notifyGridProjectionRemoved(GridProjection projection) { + listeners.projectionRemoved(projection); + } + + /** + * Notifies registered {@code ContainerGridGroupsListener}s that + * a {@code ContainerNode} has been added to a {@code ContainerGroup}. + * + * @param projection the grid projection + * @param member the grid member + */ + protected void notifyMemberAdded(GridProjection projection, GridMember member) { + listeners.memberAdded(projection, member); + } + + /** + * Notifies registered {@code ContainerGridGroupsListener}s that + * a {@code ContainerNode} has been removed from a {@code ContainerGroup}. + * + * @param projection the group + * @param member the node + */ + protected void notifyMemberRemoved(GridProjection projection, GridMember member) { + listeners.memberRemoved(projection, member); + } + + private class ProjectionAcceptInterceptor implements GridMemberInterceptor { + + @Override + public GridMember preAdd(GridMember member, Grid grid) { + // intercept and check if any projection + // accepts this member + for (GridProjection projection : getProjections()) { + if (projection.acceptMember(member)) { + notifyMemberAdded(projection, member); + return member; + } + } + return null; + } + + } + + private class ProjectionHandlingGridListener implements GridListener { + + @Override + public void memberAdded(GridMember member) { + } + + @Override + public void memberRemoved(GridMember member) { + for (GridProjection projection : projections) { + if (projection.removeMember(member) != null) { + notifyMemberRemoved(projection, member); + } + } + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AnyGridProjection.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AnyGridProjection.java new file mode 100644 index 000000000..d8a1bd221 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/AnyGridProjection.java @@ -0,0 +1,82 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.ArrayList; +import java.util.Iterator; + +import org.springframework.yarn.am.grid.GridMember; +import org.springframework.yarn.am.grid.GridProjection; + + +/** + * {@link GridProjection} which accepts any node for its members. + * + * @author Janne Valkealahti + * + */ +public class AnyGridProjection extends AbstractGridProjection { + + private int count; + + public AnyGridProjection() { + super(); + } + + public AnyGridProjection(int count) { + super(); + this.count = count; + } + + @Override + public SatisfyStateData getSatisfyState() { + SatisfyStateData data = new SatisfyStateData(); + + // simply add delta for current size vs. requested count + int delta = count - getMembers().size(); + data.getAllocateData().addAny(Math.max(delta, 0)); + + // Simply remove using negative delta + int removeCount = Math.max(-delta, 0); + Iterator iterator = getMembers().iterator(); + ArrayList remove = new ArrayList(); + while (iterator.hasNext() && removeCount-- > 0) { + remove.add(iterator.next()); + } + data.setRemoveData(remove); + + return data; + } + + @Override + public void setProjectionData(ProjectionData data) { + super.setProjectionData(data); + count = data.getAny() != null ? data.getAny() : 0; + } + + @Override + public boolean acceptMember(GridMember member) { + if (!isSamePriority(member)) { + return false; + } + if (getMembers().size() < count) { + return addMember(member); + } else { + return false; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGrid.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGrid.java new file mode 100644 index 000000000..447ac4f08 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGrid.java @@ -0,0 +1,20 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +public class DefaultGrid extends AbstractGrid { + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridMember.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridMember.java new file mode 100644 index 000000000..213488132 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridMember.java @@ -0,0 +1,27 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import org.apache.hadoop.yarn.api.records.Container; + +public class DefaultGridMember extends AbstractGridMember { + + public DefaultGridMember(Container container) { + super(container); + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridProjectionFactory.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridProjectionFactory.java new file mode 100644 index 000000000..524f0a64d --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultGridProjectionFactory.java @@ -0,0 +1,67 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.GridProjectionFactory; + +/** + * A default implementation of a {@link GridProjectionFactory}. + * + * @author Janne Valkealahti + * + */ +public class DefaultGridProjectionFactory implements GridProjectionFactory { + + private final Map defaults = new HashMap(); + + public DefaultGridProjectionFactory(Map defaults) { + if (defaults != null) { + this.defaults.putAll(defaults); + } + } + + @Override + public Map getProjectionDatas() { + return defaults; + } + + @Override + public GridProjection getGridProjection(ProjectionData projectionData) { + GridProjection projection = null; + if ("hosts".equalsIgnoreCase(projectionData.getType())) { + HostsGridProjection p = new HostsGridProjection(); + p.setPriority(projectionData.getPriority()); + projection = p; + } else if ("racks".equalsIgnoreCase(projectionData.getType())) { + RacksGridProjection p = new RacksGridProjection(); + p.setPriority(projectionData.getPriority()); + projection = p; + } else if ("any".equalsIgnoreCase(projectionData.getType())) { + AnyGridProjection p = new AnyGridProjection(); + p.setPriority(projectionData.getPriority()); + projection = p; + } + if (projection != null) { + projection.setProjectionData(projectionData); + } + return projection; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultProjectedGrid.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultProjectedGrid.java new file mode 100644 index 000000000..754a07fa0 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/DefaultProjectedGrid.java @@ -0,0 +1,26 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import org.springframework.yarn.am.grid.Grid; + +public class DefaultProjectedGrid extends AbstractProjectedGrid { + + public DefaultProjectedGrid(Grid grid) { + super(grid); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptor.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptor.java new file mode 100644 index 000000000..aebbd8c0c --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptor.java @@ -0,0 +1,25 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import org.springframework.yarn.am.grid.Grid; +import org.springframework.yarn.am.grid.GridMember; + +public interface GridMemberInterceptor { + + GridMember preAdd(GridMember member, Grid grid); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptorChain.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptorChain.java new file mode 100644 index 000000000..577b0d1e3 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/GridMemberInterceptorChain.java @@ -0,0 +1,67 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.yarn.am.grid.Grid; +import org.springframework.yarn.am.grid.GridMember; + +public class GridMemberInterceptorChain { + + private static final Log log = LogFactory.getLog(GridMemberInterceptorChain.class); + + private final List interceptors = new CopyOnWriteArrayList(); + + public boolean set(List interceptors) { + synchronized (this.interceptors) { + this.interceptors.clear(); + return this.interceptors.addAll(interceptors); + } + } + + public boolean add(GridMemberInterceptor interceptor) { + return this.interceptors.add(interceptor); + } + + public List getInterceptors() { + return Collections.unmodifiableList(this.interceptors); + } + + public GridMember preAdd(GridMember member, Grid grid) { + + // if we don't have interceptors just pass + // through grid member + if (interceptors.isEmpty()) { + return member; + } + + for (GridMemberInterceptor interceptor : interceptors) { + if (interceptor.preAdd(member, grid) != null) { + return member; + } + } + if (log.isDebugEnabled()) { + log.debug("Returning null from preAdd because all interceptors returned null"); + } + return null; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/HostsGridProjection.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/HostsGridProjection.java new file mode 100644 index 000000000..7f032dbbf --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/HostsGridProjection.java @@ -0,0 +1,98 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.yarn.am.grid.GridMember; + +public class HostsGridProjection extends AbstractGridProjection { + + private static final Log log = LogFactory.getLog(HostsGridProjection.class); + + public HostsGridProjection() { + super(); + } + + @Override + public SatisfyStateData getSatisfyState() { + SatisfyStateData data = new SatisfyStateData(); + ArrayList remove = new ArrayList(); + + if (getProjectionData() != null && getProjectionData().getHosts() != null) { + // make a copy of existing hosts backed by counts + List hostCountHosts = new ArrayList(getHostCountHosts()); + + for (String phost : getProjectionData().getHosts().keySet()) { + hostCountHosts.remove(phost); + Integer target = getProjectionData().getHosts().get(phost); + int delta = target - getHostCount(phost); + data.getAllocateData().addHosts(phost, Math.max(delta, 0)); + int removeCount = Math.max(-delta, 0); + log.debug("About to remove " + removeCount + " containers from " + phost); + + // wipe out nodes if ramp down happened + Iterator iterator = getHostCountMembers(phost).iterator(); + while (iterator.hasNext() && removeCount-- > 0) { + GridMember next = iterator.next(); + log.debug("Adding " + next.getId() + " to remove list for " + phost); + remove.add(next); + } + + } + + // wipe out remaining node not tracked anymore + for (String rhost : hostCountHosts) { + Iterator iterator = getHostCountMembers(rhost).iterator(); + while (iterator.hasNext()) { + GridMember next = iterator.next(); + log.debug("Adding " + next.getId() + " to remove list for " + rhost); + remove.add(next); + } + } + } + data.setRemoveData(remove); + return data; + } + + @Override + public boolean acceptMember(GridMember member) { + + if (!isSamePriority(member)) { + return false; + } + + String host = member.getContainer().getNodeId().getHost(); + Collection hostCountMembers = getHostCountMembers(host); + + Integer target = getProjectionData().getHosts().get(host); + if (target == null) { + return false; + } + + if (hostCountMembers.size() < target) { + return addMember(member); + } else { + return false; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/ProjectionData.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/ProjectionData.java new file mode 100644 index 000000000..2ac364f8f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/ProjectionData.java @@ -0,0 +1,102 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.HashMap; +import java.util.Map; + +public class ProjectionData { + + private String type; + private Integer priority; + private Integer any = 0; + private Map hosts = new HashMap(); + private Map racks = new HashMap(); + + public ProjectionData() { + } + + public ProjectionData(Integer any) { + this(any, null, null); + } + + public ProjectionData(Integer any, Map hosts, Map racks) { + this(any, hosts, racks, null, null); + } + + public ProjectionData(Integer any, Map hosts, Map racks, String type, Integer priority) { + this.any = any; + this.hosts = hosts; + this.racks = racks; + this.type = type; + this.priority = priority; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Integer getPriority() { + return priority; + } + + public void setPriority(Integer priority) { + this.priority = priority; + } + + public Integer getAny() { + return any; + } + + public void setAny(int any) { + this.any = any; + } + + public Map getHosts() { + return hosts; + } + + public void setHosts(Map hosts) { + this.hosts = hosts; + } + + public Map getRacks() { + return racks; + } + + public void setRacks(Map racks) { + this.racks = racks; + } + + public void setHost(String host, Integer count) { + hosts.put(host, count); + } + + public void setRack(String rack, Integer count) { + racks.put(rack, count); + } + + @Override + public String toString() { + return "ProjectionData [type=" + type + ", priority=" + priority + ", any=" + any + ", hosts=" + hosts + + ", racks=" + racks + "]"; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/RacksGridProjection.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/RacksGridProjection.java new file mode 100644 index 000000000..d7aa2464d --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/RacksGridProjection.java @@ -0,0 +1,115 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.util.RackResolver; +import org.springframework.yarn.am.grid.GridMember; + +public class RacksGridProjection extends AbstractGridProjection { + + private static final Log log = LogFactory.getLog(RacksGridProjection.class); + + public RacksGridProjection() { + super(); + } + + public RacksGridProjection(Configuration configuration) { + super(configuration); + } + + @Override + public SatisfyStateData getSatisfyState() { + SatisfyStateData data = new SatisfyStateData(); + ArrayList remove = new ArrayList(); + + if (getProjectionData() != null && getProjectionData().getRacks() != null) { + + // make a copy of existing hosts backed by counts + List rackCountRacks = new ArrayList(getRackCountHosts()); + + for (String phost : getProjectionData().getRacks().keySet()) { + rackCountRacks.remove(phost); + Integer target = getProjectionData().getRacks().get(phost); + int delta = target - getRackCount(phost); + data.getAllocateData().addRacks(phost, Math.max(delta, 0)); + int removeCount = Math.max(-delta, 0); + + // wipe out nodes if ramp down happened + for (String chost : getRackCountHosts()) { + Iterator iterator = getRackCountMembers(chost).iterator(); + while (iterator.hasNext() && removeCount-- > 0) { + remove.add(iterator.next()); + } + } + } + + // wipe out remaining node not tracked anymore + for (String rhost : rackCountRacks) { + Iterator iterator = getRackCountMembers(rhost).iterator(); + while (iterator.hasNext()) { + remove.add(iterator.next()); + } + } + + } + data.setRemoveData(remove); + return data; + } + + @Override + public boolean acceptMember(GridMember member) { + + if (!isSamePriority(member)) { + return false; + } + + String host = member.getContainer().getNodeId().getHost(); + Collection hostCountMembers = getHostCountMembers(host); + + Integer target = null; + if (getConfiguration() != null) { + String rack = RackResolver.resolve(getConfiguration(), host).getNetworkLocation(); + if (rack == null) { + log.warn("Failed to resolve rack for node " + host + "."); + } else { + log.info("Resolve rack for node " + host + " into " + rack); + target = getProjectionData().getRacks().get(rack); + } + } else { + log.warn("Failed to resolve rack for node - no configuration"); + } + + + if (target == null) { + return false; + } + + if (hostCountMembers.size() < target) { + return addMember(member); + } else { + return false; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/SatisfyStateData.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/SatisfyStateData.java new file mode 100644 index 000000000..823f7dc63 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/am/grid/support/SatisfyStateData.java @@ -0,0 +1,63 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.yarn.am.allocate.ContainerAllocateData; +import org.springframework.yarn.am.grid.GridMember; + +public class SatisfyStateData { + + private List remove; + + private ContainerAllocateData allocateData; + + public SatisfyStateData() { + this(new ArrayList(), new ContainerAllocateData()); + } + + public SatisfyStateData(List remove) { + this(remove, new ContainerAllocateData()); + } + + public SatisfyStateData(ContainerAllocateData allocateData) { + this(new ArrayList(), allocateData); + } + + public SatisfyStateData(List remove, ContainerAllocateData allocateData) { + this.remove = remove; + this.allocateData = allocateData; + } + + public List getRemoveData() { + return remove; + } + + public void setRemoveData(List remove) { + this.remove = remove; + } + + public ContainerAllocateData getAllocateData() { + return allocateData; + } + + public void setAllocateData(ContainerAllocateData allocateData) { + this.allocateData = allocateData; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/SpringYarnConfigBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/SpringYarnConfigBuilder.java index d6d1ac286..59e133c60 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/SpringYarnConfigBuilder.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/SpringYarnConfigBuilder.java @@ -77,7 +77,8 @@ protected SpringYarnConfigs performBuild() throws Exception { YarnEnvironmentBuilder yarnEnvironmentBuilder = getSharedObject(YarnEnvironmentBuilder.class); yarnEnvironmentBuilder.configuration(configuration); - Map env = yarnEnvironmentBuilder.build(); + Map> envs = yarnEnvironmentBuilder.build(); + Map env = envs.get(null); config.setEnvironment(env); YarnClientBuilder yarnClientBuilder = getSharedObject(YarnClientBuilder.class); @@ -93,6 +94,7 @@ protected SpringYarnConfigs performBuild() throws Exception { yarnAppmasterBuilder.configuration(configuration); yarnAppmasterBuilder.setResourceLocalizer(localizer); yarnAppmasterBuilder.setEnvironment(env); + yarnAppmasterBuilder.setEnvironments(envs); config.setYarnAppmaster(yarnAppmasterBuilder.build()); } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterBuilder.java index 2e1edadef..98d1fbb46 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterBuilder.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterBuilder.java @@ -15,7 +15,9 @@ */ package org.springframework.yarn.config.annotation.builders; +import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.springframework.beans.BeanUtils; @@ -54,7 +56,8 @@ public final class YarnAppmasterBuilder extends AbstractConfiguredAnnotationBuil private ResourceLocalizer resourceLocalizer; private ContainerAllocator containerAllocator; private Map environment; - private String[] commands; + private Map> environments = new HashMap>(); + private final Map commands = new HashMap(); public YarnAppmasterBuilder() { super(); @@ -71,8 +74,12 @@ protected YarnAppmaster performBuild() throws Exception { if (appmaster instanceof AbstractAppmaster) { AbstractAppmaster abstractAppmaster = (AbstractAppmaster) appmaster; - if (commands != null) { - abstractAppmaster.setCommands(commands); + for (Entry entry : commands.entrySet()) { + abstractAppmaster.setCommands(entry.getKey(), entry.getValue()); + } + + for (Entry> entry : environments.entrySet()) { + abstractAppmaster.setEnvironment(entry.getKey(), entry.getValue()); } abstractAppmaster.setConfiguration(configuration); @@ -127,6 +134,10 @@ public void setEnvironment(Map environment) { this.environment = environment; } + public void setEnvironments(Map> environments) { + this.environments.putAll(environments); + } + @Override public YarnAppmasterBuilder appmasterClass(Class clazz) { appmasterClass = clazz; @@ -151,8 +162,15 @@ public YarnAppmasterBuilder appmasterClass(String clazz) { } @Override - public YarnAppmasterBuilder containerCommands(String... commands) { - this.commands = commands; + public YarnAppmasterBuilder containerCommands(String[] commands) { + // null indicates a default value + containerCommands(null, commands); + return this; + } + + @Override + public YarnAppmasterBuilder containerCommands(String id, String[] commands) { + this.commands.put(id, commands); return this; } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterConfigurer.java index 5a8ef85d3..9e172ff5c 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnAppmasterConfigurer.java @@ -146,7 +146,19 @@ public interface YarnAppmasterConfigurer { * @param commands The Yarn container commands * @return {@link YarnAppmasterConfigurer} for chaining */ - YarnAppmasterConfigurer containerCommands(String... commands); + YarnAppmasterConfigurer containerCommands(String[] commands); + + /** + * Specify a raw array of commands used to start a container. This method + * also allows to associate commands with an identifier which is used + * for example with container groups where different commands are used. + * + * @param id the commands identifier + * @param commands The Yarn container commands + * @return {@link YarnAppmasterConfigurer} for chaining + * @see #containerCommands(String[]) + */ + YarnAppmasterConfigurer containerCommands(String id, String[] commands); /** * Specify a {@code YarnAppmaster} class. diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentBuilder.java index c9fa5e713..891adc9bf 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentBuilder.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentBuilder.java @@ -17,7 +17,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import org.apache.hadoop.conf.Configuration; @@ -25,12 +27,13 @@ import org.springframework.core.io.ClassPathResource; import org.springframework.data.hadoop.config.common.annotation.AbstractConfiguredAnnotationBuilder; import org.springframework.data.hadoop.config.common.annotation.AnnotationBuilder; +import org.springframework.data.hadoop.config.common.annotation.ObjectPostProcessor; +import org.springframework.data.hadoop.config.common.annotation.configurers.DefaultPropertiesConfigurer; import org.springframework.data.hadoop.config.common.annotation.configurers.PropertiesConfigurer; import org.springframework.data.hadoop.config.common.annotation.configurers.PropertiesConfigurerAware; -import org.springframework.data.hadoop.config.common.annotation.configurers.DefaultPropertiesConfigurer; import org.springframework.util.StringUtils; -import org.springframework.yarn.config.annotation.configurers.EnvironmentClasspathConfigurer; import org.springframework.yarn.config.annotation.configurers.DefaultEnvironmentClasspathConfigurer; +import org.springframework.yarn.config.annotation.configurers.EnvironmentClasspathConfigurer; import org.springframework.yarn.configuration.EnvironmentFactoryBean; /** @@ -40,45 +43,45 @@ * */ public final class YarnEnvironmentBuilder - extends AbstractConfiguredAnnotationBuilder, YarnEnvironmentConfigurer, YarnEnvironmentBuilder> + extends AbstractConfiguredAnnotationBuilder>, YarnEnvironmentConfigurer, YarnEnvironmentBuilder> implements PropertiesConfigurerAware, YarnEnvironmentConfigurer { private Configuration configuration; - private boolean useDefaultYarnClasspath = true; - private boolean useDefaultMapreduceClasspath = true; - private boolean includeBaseDirectory = true; - private boolean includeLocalSystemEnv = false; - private String defaultYarnAppClasspath; - private String defaultMapreduceAppClasspath; - private String delimiter = ":"; - private Properties properties = new Properties(); - private ArrayList classpathEntries = new ArrayList(); + private final HashMap datas = new HashMap(); /** * Instantiates a new yarn environment builder. */ - public YarnEnvironmentBuilder() {} + public YarnEnvironmentBuilder() { + super(ObjectPostProcessor.QUIESCENT_POSTPROCESSOR, true); + } @Override - protected Map performBuild() throws Exception { - EnvironmentFactoryBean fb = new EnvironmentFactoryBean(); - fb.setConfiguration(configuration); - fb.setProperties(properties); - fb.setClasspath(StringUtils.collectionToDelimitedString(classpathEntries, delimiter)); - fb.setDelimiter(delimiter); - fb.setDefaultYarnAppClasspath(defaultYarnAppClasspath); - fb.setDefaultMapreduceAppClasspath(defaultMapreduceAppClasspath); - fb.setUseDefaultYarnClasspath(useDefaultYarnClasspath); - fb.setUseDefaultMapreduceClasspath(useDefaultMapreduceClasspath); - fb.setIncludeLocalSystemEnv(includeLocalSystemEnv); - fb.setIncludeBaseDirectory(includeBaseDirectory); - fb.afterPropertiesSet(); - return fb.getObject(); + protected Map> performBuild() throws Exception { + Map> envs = new HashMap>(); + for (Entry entry : datas.entrySet()) { + String id = entry.getKey(); + EnvironmentFactoryBean fb = new EnvironmentFactoryBean(); + fb.setConfiguration(configuration); + fb.setProperties(getDataHolder(id).properties); + fb.setClasspath(StringUtils.collectionToDelimitedString(getDataHolder(id).classpathEntries, getDataHolder(id).delimiter)); + fb.setDelimiter(getDataHolder(id).delimiter); + fb.setDefaultYarnAppClasspath(getDataHolder(id).defaultYarnAppClasspath); + fb.setDefaultMapreduceAppClasspath(getDataHolder(id).defaultMapreduceAppClasspath); + fb.setUseDefaultYarnClasspath(getDataHolder(id).useDefaultYarnClasspath); + fb.setUseDefaultMapreduceClasspath(getDataHolder(id).useDefaultMapreduceClasspath); + fb.setIncludeLocalSystemEnv(getDataHolder(id).includeLocalSystemEnv); + fb.setIncludeBaseDirectory(getDataHolder(id).includeBaseDirectory); + fb.afterPropertiesSet(); + envs.put(id, fb.getObject()); + } + return envs; } @Override public void configureProperties(Properties properties) { - this.properties.putAll(properties); + // TODO: missing id + getDataHolder(null).properties.putAll(properties); } @Override @@ -86,32 +89,57 @@ public EnvironmentClasspathConfigurer withClasspath() throws Exception { return apply(new DefaultEnvironmentClasspathConfigurer()); } + @Override + public EnvironmentClasspathConfigurer withClasspath(String id) throws Exception { + return apply(new DefaultEnvironmentClasspathConfigurer(id)); + } + @Override public YarnEnvironmentConfigurer entry(String key, String value) { - properties.put(key, value); + return entry(null, key, value); + } + + @Override + public YarnEnvironmentConfigurer entry(String id, String key, String value) { + getDataHolder(id).properties.put(key, value); return this; } @Override public YarnEnvironmentConfigurer propertiesLocation(String... locations) throws IOException { + return propertiesLocationId(null, locations); + } + + @Override + public YarnEnvironmentConfigurer propertiesLocationId(String id, String[] locations) throws IOException { for (String location : locations) { PropertiesFactoryBean fb = new PropertiesFactoryBean(); fb.setLocation(new ClassPathResource(location)); fb.afterPropertiesSet(); - properties.putAll(fb.getObject()); + getDataHolder(id).properties.putAll(fb.getObject()); } return this; } @Override public YarnEnvironmentConfigurer includeLocalSystemEnv(boolean includeLocalSystemEnv) { - this.includeLocalSystemEnv = includeLocalSystemEnv; + return includeLocalSystemEnv(null, includeLocalSystemEnv); + } + + @Override + public YarnEnvironmentConfigurer includeLocalSystemEnv(String id, boolean includeLocalSystemEnv) { + getDataHolder(id).includeLocalSystemEnv = includeLocalSystemEnv; return this; } @Override public PropertiesConfigurer withProperties() throws Exception { - return apply(new DefaultPropertiesConfigurer, YarnEnvironmentConfigurer, YarnEnvironmentBuilder>()); + return apply(new DefaultPropertiesConfigurer>, YarnEnvironmentConfigurer, YarnEnvironmentBuilder>()); + } + + @Override + public PropertiesConfigurer withProperties(String id) throws Exception { + return apply(new DefaultPropertiesConfigurer>, YarnEnvironmentConfigurer, YarnEnvironmentBuilder>()); } /** @@ -119,8 +147,8 @@ public PropertiesConfigurer withProperties() throws E * * @param classpathEntries the classpath entries */ - public void addClasspathEntries(ArrayList classpathEntries) { - this.classpathEntries.addAll(classpathEntries); + public void addClasspathEntries(String id, ArrayList classpathEntries) { + getDataHolder(id).classpathEntries.addAll(classpathEntries); } /** @@ -137,8 +165,8 @@ public void configuration(Configuration configuration) { * * @param useDefaultClasspath the new default classpath */ - public void setUseDefaultYarnClasspath(boolean useDefaultClasspath) { - this.useDefaultYarnClasspath = useDefaultClasspath; + public void setUseDefaultYarnClasspath(String id, boolean useDefaultClasspath) { + getDataHolder(id).useDefaultYarnClasspath = useDefaultClasspath; } /** @@ -146,8 +174,8 @@ public void setUseDefaultYarnClasspath(boolean useDefaultClasspath) { * * @param useDefaultClasspath the new default classpath */ - public void setUseDefaultMapreduceClasspath(boolean useDefaultClasspath) { - this.useDefaultMapreduceClasspath = useDefaultClasspath; + public void setUseDefaultMapreduceClasspath(String id, boolean useDefaultClasspath) { + getDataHolder(id).useDefaultMapreduceClasspath = useDefaultClasspath; } /** @@ -155,8 +183,8 @@ public void setUseDefaultMapreduceClasspath(boolean useDefaultClasspath) { * * @param includeBaseDirectory the new include base directory */ - public void setIncludeBaseDirectory(boolean includeBaseDirectory) { - this.includeBaseDirectory = includeBaseDirectory; + public void setIncludeBaseDirectory(String id, boolean includeBaseDirectory) { + getDataHolder(id).includeBaseDirectory = includeBaseDirectory; } /** @@ -164,8 +192,8 @@ public void setIncludeBaseDirectory(boolean includeBaseDirectory) { * * @param delimiter the new delimiter */ - public void setDelimiter(String delimiter) { - this.delimiter = delimiter; + public void setDelimiter(String id, String delimiter) { + getDataHolder(id).delimiter = delimiter; } /** @@ -173,8 +201,8 @@ public void setDelimiter(String delimiter) { * * @param defaultYarnAppClasspath the new default yarn app classpath */ - public void setDefaultYarnAppClasspath(String defaultYarnAppClasspath) { - this.defaultYarnAppClasspath = defaultYarnAppClasspath; + public void setDefaultYarnAppClasspath(String id, String defaultYarnAppClasspath) { + getDataHolder(id).defaultYarnAppClasspath = defaultYarnAppClasspath; } /** @@ -182,8 +210,29 @@ public void setDefaultYarnAppClasspath(String defaultYarnAppClasspath) { * * @param defaultMapreduceAppClasspath the new default mr app classpath */ - public void setDefaultMapreduceAppClasspath(String defaultMapreduceAppClasspath) { - this.defaultMapreduceAppClasspath = defaultMapreduceAppClasspath; + public void setDefaultMapreduceAppClasspath(String id, String defaultMapreduceAppClasspath) { + getDataHolder(id).defaultMapreduceAppClasspath = defaultMapreduceAppClasspath; + } + + private synchronized DataHolder getDataHolder(String id) { + DataHolder holder = datas.get(id); + if (holder == null) { + holder = new DataHolder(); + datas.put(id, holder); + } + return holder; + } + + private static class DataHolder { + private boolean useDefaultYarnClasspath = true; + private boolean useDefaultMapreduceClasspath = true; + private boolean includeBaseDirectory = true; + private boolean includeLocalSystemEnv = false; + private String defaultYarnAppClasspath; + private String defaultMapreduceAppClasspath; + private String delimiter = ":"; + private Properties properties = new Properties(); + private ArrayList classpathEntries = new ArrayList(); } } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentConfigurer.java index ff5893334..ec805e90f 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnEnvironmentConfigurer.java @@ -76,11 +76,21 @@ public interface YarnEnvironmentConfigurer { * </yarn:environment> * * - * @return {@link DefaultEnvironmentClasspathConfigurer} for classpath + * @return {@link EnvironmentClasspathConfigurer} for classpath * @throws Exception if error occurred */ EnvironmentClasspathConfigurer withClasspath() throws Exception; + /** + * Specify a classpath environment variable using an identifier. + * + * @param id the identifier + * @return {@link EnvironmentClasspathConfigurer} for classpath + * @throws Exception if error occurred + * @see #withClasspath() + */ + EnvironmentClasspathConfigurer withClasspath(String id) throws Exception; + /** * Specify an environment variable. * @@ -107,6 +117,17 @@ public interface YarnEnvironmentConfigurer { */ YarnEnvironmentConfigurer entry(String key, String value); + /** + * Specify an environment variable using an identifier. + * + * @param id the identifier + * @param key The environment key + * @param value The environment value + * @return {@link YarnEnvironmentConfigurer} for chaining + * @see #entry(String, String) + */ + YarnEnvironmentConfigurer entry(String id, String key, String value); + /** * Specify properties locations. * @@ -134,6 +155,17 @@ public interface YarnEnvironmentConfigurer { */ YarnEnvironmentConfigurer propertiesLocation(String... locations) throws IOException; + /** + * Specify properties locations with an identifier. + * + * @param id the identifier + * @param locations the properties file locations + * @return + * @throws IOException + * @see {@link #propertiesLocation(String...)} + */ + YarnEnvironmentConfigurer propertiesLocationId(String id, String[] locations) throws IOException; + /** * Specify if existing system environment variables should * be included automatically. @@ -156,6 +188,17 @@ public interface YarnEnvironmentConfigurer { */ YarnEnvironmentConfigurer includeLocalSystemEnv(boolean includeLocalSystemEnv); + /** + * Specify if existing system environment variables should + * be included automatically with an identifier. + * + * @param id the identifier + * @param includeLocalSystemEnv if system env variables should be included + * @return {@link YarnEnvironmentConfigurer} for chaining + * @see #includeLocalSystemEnv(boolean) + */ + YarnEnvironmentConfigurer includeLocalSystemEnv(String id, boolean includeLocalSystemEnv); + /** * Specify properties with a {@link org.springframework.data.hadoop.config.common.annotation.configurers.PropertiesConfigurer}. * @@ -184,4 +227,15 @@ public interface YarnEnvironmentConfigurer { */ PropertiesConfigurer withProperties() throws Exception; + /** + * Specify properties with a {@link org.springframework.data.hadoop.config.common.annotation.configurers.PropertiesConfigurer} + * with an identifier. + * + * @param id the identifier + * @return {@link org.springframework.data.hadoop.config.common.annotation.configurers.PropertiesConfigurer} for chaining + * @throws Exception if error occurred + * @see #withProperties() + */ + PropertiesConfigurer withProperties(String id) throws Exception; + } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerBuilder.java index dc9c6cd04..157f183db 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerBuilder.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerBuilder.java @@ -17,12 +17,15 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.springframework.data.hadoop.config.common.annotation.AbstractConfiguredAnnotationBuilder; import org.springframework.data.hadoop.config.common.annotation.AnnotationBuilder; +import org.springframework.data.hadoop.config.common.annotation.ObjectPostProcessor; import org.springframework.yarn.config.annotation.configurers.DefaultLocalResourcesCopyConfigurer; import org.springframework.yarn.config.annotation.configurers.DefaultLocalResourcesHdfsConfigurer; import org.springframework.yarn.config.annotation.configurers.LocalResourcesCopyConfigurer; @@ -51,10 +54,13 @@ public final class YarnResourceLocalizerBuilder private Collection transferEntries; private Collection rawEntries; + private final HashMap> transferEntries2 = new HashMap>(); + /** * Instantiates a new yarn resource localizer builder. */ public YarnResourceLocalizerBuilder() { + super(ObjectPostProcessor.QUIESCENT_POSTPROCESSOR, true); } @Override @@ -66,6 +72,9 @@ protected ResourceLocalizer performBuild() throws Exception { fb.setConfiguration(configuration); fb.setCopyEntries(copyEntries != null ? copyEntries : new ArrayList()); fb.setHdfsEntries(transferEntries != null ? transferEntries : new ArrayList()); + for (Entry> entry : transferEntries2.entrySet()) { + fb.setHdfsEntries(entry.getKey(), entry.getValue()); + } if (rawEntries != null) { fb.setRawCopyEntries(rawEntries); } @@ -83,6 +92,11 @@ public LocalResourcesHdfsConfigurer withHdfs() throws Exception { return apply(new DefaultLocalResourcesHdfsConfigurer()); } + @Override + public LocalResourcesHdfsConfigurer withHdfs(String id) throws Exception { + return apply(new DefaultLocalResourcesHdfsConfigurer(id)); + } + @Override public YarnResourceLocalizerConfigurer stagingDirectory(String stagingDirectory) { this.stagingDirectory = stagingDirectory; @@ -111,6 +125,10 @@ public void setHdfsEntries(Collection transferEntries) { this.transferEntries = transferEntries; } + public void setHdfsEntries(String id, Collection transferEntries) { + this.transferEntries2.put(id, transferEntries); + } + public void setRawCopyEntries(Collection rawEntries) { this.rawEntries = rawEntries; } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerConfigurer.java index bdca013cb..350d1eb28 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/builders/YarnResourceLocalizerConfigurer.java @@ -16,6 +16,7 @@ package org.springframework.yarn.config.annotation.builders; import org.springframework.yarn.config.annotation.SpringYarnConfigurerAdapter; +import org.springframework.yarn.config.annotation.configurers.DefaultLocalResourcesHdfsConfigurer; import org.springframework.yarn.config.annotation.configurers.LocalResourcesCopyConfigurer; import org.springframework.yarn.config.annotation.configurers.DefaultLocalResourcesCopyConfigurer; import org.springframework.yarn.config.annotation.configurers.LocalResourcesHdfsConfigurer; @@ -74,7 +75,7 @@ public interface YarnResourceLocalizerConfigurer { LocalResourcesCopyConfigurer withCopy() throws Exception; /** - * Specify configuration options as properties with a {@link DefaultLocalResourcesCopyConfigurer}. + * Specify configuration options as properties with a {@link DefaultLocalResourcesHdfsConfigurer}. * *
JavaConfig: * 

@@ -92,11 +93,29 @@ public interface YarnResourceLocalizerConfigurer {
 	 * </yarn:localresources>
 	 *
* - * @return {@link LocalResourcesCopyConfigurer} for chaining + * @return {@link LocalResourcesHdfsConfigurer} for chaining * @throws Exception if error occurred */ LocalResourcesHdfsConfigurer withHdfs() throws Exception; + /** + * Specify configuration options as properties with a {@link DefaultLocalResourcesHdfsConfigurer} + * with an identifier. + * + * @param id the identifier + * @return {@link LocalResourcesHdfsConfigurer} for chaining + * @throws Exception if error occurred + * @see #withHdfs() + */ + LocalResourcesHdfsConfigurer withHdfs(String id) throws Exception; + + /** + * Specify a staging directory. + * + * @param stagingDirectory the staging directory + * @return {@link YarnResourceLocalizerConfigurer} for chaining + * @throws Exception if error occurred + */ YarnResourceLocalizerConfigurer stagingDirectory(String stagingDirectory) throws Exception; } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultEnvironmentClasspathConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultEnvironmentClasspathConfigurer.java index ab85f285c..ac7e157e6 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultEnvironmentClasspathConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultEnvironmentClasspathConfigurer.java @@ -33,7 +33,7 @@ * */ public class DefaultEnvironmentClasspathConfigurer - extends AnnotationConfigurerAdapter, YarnEnvironmentConfigurer, YarnEnvironmentBuilder> + extends AnnotationConfigurerAdapter>, YarnEnvironmentConfigurer, YarnEnvironmentBuilder> implements EnvironmentClasspathConfigurer { private boolean useDefaultYarnClasspath = false; @@ -45,16 +45,25 @@ public class DefaultEnvironmentClasspathConfigurer private ArrayList classpathEntries = new ArrayList(); + private String id; + + public DefaultEnvironmentClasspathConfigurer() { + } + + public DefaultEnvironmentClasspathConfigurer(String id) { + this.id = id; + } + @Override public void configure(YarnEnvironmentBuilder builder) throws Exception { - builder.addClasspathEntries(classpathEntries); - builder.setUseDefaultYarnClasspath(useDefaultYarnClasspath); - builder.setUseDefaultMapreduceClasspath(useDefaultMapreduceClasspath); - builder.setDefaultYarnAppClasspath(defaultYarnAppClasspath); - builder.setDefaultMapreduceAppClasspath(defaultMapreduceAppClasspath); - builder.setIncludeBaseDirectory(includeBaseDirectory); + builder.addClasspathEntries(id, classpathEntries); + builder.setUseDefaultYarnClasspath(id, useDefaultYarnClasspath); + builder.setUseDefaultMapreduceClasspath(id, useDefaultMapreduceClasspath); + builder.setDefaultYarnAppClasspath(id, defaultYarnAppClasspath); + builder.setDefaultMapreduceAppClasspath(id, defaultMapreduceAppClasspath); + builder.setIncludeBaseDirectory(id, includeBaseDirectory); if (StringUtils.hasText(delimiter)) { - builder.setDelimiter(delimiter); + builder.setDelimiter(id, delimiter); } } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultLocalResourcesHdfsConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultLocalResourcesHdfsConfigurer.java index 924c9fe4a..24ad7baba 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultLocalResourcesHdfsConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultLocalResourcesHdfsConfigurer.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,11 +30,32 @@ public class DefaultLocalResourcesHdfsConfigurer extends AnnotationConfigurerAdapter implements LocalResourcesHdfsConfigurer { - private Collection hdfsEntries = new ArrayList(); + private final Collection hdfsEntries = new ArrayList(); + private final String id; + + /** + * Instantiates a new default local resources hdfs configurer. + */ + public DefaultLocalResourcesHdfsConfigurer() { + this(null); + } + + /** + * Instantiates a new default local resources hdfs configurer. + * + * @param id the identifier + */ + public DefaultLocalResourcesHdfsConfigurer(String id) { + this.id = id; + } @Override public void configure(YarnResourceLocalizerBuilder builder) throws Exception { - builder.setHdfsEntries(hdfsEntries); + if (id == null) { + builder.setHdfsEntries(hdfsEntries); + } else { + builder.setHdfsEntries(id, hdfsEntries); + } } @Override diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultMasterContainerAllocatorConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultMasterContainerAllocatorConfigurer.java index 3c2a202ef..58e80a45e 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultMasterContainerAllocatorConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/DefaultMasterContainerAllocatorConfigurer.java @@ -15,13 +15,24 @@ */ package org.springframework.yarn.config.annotation.configurers; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurer; import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurerAdapter; +import org.springframework.util.StringUtils; import org.springframework.yarn.am.YarnAppmaster; import org.springframework.yarn.am.allocate.DefaultContainerAllocator; import org.springframework.yarn.config.annotation.builders.YarnAppmasterBuilder; import org.springframework.yarn.config.annotation.builders.YarnAppmasterConfigurer; import org.springframework.yarn.support.ParsingUtils; +/** + * {@link AnnotationConfigurer} for {@link YarnAppmaster} container allocator. + * + * @author Janne Valkealahti + * + */ public class DefaultMasterContainerAllocatorConfigurer extends AnnotationConfigurerAdapter implements MasterContainerAllocatorConfigurer { @@ -31,20 +42,30 @@ public class DefaultMasterContainerAllocatorConfigurer private Integer virtualCores; private boolean locality; + private final List collectionConfigurers = + new ArrayList(); + @Override public void configure(YarnAppmasterBuilder builder) throws Exception { - DefaultContainerAllocator containerAllocator = new DefaultContainerAllocator(); + DefaultContainerAllocator allocator = new DefaultContainerAllocator(); if (priority != null) { - containerAllocator.setPriority(priority); + allocator.setPriority(priority); } if (virtualCores != null) { - containerAllocator.setVirtualcores(virtualCores); + allocator.setVirtualcores(virtualCores); } if (memory != null) { - containerAllocator.setMemory(ParsingUtils.parseBytesAsMegs(memory)); + allocator.setMemory(ParsingUtils.parseBytesAsMegs(memory)); + } + allocator.setLocality(locality); + + for (DefaultMasterContainerAllocatorCollectionConfigurer configurer : collectionConfigurers) { + allocator.setAllocationValues(configurer.id, configurer.priority, configurer.virtualCores, + StringUtils.hasText(configurer.memory) ? ParsingUtils.parseBytesAsMegs(configurer.memory) : null, + configurer.locality); } - containerAllocator.setLocality(locality); - builder.setContainerAllocator(containerAllocator); + + builder.setContainerAllocator(allocator); } @Override @@ -71,9 +92,67 @@ public MasterContainerAllocatorConfigurer memory(int memory) { return this; } + @Override public MasterContainerAllocatorConfigurer locality(boolean locality) { this.locality = locality; return this; } + @Override + public MasterContainerAllocatorCollectionConfigurer withCollection(String id) { + DefaultMasterContainerAllocatorCollectionConfigurer configurer = + new DefaultMasterContainerAllocatorCollectionConfigurer(id); + collectionConfigurers.add(configurer); + return configurer; + } + + public final class DefaultMasterContainerAllocatorCollectionConfigurer implements MasterContainerAllocatorCollectionConfigurer { + + private final String id; + private Integer priority; + private String memory; + private Integer virtualCores; + private boolean locality; + + public DefaultMasterContainerAllocatorCollectionConfigurer(String id) { + this.id = id; + } + + @Override + public MasterContainerAllocatorCollectionConfigurer priority(Integer priority) { + this.priority = priority; + return this; + } + + @Override + public MasterContainerAllocatorCollectionConfigurer virtualCores(Integer virtualCores) { + this.virtualCores = virtualCores; + return this; + } + + @Override + public MasterContainerAllocatorCollectionConfigurer memory(String memory) { + this.memory = memory; + return this; + } + + @Override + public MasterContainerAllocatorCollectionConfigurer memory(int memory) { + this.memory = Integer.toString(memory); + return this; + } + + @Override + public MasterContainerAllocatorCollectionConfigurer locality(boolean locality) { + this.locality = locality; + return this; + } + + @Override + public MasterContainerAllocatorConfigurer and() { + return DefaultMasterContainerAllocatorConfigurer.this; + } + + } + } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/MasterContainerAllocatorConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/MasterContainerAllocatorConfigurer.java index 110b42574..93dd171fa 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/MasterContainerAllocatorConfigurer.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/config/annotation/configurers/MasterContainerAllocatorConfigurer.java @@ -18,6 +18,7 @@ import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurerBuilder; import org.springframework.yarn.am.allocate.ContainerAllocator; import org.springframework.yarn.config.annotation.builders.YarnAppmasterConfigurer; +import org.springframework.yarn.config.annotation.configurers.DefaultMasterContainerAllocatorConfigurer.DefaultMasterContainerAllocatorCollectionConfigurer; /** * {@link AnnotationConfigurerBuilder} for configuring {@link ContainerAllocator}. @@ -199,4 +200,88 @@ public interface MasterContainerAllocatorConfigurer extends AnnotationConfigurer */ MasterContainerAllocatorConfigurer locality(boolean locality); + /** + * Specify a collection of container allocator attributes. Applies a new + * {@link DefaultMasterContainerAllocatorCollectionConfigurer} into a current configurer. + * + *

+ *

JavaConfig: + *

+ * 

+	 *
+	 * public void configure(YarnAppmasterConfigure master) throws Exception {
+	 *   master
+	 *     .withContainerAllocator()
+	 *       .withCollection("id")
+	 *         .priority(0);
+	 * }
+	 *
+ * + * @param id + * @return {@link MasterContainerAllocatorCollectionConfigurer} for chaining + */ + MasterContainerAllocatorCollectionConfigurer withCollection(String id); + + /** + * {@code MasterContainerAllocatorCollectionConfigurer} is an interface + * for {@code DefaultMasterContainerAllocatorCollectionConfigurer} which is + * used to configure {@link MasterContainerAllocatorConfigurer} parameters + * as an identified collection. + */ + public interface MasterContainerAllocatorCollectionConfigurer { + + /** + * Specify a container priority for {@link ContainerAllocator}. + * + * @param priority the priority + * @return {@link MasterContainerAllocatorCollectionConfigurer} for chaining + * @see MasterContainerAllocatorConfigurer#priority(Integer) + */ + MasterContainerAllocatorCollectionConfigurer priority(Integer priority); + + /** + * Specify a container virtual cores for {@link ContainerAllocator}. + * + * @param virtualCores the virtual cores + * @return {@link MasterContainerAllocatorCollectionConfigurer} for chaining + * @see MasterContainerAllocatorConfigurer#virtualCores(Integer) + */ + MasterContainerAllocatorCollectionConfigurer virtualCores(Integer virtualCores); + + /** + * Specify a container memory for {@link ContainerAllocator}. + * + * @param memory the memory + * @return {@link MasterContainerAllocatorCollectionConfigurer} for chaining + * @see MasterContainerAllocatorConfigurer#memory(String) + */ + MasterContainerAllocatorCollectionConfigurer memory(String memory); + + /** + * Specify a container memory for {@link ContainerAllocator}. + * + * @param memory the memory + * @return {@link MasterContainerAllocatorCollectionConfigurer} for chaining + * @see MasterContainerAllocatorConfigurer#memory(String) + */ + MasterContainerAllocatorCollectionConfigurer memory(int memory); + + /** + * Specify a locality relaxing for {@link ContainerAllocator}. + * + * @param locality the locality flag for resource relaxing + * @return {@link MasterContainerAllocatorCollectionConfigurer} for chaining + * @see MasterContainerAllocatorConfigurer#locality(boolean) + */ + MasterContainerAllocatorCollectionConfigurer locality(boolean locality); + + /** + * Returns a parent {@link MasterContainerAllocatorConfigurer} configurer. + * + * @return {@link MasterContainerAllocatorConfigurer} for chaining + */ + MasterContainerAllocatorConfigurer and(); + + } + } diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/DefaultMultiResourceLocalizer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/DefaultMultiResourceLocalizer.java new file mode 100644 index 000000000..d47baf0d4 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/DefaultMultiResourceLocalizer.java @@ -0,0 +1,62 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.fs; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.LocalResource; + +/** + * Default implementation of a {@link MultiResourceLocalizer}. + * + * @author Janne Valkealahti + * + */ +public class DefaultMultiResourceLocalizer implements MultiResourceLocalizer { + + private final HashMap localizers = new HashMap(); + + /** + * Instantiates a new default multi resource localizer. + * + * @param localizer the default localizer + * @param localizers the custom localizers + */ + public DefaultMultiResourceLocalizer(ResourceLocalizer localizer, Map localizers) { + this.localizers.put(null, localizer); + this.localizers.putAll(localizers); + } + + @Override + public Map getResources() { + return getResources(null); + } + + @Override + public Map getResources(String id) { + ResourceLocalizer localizer = localizers.get(id); + if (localizer == null) { + localizer = localizers.get(null); + } + if (localizer != null) { + return localizer.getResources(); + } else { + return null; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/LocalResourcesFactoryBean.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/LocalResourcesFactoryBean.java index d8deb1f39..7e04a7a2d 100644 --- a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/LocalResourcesFactoryBean.java +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/LocalResourcesFactoryBean.java @@ -15,9 +15,11 @@ */ package org.springframework.yarn.fs; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -38,9 +40,12 @@ public class LocalResourcesFactoryBean implements InitializingBean, FactoryBean< /** Localizer returned from this factory */ private ResourceLocalizer resources; - /** Localizer transfer entries*/ + /** Localizer transfer entries */ private Collection hdfsEntries; + /** Localizer additional transfer entries */ + private final HashMap> additionalHdfsEntries = new HashMap>(); + /** Localizer copy entries*/ private Collection copyEntries; @@ -97,7 +102,24 @@ public void afterPropertiesSet() throws Exception { defaultResourceLocalizer.setRawFileContents(rawFileContents); } - resources = defaultResourceLocalizer; + if (additionalHdfsEntries.size() == 0) { + resources = defaultResourceLocalizer; + } else { + Map localizers = new HashMap(); + for (Entry> entry : additionalHdfsEntries.entrySet()) { + for(TransferEntry e : entry.getValue()) { + if(e.type == null) { + e.type = (defaultType != null ? defaultType : LocalResourceType.FILE); + } + if(e.visibility == null) { + e.visibility = (defaultVisibility != null ? defaultVisibility : LocalResourceVisibility.APPLICATION); + } + } + localizers.put(entry.getKey(), new DefaultResourceLocalizer(configuration, entry.getValue(), + new ArrayList())); + } + resources = new DefaultMultiResourceLocalizer(defaultResourceLocalizer, localizers); + } } /** @@ -127,6 +149,16 @@ public void setHdfsEntries(Collection hdfsEntries) { this.hdfsEntries = hdfsEntries; } + /** + * Sets hdfs entries reference for this factory. + * + * @param id the identifier + * @param hdfsEntries Collection of hdfs entries + */ + public void setHdfsEntries(String id, Collection hdfsEntries) { + this.additionalHdfsEntries.put(id, hdfsEntries); + } + /** * Sets copy entries reference for this factory. * diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiLocalResourcesSelector.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiLocalResourcesSelector.java new file mode 100644 index 000000000..418a9c873 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiLocalResourcesSelector.java @@ -0,0 +1,42 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.fs; + +import java.util.List; + +import org.springframework.yarn.fs.LocalResourcesSelector.Entry; + +/** + * Extension of {@link LocalResourcesSelector} adding functionality to + * use multiple {@link LocalResourcesSelector}s. + * + * @author Janne Valkealahti + * + */ +public interface MultiLocalResourcesSelector extends LocalResourcesSelector { + + /** + * Select a {@link List} of {@link Entry}s identified by + * an arbitrary id. + * + * @param id the identifier + * @param dir the base directory + * @return the list of entries + * @see LocalResourcesSelector#select(String) + */ + List select(String id, String dir); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiResourceLocalizer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiResourceLocalizer.java new file mode 100644 index 000000000..945ae37ca --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/fs/MultiResourceLocalizer.java @@ -0,0 +1,42 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.fs; + +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.LocalResource; + +/** + * Extension of {@link ResourceLocalizer} adding functionality to + * use multiple {@link ResourceLocalizer}s. + * + * @author Janne Valkealahti + * + */ +public interface MultiResourceLocalizer extends ResourceLocalizer { + + /** + * Gets a map of {@link LocalResource} instances identified by + * an arbitrary id. Passing null or unknown id + * will fall back to base method {@link ResourceLocalizer#getResources()}. + * + * @param id identifier for local resources + * @return The map containing {@link LocalResource} instances + * @see ResourceLocalizer#getResources() + */ + Map getResources(String id); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/console/ContainerClusterReport.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/console/ContainerClusterReport.java new file mode 100644 index 000000000..1c971fd35 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/console/ContainerClusterReport.java @@ -0,0 +1,342 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.console; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.util.StringUtils; + +/** + * Utility class creating reports for container clusters. + * + * @author Janne Valkealahti + * + */ +public class ContainerClusterReport { + + private final Table table; + + private ContainerClusterReport(Table table) { + this.table = table; + } + + @Override + public String toString() { + return table.toString(); + } + + public static ClustersInfoReportBuilder clustersInfoReportBuilder() { + return new ClustersInfoReportBuilder(); + } + + public static ClusterInfoReportBuilder clusterInfoReportBuilder() { + return new ClusterInfoReportBuilder(); + } + + public static class ClustersInfoReportBuilder { + + private ArrayList fields = new ArrayList(); + + private List reports; + + private Map headerNameOverrides = new HashMap(); + + private ClustersInfoReportBuilder() { + } + + /** + * Adds a new field into a report. + * + * @param f the field + * @return the builder for chaining + */ + public ClustersInfoReportBuilder add(ClustersInfoField f) { + fields.add(f); + return this; + } + + /** + * Adds a new fields into a report. + * + * @param f the field + * @return the builder for chaining + */ + public ClustersInfoReportBuilder add(ClustersInfoField... f) { + for (ClustersInfoField ff : f) { + fields.add(ff); + } + return this; + } + + /** + * Add a source this report will be build from. + * + * @param reports the reports + * @return the builder for chaining + */ + public ClustersInfoReportBuilder from(List reports) { + this.reports = reports; + return this; + } + + public ClustersInfoReportBuilder header(String from, String to) { + headerNameOverrides.put(from.toLowerCase(), to); + return this; + } + + /** + * Builds an applications report. + * + * @return the applications report + */ + public ContainerClusterReport build() { + Table table = new Table(); + addHeader(table); + if (reports != null) { + for (String report : reports) { + addRow(table, report); + } + } + return new ContainerClusterReport(table); + } + + private void addHeader(Table table) { + int index = 1; + for (ClustersInfoField f : fields) { + table.addHeader(index++, new TableHeader(getHeaderNameMayOverride(f))); + } + } + + private String getHeaderNameMayOverride(ClustersInfoField f) { + String n = headerNameOverrides.get(f.toString().toLowerCase()); + return StringUtils.hasText(n) ? n : f.getName(); + } + + private void addRow(Table table, String report) { + final TableRow row = new TableRow(); + int index = 1; + for (ClustersInfoField f : fields) { + if (ClustersInfoField.ID == f) { + row.addValue(index++, report); + } + } + table.getRows().add(row); + } + } + + public static class ClusterInfoReportBuilder { + + private ArrayList fields = new ArrayList(); + + private List reports; + + private Map headerNameOverrides = new HashMap(); + + private ClusterInfoReportBuilder() { + } + + /** + * Adds a new field into a report. + * + * @param f the field + * @return the builder for chaining + */ + public ClusterInfoReportBuilder add(ClusterInfoField f) { + fields.add(f); + return this; + } + + /** + * Adds a new fields into a report. + * + * @param f the field + * @return the builder for chaining + */ + public ClusterInfoReportBuilder add(ClusterInfoField... f) { + for (ClusterInfoField ff : f) { + fields.add(ff); + } + return this; + } + + /** + * Add a source this report will be build from. + * + * @param reports the reports + * @return the builder for chaining + */ + public ClusterInfoReportBuilder from(List reports) { + this.reports = reports; + return this; + } + + public ClusterInfoReportBuilder header(String from, String to) { + headerNameOverrides.put(from.toLowerCase(), to); + return this; + } + + /** + * Builds an applications report. + * + * @return the applications report + */ + public ContainerClusterReport build() { + Table table = new Table(); + addHeader(table); + if (reports != null) { + for (ClustersInfoReportData report : reports) { + addRow(table, report); + } + } + return new ContainerClusterReport(table); + } + + private void addHeader(Table table) { + int index = 1; + for (ClusterInfoField f : fields) { + table.addHeader(index++, new TableHeader(getHeaderNameMayOverride(f))); + } + } + + private String getHeaderNameMayOverride(ClusterInfoField f) { + String n = headerNameOverrides.get(f.toString().toLowerCase()); + return StringUtils.hasText(n) ? n : f.getName(); + } + + private void addRow(Table table, ClustersInfoReportData report) { + final TableRow row = new TableRow(); + int index = 1; + for (ClusterInfoField f : fields) { + if (ClusterInfoField.STATE == f) { + row.addValue(index++, report.getState()); + } else if (ClusterInfoField.MEMBERS == f) { + row.addValue(index++, report.getCount() != null ? report.getCount().toString() : "N/A"); + } else if (ClusterInfoField.PROJECTIONANY == f) { + row.addValue(index++, report.getProjectionAny() != null ? report.getProjectionAny().toString() : "N/A"); + } else if (ClusterInfoField.PROJECTIONHOSTS == f) { + row.addValue(index++, report.getProjectionHosts() != null ? report.getProjectionHosts().toString() : "N/A"); + } else if (ClusterInfoField.PROJECTIONRACKS == f) { + row.addValue(index++, report.getProjectionRacks() != null ? report.getProjectionRacks().toString() : "N/A"); + } else if (ClusterInfoField.SATISFYANY == f) { + row.addValue(index++, report.getSatisfyAny() != null ? report.getSatisfyAny().toString() : "N/A"); + } else if (ClusterInfoField.SATISFYHOSTS == f) { + row.addValue(index++, report.getSatisfyHosts() != null ? report.getSatisfyHosts().toString() : "N/A"); + } else if (ClusterInfoField.SATISFYRACKS == f) { + row.addValue(index++, report.getSatisfyRacks() != null ? report.getSatisfyRacks().toString() : "N/A"); + } + } + table.getRows().add(row); + } + } + + /** + * Enums for clusters fields. + */ + public static enum ClustersInfoField { + ID("CLUSTER ID"); + + private String name; + + private ClustersInfoField() { + } + + private ClustersInfoField(String name) { + this.name = name; + } + + protected String getName() { + return StringUtils.hasText(name) ? name : this.toString(); + } + } + + /** + * Enums for cluster fields. + */ + public static enum ClusterInfoField { + STATE("CLUSTER STATE"), + MEMBERS("MEMBER COUNT"), + PROJECTIONANY("ANY PROJECTION"), + PROJECTIONHOSTS("HOSTS PROJECTION"), + PROJECTIONRACKS("RACKS PROJECTION"), + SATISFYANY("ANY SATISFY"), + SATISFYHOSTS("HOSTS SATISFY"), + SATISFYRACKS("RACKS SATISFY"); + + private String name; + + private ClusterInfoField() { + } + + private ClusterInfoField(String name) { + this.name = name; + } + + protected String getName() { + return StringUtils.hasText(name) ? name : this.toString(); + } + } + + public static class ClustersInfoReportData { + private String state; + private Integer count; + private Integer pany; + private Map phosts; + private Map pracks; + private Integer sany; + private Map shosts; + private Map sracks; + + public ClustersInfoReportData(String state, Integer count, Integer pany, Map phosts, + Map pracks, Integer sany, Map shosts, Map sracks) { + this.state = state; + this.count = count; + this.pany = pany; + this.phosts = phosts; + this.pracks = pracks; + this.sany = sany; + this.shosts = shosts; + this.sracks = sracks; + } + public String getState() { + return state; + } + public Integer getCount() { + return count; + } + public Integer getProjectionAny() { + return pany; + } + public Map getProjectionHosts() { + return phosts; + } + public Map getProjectionRacks() { + return pracks; + } + public Integer getSatisfyAny() { + return sany; + } + public Map getSatisfyHosts() { + return shosts; + } + public Map getSatisfyRacks() { + return sracks; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/AbstractStateMachine.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/AbstractStateMachine.java new file mode 100644 index 000000000..cc4edea2f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/AbstractStateMachine.java @@ -0,0 +1,173 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.ListIterator; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.yarn.support.LifecycleObjectSupport; +import org.springframework.yarn.support.statemachine.state.State; +import org.springframework.yarn.support.statemachine.transition.Transition; +import org.springframework.yarn.support.statemachine.transition.TransitionKind; +import org.springframework.yarn.support.statemachine.trigger.Trigger; + +public abstract class AbstractStateMachine extends LifecycleObjectSupport implements StateMachine, E> { + + private static final Log log = LogFactory.getLog(AbstractStateMachine.class); + + private Collection> states; + + private Collection> transitions; + + private final State initialState; + + private State currentState; + + private volatile Runnable task; + + private final Queue> eventQueue = new ConcurrentLinkedQueue>(); + + private final LinkedList> deferList = new LinkedList>(); + + public AbstractStateMachine(Collection> states, Collection> transitions, State initialState) { + super(); + this.states = states; + this.transitions = transitions; + this.initialState = initialState; + } + + public Collection> getStates() { + return states; + } + + @Override + public State getState() { + return currentState; + } + + @Override + public State getInitialState() { + return initialState; + } + + @Override + public void sendEvent(Message event) { + event = MessageBuilder.fromMessage(event).setHeader("machine", this).build(); + if (log.isDebugEnabled()) { + log.debug("Queue event " + event); + } + eventQueue.add(event); + scheduleEventQueueProcessing(); + } + + @Override + public void sendEvent(E event) { + sendEvent(MessageBuilder.withPayload(event).build()); + } + + @Override + protected void doStart() { + super.doStart(); + switchToState(initialState); + } + + private void switchToState(State state) { + log.info("Moving into state=" + state + " from " + currentState); + currentState = state; + + for (Transition transition : transitions) { + State source = transition.getSource(); + State target = transition.getTarget(); + if (transition.getTrigger() == null && source.equals(currentState)) { + switchToState(target); + } + + } + + } + + private void processEventQueue() { + log.debug("Process event queue"); + Message queuedEvent = null; + while ((queuedEvent = eventQueue.poll()) != null) { + Message defer = null; + for (Transition transition : transitions) { + State source = transition.getSource(); + State target = transition.getTarget(); + Trigger trigger = transition.getTrigger(); + if (source.equals(currentState)) { + if (trigger != null && trigger.evaluate(queuedEvent.getPayload())) { + transition.transit(queuedEvent.getHeaders()); + if (transition.getKind() != TransitionKind.INTERNAL) { + switchToState(target); + } + break; + } else if (source.getDeferredEvents() != null && source.getDeferredEvents().contains(queuedEvent.getPayload())) { + defer = queuedEvent; + } + } + } + if (defer != null) { + log.info("Deferring event " + defer); + deferList.addLast(defer); + } + } + } + + private void processDeferList() { + log.debug("Process defer list"); + ListIterator> iterator = deferList.listIterator(); + while (iterator.hasNext()) { + Message event = iterator.next(); + for (Transition transition : transitions) { + State source = transition.getSource(); + State target = transition.getTarget(); + Trigger trigger = transition.getTrigger(); + if (source.equals(currentState)) { + if (trigger != null && trigger.evaluate(event.getPayload())) { + transition.transit(event.getHeaders()); + if (transition.getKind() != TransitionKind.INTERNAL) { + switchToState(target); + } + iterator.remove(); + } + } + } + } + } + + private void scheduleEventQueueProcessing() { + if (task == null) { + task = new Runnable() { + @Override + public void run() { + processEventQueue(); + processDeferList(); + task = null; + } + }; + getTaskExecutor().execute(task); + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/EnumStateMachine.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/EnumStateMachine.java new file mode 100644 index 000000000..122fb6b05 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/EnumStateMachine.java @@ -0,0 +1,30 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.state.State; +import org.springframework.yarn.support.statemachine.transition.Transition; + +public class EnumStateMachine, E extends Enum> extends AbstractStateMachine { + + public EnumStateMachine(Collection> states, Collection> transitions, State initialState) { + super(states, transitions, initialState); + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachine.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachine.java new file mode 100644 index 000000000..b84bc10f1 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachine.java @@ -0,0 +1,32 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine; + +import org.springframework.messaging.Message; + +public interface StateMachine { + + S getState(); + + S getInitialState(); + + void start(); + + void sendEvent(Message event); + + void sendEvent(E event); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachineSystemConstants.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachineSystemConstants.java new file mode 100644 index 000000000..ce6a342a6 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/StateMachineSystemConstants.java @@ -0,0 +1,26 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine; + +public abstract class StateMachineSystemConstants { + + /** Default bean id for state machine. */ + public static final String DEFAULT_ID_STATEMACHINE = "stateMachine"; + + /** Default bean id for state machine factory. */ + public static final String DEFAULT_ID_STATEMACHINEFACTORY = "stateMachineFactory"; + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/action/Action.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/action/Action.java new file mode 100644 index 000000000..7f58998dd --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/action/Action.java @@ -0,0 +1,24 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.action; + +import org.springframework.messaging.MessageHeaders; + +public interface Action { + + void execute(MessageHeaders headers); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachine.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachine.java new file mode 100644 index 000000000..40c39d433 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachine.java @@ -0,0 +1,42 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.hadoop.config.common.annotation.EnableAnnotationConfiguration; +import org.springframework.data.hadoop.config.common.annotation.configuration.ObjectPostProcessorConfiguration; +import org.springframework.yarn.support.statemachine.config.configuration.StateMachineConfiguration; + +/** + * Example annotation which imports @{@link Configuration}s. + * + * @author Janne Valkealahti + * + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +@Documented +@EnableAnnotationConfiguration +@Import({StateMachineConfiguration.class,ObjectPostProcessorConfiguration.class}) +public @interface EnableStateMachine { +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachineFactory.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachineFactory.java new file mode 100644 index 000000000..6ae981fc6 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnableStateMachineFactory.java @@ -0,0 +1,42 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.hadoop.config.common.annotation.EnableAnnotationConfiguration; +import org.springframework.data.hadoop.config.common.annotation.configuration.ObjectPostProcessorConfiguration; +import org.springframework.yarn.support.statemachine.config.configuration.StateMachineFactoryConfiguration; + +/** + * Example annotation which imports @{@link Configuration}s. + * + * @author Janne Valkealahti + * + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +@Documented +@EnableAnnotationConfiguration +@Import({StateMachineFactoryConfiguration.class,ObjectPostProcessorConfiguration.class}) +public @interface EnableStateMachineFactory { +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineConfigurerAdapter.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineConfigurerAdapter.java new file mode 100644 index 000000000..70d0197f1 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineConfigurerAdapter.java @@ -0,0 +1,20 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +public class EnumStateMachineConfigurerAdapter, E extends Enum> extends StateMachineConfigurerAdapter { + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineFactory.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineFactory.java new file mode 100644 index 000000000..d32175a01 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/EnumStateMachineFactory.java @@ -0,0 +1,79 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import org.springframework.yarn.support.LifecycleObjectSupport; +import org.springframework.yarn.support.statemachine.EnumStateMachine; +import org.springframework.yarn.support.statemachine.StateMachine; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates.StateData; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions.TransitionData; +import org.springframework.yarn.support.statemachine.state.EnumState; +import org.springframework.yarn.support.statemachine.state.State; +import org.springframework.yarn.support.statemachine.transition.DefaultExternalTransition; +import org.springframework.yarn.support.statemachine.transition.Transition; + +public class EnumStateMachineFactory, E extends Enum> extends LifecycleObjectSupport implements + StateMachineFactory, E> { + + private StateMachineTransitions stateMachineTransitions; + private StateMachineStates stateMachineStates; + + public EnumStateMachineFactory(StateMachineTransitions stateMachineTransitions, + StateMachineStates stateMachineStates) { + this.stateMachineTransitions = stateMachineTransitions; + this.stateMachineStates = stateMachineStates; + } + + @Override + public StateMachine, E> getStateMachine() { + return stateMachine(); + } + + public StateMachine, E> stateMachine() { + Map> stateMap = new HashMap>(); + for (StateData stateData : stateMachineStates.getStates()) { + stateMap.put(stateData.getState(), new EnumState(stateData.getState(), stateData.getDeferred())); + } + + Collection> transitions = new ArrayList>(); + for (TransitionData transitionData : stateMachineTransitions.getTransitions()) { + S source = transitionData.getSource(); + S target = transitionData.getTarget(); + E event = transitionData.getEvent(); + DefaultExternalTransition transition = new DefaultExternalTransition(stateMap.get(source), + stateMap.get(target), transitionData.getActions(), event); + transitions.add(transition); + } + + EnumStateMachine machine = new EnumStateMachine(stateMap.values(), transitions, stateMap.get(stateMachineStates + .getInitialState())); + machine.afterPropertiesSet(); + if (getBeanFactory() != null) { + machine.setBeanFactory(getBeanFactory()); + } + machine.setAutoStartup(isAutoStartup()); + machine.start(); + return machine; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfig.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfig.java new file mode 100644 index 000000000..fc840f6ee --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfig.java @@ -0,0 +1,39 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions; + +public class StateMachineConfig { + + public final StateMachineTransitions transitions; + + public final StateMachineStates states; + + public StateMachineConfig(StateMachineTransitions transitions, StateMachineStates states) { + this.transitions = transitions; + this.states = states; + } + + public StateMachineTransitions getTransitions() { + return transitions; + } + + public StateMachineStates getStates() { + return states; + } +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfigurerAdapter.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfigurerAdapter.java new file mode 100644 index 000000000..23553bc81 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineConfigurerAdapter.java @@ -0,0 +1,73 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationBuilder; +import org.springframework.data.hadoop.config.common.annotation.ObjectPostProcessor; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineConfigBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +public class StateMachineConfigurerAdapter implements StateMachineConfigurer { + + private StateMachineTransitionBuilder transitionBuilder; + private StateMachineStateBuilder stateBuilder; + + @Override + public final void init(StateMachineConfigBuilder config) throws Exception { + config.setSharedObject(StateMachineTransitionBuilder.class, getStateMachineTransitionBuilder()); + config.setSharedObject(StateMachineStateBuilder.class, getStateMachineStateBuilder()); + } + + @Override + public void configure(StateMachineConfigBuilder config) throws Exception { + } + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + } + + @Override + public boolean isAssignable(AnnotationBuilder> builder) { + return builder instanceof StateMachineConfigBuilder; + } + + protected final StateMachineTransitionBuilder getStateMachineTransitionBuilder() throws Exception { + if (transitionBuilder != null) { + return transitionBuilder; + } + transitionBuilder = new StateMachineTransitionBuilder(ObjectPostProcessor.QUIESCENT_POSTPROCESSOR, true); + configure(transitionBuilder); + return transitionBuilder; + } + + protected final StateMachineStateBuilder getStateMachineStateBuilder() throws Exception { + if (stateBuilder != null) { + return stateBuilder; + } + stateBuilder = new StateMachineStateBuilder(ObjectPostProcessor.QUIESCENT_POSTPROCESSOR, true); + configure(stateBuilder); + return stateBuilder; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineFactory.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineFactory.java new file mode 100644 index 000000000..c462e26d8 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/StateMachineFactory.java @@ -0,0 +1,24 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config; + +import org.springframework.yarn.support.statemachine.StateMachine; + +public interface StateMachineFactory { + + StateMachine getStateMachine(); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigBuilder.java new file mode 100644 index 000000000..b7b89a253 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigBuilder.java @@ -0,0 +1,36 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import org.springframework.data.hadoop.config.common.annotation.AbstractConfiguredAnnotationBuilder; +import org.springframework.yarn.support.statemachine.config.StateMachineConfig; + +public class StateMachineConfigBuilder + extends + AbstractConfiguredAnnotationBuilder, StateMachineConfigBuilder, StateMachineConfigBuilder> { + + @SuppressWarnings("unchecked") + @Override + protected StateMachineConfig performBuild() throws Exception { + StateMachineTransitionBuilder sharedObject = getSharedObject(StateMachineTransitionBuilder.class); + StateMachineStateBuilder sharedObject2 = getSharedObject(StateMachineStateBuilder.class); + StateMachineStates states = (StateMachineStates) sharedObject2.build(); + StateMachineTransitions transitions = (StateMachineTransitions) sharedObject.build(); + StateMachineConfig bean = new StateMachineConfig(transitions, states); + return bean; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigurer.java new file mode 100644 index 000000000..9443a6e18 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineConfigurer.java @@ -0,0 +1,28 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurer; +import org.springframework.yarn.support.statemachine.config.StateMachineConfig; + +public interface StateMachineConfigurer extends + AnnotationConfigurer, StateMachineConfigBuilder> { + + void configure(StateMachineStateConfigurer transitions) throws Exception; + + void configure(StateMachineTransitionConfigurer transitions) throws Exception; + +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateBuilder.java new file mode 100644 index 000000000..56709fb82 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateBuilder.java @@ -0,0 +1,67 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import java.util.ArrayList; +import java.util.Collection; + +import org.springframework.data.hadoop.config.common.annotation.AbstractConfiguredAnnotationBuilder; +import org.springframework.data.hadoop.config.common.annotation.ObjectPostProcessor; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates.StateData; +import org.springframework.yarn.support.statemachine.config.configurers.DefaultStateConfigurer; +import org.springframework.yarn.support.statemachine.config.configurers.StateConfigurer; + +public class StateMachineStateBuilder + extends AbstractConfiguredAnnotationBuilder, StateMachineStateConfigurer, StateMachineStateBuilder> + implements StateMachineStateConfigurer { + + private Collection> states = new ArrayList>(); + private S initialState; + + public StateMachineStateBuilder() { + super(); + } + + public StateMachineStateBuilder(ObjectPostProcessor objectPostProcessor, + boolean allowConfigurersOfSameType) { + super(objectPostProcessor, allowConfigurersOfSameType); + } + + public StateMachineStateBuilder(ObjectPostProcessor objectPostProcessor) { + super(objectPostProcessor); + } + + @Override + protected StateMachineStates performBuild() throws Exception { + StateMachineStates bean = new StateMachineStates(initialState, states); + return bean; + } + + @Override + public StateConfigurer withStates() throws Exception { + return apply(new DefaultStateConfigurer()); + } + + public void add(Collection> states) { + this.states.addAll(states); + } + + public void setInitialState(S state) { + this.initialState = state; + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateConfigurer.java new file mode 100644 index 000000000..360ec72bd --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStateConfigurer.java @@ -0,0 +1,24 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import org.springframework.yarn.support.statemachine.config.configurers.StateConfigurer; + +public interface StateMachineStateConfigurer { + + StateConfigurer withStates() throws Exception; + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStates.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStates.java new file mode 100644 index 000000000..3a67dd38b --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineStates.java @@ -0,0 +1,59 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; + +public class StateMachineStates { + + private Collection> states; + + private final S initialState; + + public StateMachineStates(S initialState, Collection> states) { + this.states = states; + this.initialState = initialState; + } + + public Collection> getStates() { + return states; + } + + public S getInitialState() { + return initialState; + } + + public static class StateData { + private S state; + private Collection deferred; + public StateData(S state, Collection deferred) { + this.state = state; + this.deferred = deferred; + } + public StateData(S state, E[] deferred) { + this(state, deferred != null ? Arrays.asList(deferred) : new ArrayList()); + } + public S getState() { + return state; + } + public Collection getDeferred() { + return deferred; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionBuilder.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionBuilder.java new file mode 100644 index 000000000..599f32bd0 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionBuilder.java @@ -0,0 +1,63 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import java.util.ArrayList; +import java.util.Collection; + +import org.springframework.data.hadoop.config.common.annotation.AbstractConfiguredAnnotationBuilder; +import org.springframework.data.hadoop.config.common.annotation.ObjectPostProcessor; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions.TransitionData; +import org.springframework.yarn.support.statemachine.config.configurers.DefaultExternalTransitionConfigurer; +import org.springframework.yarn.support.statemachine.config.configurers.ExternalTransitionConfigurer; + +public class StateMachineTransitionBuilder + extends + AbstractConfiguredAnnotationBuilder, StateMachineTransitionConfigurer, StateMachineTransitionBuilder> + implements StateMachineTransitionConfigurer { + + private Collection> transitionData = new ArrayList>(); + + public StateMachineTransitionBuilder() { + super(); + } + + public StateMachineTransitionBuilder(ObjectPostProcessor objectPostProcessor, + boolean allowConfigurersOfSameType) { + super(objectPostProcessor, allowConfigurersOfSameType); + } + + public StateMachineTransitionBuilder(ObjectPostProcessor objectPostProcessor) { + super(objectPostProcessor); + } + + @Override + protected StateMachineTransitions performBuild() throws Exception { + StateMachineTransitions bean = new StateMachineTransitions(transitionData); + return bean; + } + + @Override + public ExternalTransitionConfigurer withExternal() throws Exception { + return apply(new DefaultExternalTransitionConfigurer()); + } + + public void add(S source, S target, E event, Collection actions) { + transitionData.add(new TransitionData(source, target, event, actions)); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionConfigurer.java new file mode 100644 index 000000000..24276184b --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitionConfigurer.java @@ -0,0 +1,24 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import org.springframework.yarn.support.statemachine.config.configurers.ExternalTransitionConfigurer; + +public interface StateMachineTransitionConfigurer { + + ExternalTransitionConfigurer withExternal() throws Exception; + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitions.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitions.java new file mode 100644 index 000000000..65545e5e2 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/builders/StateMachineTransitions.java @@ -0,0 +1,59 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.builders; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.action.Action; + +public class StateMachineTransitions { + + private Collection> transitions; + + public StateMachineTransitions(Collection> transitions) { + this.transitions = transitions; + } + + public Collection> getTransitions() { + return transitions; + } + + public static class TransitionData { + S source; + S target; + E event; + Collection actions; + public TransitionData(S source, S target, E event, Collection actions) { + this.source = source; + this.target = target; + this.event = event; + this.actions = actions; + } + public S getSource() { + return source; + } + public S getTarget() { + return target; + } + public E getEvent() { + return event; + } + public Collection getActions() { + return actions; + } + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineConfiguration.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineConfiguration.java new file mode 100644 index 000000000..bd40ce667 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineConfiguration.java @@ -0,0 +1,60 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.configuration; + +import java.util.List; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.hadoop.config.common.annotation.AbstractAnnotationConfiguration; +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurer; +import org.springframework.yarn.support.statemachine.StateMachine; +import org.springframework.yarn.support.statemachine.StateMachineSystemConstants; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineFactory; +import org.springframework.yarn.support.statemachine.config.StateMachineConfig; +import org.springframework.yarn.support.statemachine.config.StateMachineFactory; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineConfigBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions; +import org.springframework.yarn.support.statemachine.state.State; + +@Configuration +public class StateMachineConfiguration, E extends Enum> extends + AbstractAnnotationConfiguration, StateMachineConfig> { + + StateMachineConfigBuilder builder = new StateMachineConfigBuilder(); + + @Bean(name = StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE) + public StateMachine, E> stateMachine() { + + StateMachineConfig stateMachineConfig = builder.getOrBuild(); + StateMachineTransitions stateMachineTransitions = stateMachineConfig.getTransitions(); + StateMachineStates stateMachineStates = stateMachineConfig.getStates(); + + StateMachineFactory, E> stateMachineFactory = new EnumStateMachineFactory(stateMachineTransitions, stateMachineStates); + return stateMachineFactory.getStateMachine(); + } + + @Override + protected void onConfigurers(List, StateMachineConfigBuilder>> configurers) throws Exception { + for (AnnotationConfigurer, StateMachineConfigBuilder> configurer : configurers) { + if (configurer.isAssignable(builder)) { + builder.apply(configurer); + } + } + } + +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineFactoryConfiguration.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineFactoryConfiguration.java new file mode 100644 index 000000000..b79be2b92 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configuration/StateMachineFactoryConfiguration.java @@ -0,0 +1,59 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.configuration; + +import java.util.List; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.hadoop.config.common.annotation.AbstractAnnotationConfiguration; +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurer; +import org.springframework.yarn.support.statemachine.StateMachineSystemConstants; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineFactory; +import org.springframework.yarn.support.statemachine.config.StateMachineConfig; +import org.springframework.yarn.support.statemachine.config.StateMachineFactory; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineConfigBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions; +import org.springframework.yarn.support.statemachine.state.State; + +@Configuration +public class StateMachineFactoryConfiguration, E extends Enum> extends + AbstractAnnotationConfiguration, StateMachineConfig> { + + StateMachineConfigBuilder builder = new StateMachineConfigBuilder(); + + @Bean(name = StateMachineSystemConstants.DEFAULT_ID_STATEMACHINEFACTORY) + public StateMachineFactory, E> stateMachineFactory() { + + StateMachineConfig stateMachineConfig = builder.getOrBuild(); + StateMachineTransitions stateMachineTransitions = stateMachineConfig.getTransitions(); + StateMachineStates stateMachineStates = stateMachineConfig.getStates(); + + StateMachineFactory, E> stateMachineFactory = new EnumStateMachineFactory(stateMachineTransitions, stateMachineStates); + return stateMachineFactory; + } + + @Override + protected void onConfigurers(List, StateMachineConfigBuilder>> configurers) throws Exception { + for (AnnotationConfigurer, StateMachineConfigBuilder> configurer : configurers) { + if (configurer.isAssignable(builder)) { + builder.apply(configurer); + } + } + } + +} \ No newline at end of file diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultExternalTransitionConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultExternalTransitionConfigurer.java new file mode 100644 index 000000000..5137b8c34 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultExternalTransitionConfigurer.java @@ -0,0 +1,68 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.configurers; + +import java.util.ArrayList; +import java.util.Collection; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurerAdapter; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitions; + +public class DefaultExternalTransitionConfigurer + extends AnnotationConfigurerAdapter, StateMachineTransitionConfigurer, StateMachineTransitionBuilder> + implements ExternalTransitionConfigurer { + + private S source; + + private S target; + + private E event; + + private Collection actions = new ArrayList(); + + @Override + public void configure(StateMachineTransitionBuilder builder) throws Exception { + builder.add(source, target, event, actions); + } + + @Override + public ExternalTransitionConfigurer source(S source) { + this.source = source; + return this; + } + + @Override + public ExternalTransitionConfigurer target(S target) { + this.target = target; + return this; + } + + @Override + public ExternalTransitionConfigurer event(E event) { + this.event = event; + return this; + } + + @Override + public ExternalTransitionConfigurer action(Action action) { + actions.add(action); + return this; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultStateConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultStateConfigurer.java new file mode 100644 index 000000000..b190d95ba --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/DefaultStateConfigurer.java @@ -0,0 +1,67 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.configurers; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStates.StateData; + +public class DefaultStateConfigurer + extends AnnotationConfigurerAdapter, StateMachineStateConfigurer, StateMachineStateBuilder> + implements StateConfigurer { + + private final Collection> states = new ArrayList>(); + + private S initial; + + @Override + public void configure(StateMachineStateBuilder builder) throws Exception { + builder.add(states); + builder.setInitialState(initial); + } + + @Override + public StateConfigurer initial(S initial) { + this.initial = initial; + return this; + } + + @Override + public StateConfigurer state(S state) { + return state(state, (E[])null); + } + + @Override + public StateConfigurer state(S state, E... deferred) { + states.add(new StateData(state, deferred)); + return this; + } + + @Override + public StateConfigurer states(Set states) { + for (S s : states) { + state(s); + } + return this; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/ExternalTransitionConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/ExternalTransitionConfigurer.java new file mode 100644 index 000000000..28f1a2a5f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/ExternalTransitionConfigurer.java @@ -0,0 +1,33 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.configurers; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurerBuilder; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +public interface ExternalTransitionConfigurer extends + AnnotationConfigurerBuilder> { + + ExternalTransitionConfigurer source(S source); + + ExternalTransitionConfigurer target(S source); + + ExternalTransitionConfigurer event(E event); + + ExternalTransitionConfigurer action(Action action); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/StateConfigurer.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/StateConfigurer.java new file mode 100644 index 000000000..e0fbc8c48 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/config/configurers/StateConfigurer.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.config.configurers; + +import java.util.Set; + +import org.springframework.data.hadoop.config.common.annotation.AnnotationConfigurerBuilder; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; + +public interface StateConfigurer extends + AnnotationConfigurerBuilder> { + + StateConfigurer initial(S initial); + + StateConfigurer state(S state); + + StateConfigurer state(S state, E... deferred); + + StateConfigurer states(Set states); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/AbstractState.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/AbstractState.java new file mode 100644 index 000000000..0e0844b99 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/AbstractState.java @@ -0,0 +1,49 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.state; + +import java.util.Collection; + +public abstract class AbstractState implements State { + + private S id; + private Collection deferred; + + public AbstractState(S id) { + this.id = id; + } + + public AbstractState(S id, Collection deferred) { + this.id = id; + this.deferred = deferred; + } + + @Override + public S getId() { + return id; + } + + @Override + public Collection getDeferredEvents() { + return deferred; + } + + @Override + public String toString() { + return "AbstractState [id=" + id + ", deferred=" + deferred + "]"; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/EnumState.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/EnumState.java new file mode 100644 index 000000000..2d931fcbd --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/EnumState.java @@ -0,0 +1,36 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.state; + +import java.util.Collection; + +public class EnumState, E extends Enum> extends AbstractState { + + public EnumState(S id) { + super(id); + } + + public EnumState(S id, Collection deferred) { + super(id, deferred); + } + + @Override + public String toString() { + return "EnumState [getId()=" + getId() + ", getClass()=" + getClass() + ", hashCode()=" + hashCode() + + ", toString()=" + super.toString() + "]"; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/State.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/State.java new file mode 100644 index 000000000..fd764262f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/state/State.java @@ -0,0 +1,26 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.state; + +import java.util.Collection; + +public interface State { + + S getId(); + + Collection getDeferredEvents(); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractExternalTransition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractExternalTransition.java new file mode 100644 index 000000000..fcf512278 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractExternalTransition.java @@ -0,0 +1,30 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; + +public abstract class AbstractExternalTransition extends AbstractTransition implements Transition { + + public AbstractExternalTransition(State source, State target, Collection actions, E event) { + super(source, target, actions, event, TransitionKind.EXTERNAL); + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractInternalTransition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractInternalTransition.java new file mode 100644 index 000000000..a1d7aeb7e --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractInternalTransition.java @@ -0,0 +1,29 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; + +public class AbstractInternalTransition extends AbstractTransition implements Transition { + + public AbstractInternalTransition(State source,Collection actions, E event) { + super(source, source, actions, event, TransitionKind.INTERNAL); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractLocalTransition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractLocalTransition.java new file mode 100644 index 000000000..5a50487b0 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractLocalTransition.java @@ -0,0 +1,29 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; + +public class AbstractLocalTransition extends AbstractTransition implements Transition { + + public AbstractLocalTransition(State source, State target,Collection actions, E event) { + super(source, target, actions, event, TransitionKind.LOCAL); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractTransition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractTransition.java new file mode 100644 index 000000000..dda5366af --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/AbstractTransition.java @@ -0,0 +1,86 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.util.Assert; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; +import org.springframework.yarn.support.statemachine.trigger.EventTrigger; +import org.springframework.yarn.support.statemachine.trigger.Trigger; + +public class AbstractTransition implements Transition { + + private final State source; + + private final State target; + + private final Collection actions; + + private Trigger trigger; + + private final TransitionKind kind; + + public AbstractTransition(State source, State target, Collection actions, E event, TransitionKind kind) { + Assert.notNull(source, "Source must be set"); + Assert.notNull(target, "Target must be set"); + Assert.notNull(kind, "Transition type must be set"); + this.source = source; + this.target = target; + this.actions = actions; + if (event != null) { + this.trigger = new EventTrigger(event); + } + this.kind = kind; + } + + @Override + public State getSource() { + return source; + } + + @Override + public State getTarget() { + return target; + } + + @Override + public Collection getActions() { + return actions; + } + + @Override + public Trigger getTrigger() { + return trigger; + } + + @Override + public void transit(MessageHeaders headers) { + if (actions != null) { + for (Action action : actions) { + action.execute(headers); + } + } + } + + @Override + public TransitionKind getKind() { + return kind; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultExternalTransition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultExternalTransition.java new file mode 100644 index 000000000..2f89e09c0 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultExternalTransition.java @@ -0,0 +1,29 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; + +public class DefaultExternalTransition extends AbstractExternalTransition { + + public DefaultExternalTransition(State source, State target, Collection actions, E event) { + super(source, target, actions, event); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultInternalTransition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultInternalTransition.java new file mode 100644 index 000000000..8e9b9e7be --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/DefaultInternalTransition.java @@ -0,0 +1,29 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; + +public class DefaultInternalTransition extends AbstractInternalTransition { + + public DefaultInternalTransition(State source, Collection actions, E event) { + super(source, actions, event); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/Transition.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/Transition.java new file mode 100644 index 000000000..6280b8ecd --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/Transition.java @@ -0,0 +1,45 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +import java.util.Collection; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.State; +import org.springframework.yarn.support.statemachine.trigger.Trigger; + +/** + * The Interface Transition. + * + * @param the generic type + * @param the element type + */ +public interface Transition { + + State getSource(); + + State getTarget(); + + Collection getActions(); + + void transit(MessageHeaders headers); + + Trigger getTrigger(); + + TransitionKind getKind(); + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/TransitionKind.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/TransitionKind.java new file mode 100644 index 000000000..f147bc221 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/transition/TransitionKind.java @@ -0,0 +1,26 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.transition; + +public enum TransitionKind { + + EXTERNAL, + + INTERNAL, + + LOCAL + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/EventTrigger.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/EventTrigger.java new file mode 100644 index 000000000..0ebeeb715 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/EventTrigger.java @@ -0,0 +1,31 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.trigger; + +public class EventTrigger implements Trigger { + + private final E event; + + public EventTrigger(E event) { + this.event = event; + } + + @Override + public boolean evaluate(E event) { + return this.event.equals(event); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/Trigger.java b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/Trigger.java new file mode 100644 index 000000000..98781ef4e --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/main/java/org/springframework/yarn/support/statemachine/trigger/Trigger.java @@ -0,0 +1,22 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.support.statemachine.trigger; + +public interface Trigger { + + boolean evaluate(E event); + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/MockUtils.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/MockUtils.java new file mode 100644 index 000000000..de7f20832 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/MockUtils.java @@ -0,0 +1,90 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; + +public abstract class MockUtils { + + public static ContainerStatus getMockContainerStatus(ContainerId containerId, ContainerState containerState, int exitStatus) { + ContainerStatus status = mock(ContainerStatus.class); + when(status.getContainerId()).thenReturn(containerId); + when(status.getState()).thenReturn(containerState); + when(status.getExitStatus()).thenReturn(exitStatus); + return status; + } + + public static ApplicationId getMockApplicationId(int appId) { + ApplicationId applicationId = mock(ApplicationId.class); + when(applicationId.getClusterTimestamp()).thenReturn(0L); + when(applicationId.getId()).thenReturn(appId); + return applicationId; + } + + public static Container getMockContainer(ContainerId containerId, NodeId nodeId, Resource resource, + Priority priority) { + Container container = mock(Container.class); + when(container.getId()).thenReturn(containerId); + when(container.getNodeId()).thenReturn(nodeId); + when(container.getResource()).thenReturn(resource); + when(container.getPriority()).thenReturn(priority); + return container; + } + + public static ContainerId getMockContainerId(ApplicationAttemptId applicationAttemptId, int id) { + ContainerId containerId = mock(ContainerId.class); + doReturn(applicationAttemptId).when(containerId).getApplicationAttemptId(); + doReturn(id).when(containerId).getId(); + doReturn(Integer.toString(id)).when(containerId).toString(); + return containerId; + } + + public static Priority getMockPriority(int priority) { + Priority pri = mock(Priority.class); + when(pri.getPriority()).thenReturn(priority); + return pri; + } + + public static NodeId getMockNodeId(String host, int port) { + NodeId nodeId = mock(NodeId.class); + doReturn(host).when(nodeId).getHost(); + doReturn(port).when(nodeId).getPort(); + return nodeId; + } + + public static ApplicationAttemptId getMockApplicationAttemptId(int appId, int attemptId) { + ApplicationId applicationId = mock(ApplicationId.class); + when(applicationId.getClusterTimestamp()).thenReturn(0L); + when(applicationId.getId()).thenReturn(appId); + ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class); + when(applicationAttemptId.getApplicationId()).thenReturn(applicationId); + when(applicationAttemptId.getAttemptId()).thenReturn(attemptId); + return applicationAttemptId; + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/TestUtils.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/TestUtils.java index 94742ef39..33a5343fb 100644 --- a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/TestUtils.java +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/TestUtils.java @@ -60,6 +60,18 @@ public static T callMethod(String name, Object target) throws Exception { return (T) ReflectionUtils.invokeMethod(method, target); } + @SuppressWarnings("unchecked") + public static T callMethod(String name, Object target, Object[] args, Class[] argsTypes) throws Exception { + Class clazz = target.getClass(); + Method method = ReflectionUtils.findMethod(clazz, name, argsTypes); + + if (method == null) + throw new IllegalArgumentException("Cannot find method '" + method + "' in the class hierarchy of " + + target.getClass()); + method.setAccessible(true); + return (T) ReflectionUtils.invokeMethod(method, target, args); + } + public static void setField(String name, Object target, Object value) throws Exception { Field field = null; Class clazz = target.getClass(); diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/allocate/DefaultContainerAllocatorTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/allocate/DefaultContainerAllocatorTests.java index 82d0bf52f..659b60c42 100644 --- a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/allocate/DefaultContainerAllocatorTests.java +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/allocate/DefaultContainerAllocatorTests.java @@ -19,6 +19,8 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; +import java.io.Serializable; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -36,9 +38,9 @@ public class DefaultContainerAllocatorTests { @Test public void testDefaultNoRequests() throws Exception { - DefaultAllocateCountTracker allocateCountTracker = new DefaultAllocateCountTracker(new Configuration()); DefaultContainerAllocator allocator = new DefaultContainerAllocator(); - TestUtils.setField("allocateCountTracker", allocator, allocateCountTracker); + allocator.setConfiguration(new Configuration()); + TestUtils.callMethod("internalInit", allocator); List createRequests = TestUtils.callMethod("createRequests", allocator); assertThat(createRequests, notNullValue()); @@ -51,10 +53,10 @@ public void testDefaultNoRequests() throws Exception { @Test public void testAnyHostNoLocalityRequests() throws Exception { - DefaultAllocateCountTracker allocateCountTracker = new DefaultAllocateCountTracker(new Configuration()); DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); allocator.setLocality(false); - TestUtils.setField("allocateCountTracker", allocator, allocateCountTracker); + TestUtils.callMethod("internalInit", allocator); ContainerAllocateData data = new ContainerAllocateData(); data.addAny(2); @@ -73,10 +75,10 @@ public void testAnyHostNoLocalityRequests() throws Exception { @Test public void testOneHostLocalityRequests() throws Exception { - DefaultAllocateCountTracker allocateCountTracker = new DefaultAllocateCountTracker(new Configuration()); DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); allocator.setLocality(true); - TestUtils.setField("allocateCountTracker", allocator, allocateCountTracker); + TestUtils.callMethod("internalInit", allocator); ContainerAllocateData data = new ContainerAllocateData(); data.addHosts("host1", 1); @@ -107,10 +109,10 @@ public void testOneHostLocalityRequests() throws Exception { @Test public void testOneRackLocalityRequests() throws Exception { - DefaultAllocateCountTracker allocateCountTracker = new DefaultAllocateCountTracker(new Configuration()); DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); allocator.setLocality(true); - TestUtils.setField("allocateCountTracker", allocator, allocateCountTracker); + TestUtils.callMethod("internalInit", allocator); ContainerAllocateData data = new ContainerAllocateData(); data.addRacks("/default-rack", 1); @@ -135,10 +137,10 @@ public void testOneRackLocalityRequests() throws Exception { @Test public void testOneHostNoLocalityRequests() throws Exception { - DefaultAllocateCountTracker allocateCountTracker = new DefaultAllocateCountTracker(new Configuration()); DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); allocator.setLocality(false); - TestUtils.setField("allocateCountTracker", allocator, allocateCountTracker); + TestUtils.callMethod("internalInit", allocator); ContainerAllocateData data = new ContainerAllocateData(); data.addHosts("host1", 1); @@ -167,4 +169,136 @@ public void testOneHostNoLocalityRequests() throws Exception { assertThat(req2.getRelaxLocality(), is(true)); } + @Test + public void testMixedHostRequests() throws Exception { + DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); + allocator.setAllocationValues("cluster1", 1, 1, 64, false); + allocator.setAllocationValues("cluster2", 2, 2, 128, true); + TestUtils.callMethod("internalInit", allocator); + + ContainerAllocateData data1 = new ContainerAllocateData(); + data1.setId("cluster1"); + data1.addHosts("host10", 1); + allocator.allocateContainers(data1); + + ContainerAllocateData data2 = new ContainerAllocateData(); + data2.setId("cluster2"); + data2.addHosts("host20", 1); + allocator.allocateContainers(data2); + + List createRequests = TestUtils.callMethod("createRequests", allocator); + Collections.sort(createRequests, new ResourceRequestComparator()); + assertThat(createRequests, notNullValue()); + assertThat(createRequests.size(), is(7)); + + assertResourceRequest(createRequests.get(0), "*", 2, 2, false, 128, 2); + assertResourceRequest(createRequests.get(1), "/default-rack", 2, 1, false, 128, 2); + assertResourceRequest(createRequests.get(2), "host20", 2, 1, true, 128, 2); + assertResourceRequest(createRequests.get(3), "*", 1, 2, true, 64, 1); + assertResourceRequest(createRequests.get(4), "/default-rack", 1, 1, true, 64, 1); + assertResourceRequest(createRequests.get(5), "host10", 1, 1, true, 64, 1); + assertResourceRequest(createRequests.get(6), "*", 0, 0, true, 64, 1); + } + + @Test + public void testMixedAnyAndHostRequests() throws Exception { + DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); + allocator.setAllocationValues("cluster1", 1, 1, 64, false); + allocator.setAllocationValues("cluster2", 2, 2, 128, true); + TestUtils.callMethod("internalInit", allocator); + + ContainerAllocateData data1 = new ContainerAllocateData(); + data1.setId("cluster1"); + data1.addAny(1); + allocator.allocateContainers(data1); + + ContainerAllocateData data2 = new ContainerAllocateData(); + data2.setId("cluster2"); + data2.addHosts("host20", 1); + allocator.allocateContainers(data2); + + List createRequests = TestUtils.callMethod("createRequests", allocator); + Collections.sort(createRequests, new ResourceRequestComparator()); + assertThat(createRequests, notNullValue()); + assertThat(createRequests.size(), is(5)); + + assertResourceRequest(createRequests.get(0), "*", 2, 2, false, 128, 2); + assertResourceRequest(createRequests.get(1), "/default-rack", 2, 1, false, 128, 2); + assertResourceRequest(createRequests.get(2), "host20", 2, 1, true, 128, 2); + assertResourceRequest(createRequests.get(3), "*", 1, 1, true, 64, 1); + assertResourceRequest(createRequests.get(4), "*", 0, 0, true, 64, 1); + } + + @Test + public void testFallbackToDefaultsRequests() throws Exception { + DefaultContainerAllocator allocator = new DefaultContainerAllocator(); + allocator.setConfiguration(new Configuration()); + TestUtils.callMethod("internalInit", allocator); + + ContainerAllocateData data1 = new ContainerAllocateData(); + data1.setId("cluster1"); + data1.addAny(1); + allocator.allocateContainers(data1); + + List createRequests = TestUtils.callMethod("createRequests", allocator); + Collections.sort(createRequests, new ResourceRequestComparator()); + assertThat(createRequests, notNullValue()); + assertThat(createRequests.size(), is(1)); + + assertResourceRequest(createRequests.get(0), "*", 0, 1, true, 64, 1); + } + + private static void assertResourceRequest(ResourceRequest req, String resourceName, Integer priority, Integer numContainers, Boolean relaxLocality, Integer capMemory, Integer capCores) { + if (resourceName != null) { + assertThat(req.getResourceName(), is(resourceName)); + } + if (priority != null) { + assertThat(req.getPriority().getPriority(), is(priority)); + } + if (numContainers != null) { + assertThat(req.getNumContainers(), is(numContainers)); + } + if (relaxLocality != null) { + assertThat(req.getRelaxLocality(), is(relaxLocality)); + } + if (capMemory != null) { + assertThat(req.getCapability().getMemory(), is(capMemory)); + } + if (capCores != null) { + assertThat(req.getCapability().getVirtualCores(), is(capCores)); + } + } + + /** + * Comparator used to sort requests so we get expected ordering for asserts. + */ + public static class ResourceRequestComparator implements java.util.Comparator, Serializable { + + private static final long serialVersionUID = 1L; + + @Override + public int compare(ResourceRequest r1, ResourceRequest r2) { + + // Compare priority, host and capability + int ret = r1.getPriority().compareTo(r2.getPriority()); + if (ret == 0) { + String h1 = r1.getResourceName(); + String h2 = r2.getResourceName(); + ret = h1.compareTo(h2); + } + if (ret == 0) { + ret = r1.getCapability().compareTo(r2.getCapability()); + } + if (ret == 0) { + ret = Integer.compare(r1.getNumContainers(), r2.getNumContainers()); + } + if (ret == 0) { + Boolean.compare(r1.getRelaxLocality(), r2.getRelaxLocality()); + } + return ret; + } + } + } diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/AbstractManagedContainerClusterAppmasterTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/AbstractManagedContainerClusterAppmasterTests.java new file mode 100644 index 000000000..d912e4b63 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/AbstractManagedContainerClusterAppmasterTests.java @@ -0,0 +1,263 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.junit.After; +import org.junit.Before; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.yarn.MockUtils; +import org.springframework.yarn.TestUtils; +import org.springframework.yarn.am.allocate.ContainerAllocateData; +import org.springframework.yarn.am.allocate.ContainerAllocator; +import org.springframework.yarn.am.container.ContainerLauncher; +import org.springframework.yarn.am.grid.support.DefaultGridProjectionFactory; +import org.springframework.yarn.am.grid.support.SatisfyStateData; +import org.springframework.yarn.listener.ContainerAllocatorListener; +import org.springframework.yarn.support.statemachine.StateMachineSystemConstants; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineFactory; + +public abstract class AbstractManagedContainerClusterAppmasterTests { + + private AnnotationConfigApplicationContext ctx; + protected EnumStateMachineFactory stateMachineFactory; + + protected Container allocateContainer(Object appmaster, int id) throws Exception { + return allocateContainer(appmaster, id, null); + } + + protected Container allocateContainer(Object appmaster, int id, String host) throws Exception { + ContainerId containerId = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 0); + NodeId nodeId = MockUtils.getMockNodeId(host, 0); + Priority priority = MockUtils.getMockPriority(0); + Container container = MockUtils.getMockContainer(containerId, nodeId, null, priority); + TestUtils.callMethod("onContainerAllocated", appmaster, new Object[]{container}, new Class[]{Container.class}); + return container; + } + + protected void releaseContainer(Object appmaster, Container container) throws Exception { + ContainerStatus containerStatus = MockUtils.getMockContainerStatus(container.getId(), ContainerState.COMPLETE, 0); + TestUtils.callMethod("onContainerCompleted", appmaster, new Object[]{containerStatus}, new Class[]{ContainerStatus.class}); + } + + protected void assertDoTask(TestManagedContainerClusterAppmaster appmaster, Integer anySize, Integer hostsSize, Integer racksSize, Integer killSize, String cluster) throws Exception { + assertDoTask(appmaster, anySize, hostsSize, racksSize, killSize, cluster, false); + } + + protected void assertDoTask(TestManagedContainerClusterAppmaster appmaster, Integer anySize, Integer hostsSize, Integer racksSize, Integer killSize, String cluster, boolean notExist) throws Exception { + TestUtils.callMethod("doTask", appmaster); + if (notExist) { + assertThat(appmaster.getSatisfyStateDataByCluster(cluster), nullValue()); + return; + } + if (anySize != null || hostsSize != null || racksSize != null) { + assertThat(appmaster.getSatisfyStateDataByCluster(cluster).getAllocateData(), notNullValue()); + + } + if (anySize != null) { + assertThat(appmaster.getSatisfyStateDataByCluster(cluster).getAllocateData().getAny(), is(anySize)); + } + if (hostsSize != null) { + assertThat(appmaster.getSatisfyStateDataByCluster(cluster).getAllocateData().getHosts().size(), is(hostsSize)); + } + if (racksSize != null) { + assertThat(appmaster.getSatisfyStateDataByCluster(cluster).getAllocateData().getRacks().size(), is(racksSize)); + } + if (killSize != null) { + assertThat(appmaster.getSatisfyStateDataByCluster(cluster).getRemoveData(), notNullValue()); + assertThat(appmaster.getSatisfyStateDataByCluster(cluster).getRemoveData().size(), is(killSize)); + } else { +// assertThat(appmaster.getSatisfyStateDataByCluster(cluster), nullValue()); +// assertThat(appmaster.getSatisfyStateDataByCluster(cluster), notNullValue()); + } + } + +// protected static void assertSatisfyStateData(Object appmaster, Integer mapSize, Integer anySize, Integer hostsSize, Integer racksSize) throws Exception { +// Map map = TestUtils.callMethod("getSatisfyStateData", appmaster); +// assertThat(map.size(), is(mapSize)); +// SatisfyStateData data = null; +// if (mapSize != null && mapSize > 0) { +// data = map.entrySet().iterator().next().getValue(); +// } +// if (anySize != null) { +// assertThat(data.getAllocateData().getAny(), is(anySize)); +// } +// if (hostsSize != null) { +// assertThat(data.getAllocateData().getHosts().size(), is(hostsSize)); +// } +// if (racksSize != null) { +// assertThat(data.getAllocateData().getRacks().size(), is(racksSize)); +// } +// } + + protected static ManagedContainerClusterAppmaster createManagedAppmaster() throws Exception { + ManagedContainerClusterAppmaster appmaster = new ManagedContainerClusterAppmaster(); + appmaster.setConfiguration(new Configuration()); + appmaster.setGridProjectionFactory(new DefaultGridProjectionFactory(null)); + TestUtils.callMethod("onInit", appmaster); + return appmaster; + } + + @SuppressWarnings("unchecked") + @Before + public void setup() { + ctx = new AnnotationConfigApplicationContext(ContainerClusterStateMachineConfiguration.class, TestConfig.class); + stateMachineFactory = ctx.getBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINEFACTORY, EnumStateMachineFactory.class); + } + + @After + public void clean() { + if (ctx != null) { + try { + ctx.close(); + } catch (Exception e) { + } + ctx = null; + } + } + + @org.springframework.context.annotation.Configuration + public static class TestConfig { + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + } + + protected static TestManagedContainerClusterAppmaster createTestAppmaster(TestContainerAllocator allocator, TestContainerLauncher launcher) throws Exception { + TestManagedContainerClusterAppmaster appmaster = new TestManagedContainerClusterAppmaster(); + appmaster.setTaskExecutor(new SyncTaskExecutor()); + appmaster.setConfiguration(new Configuration()); + appmaster.setAllocator(allocator); + appmaster.setLauncher(launcher); + appmaster.setGridProjectionFactory(new DefaultGridProjectionFactory(null)); + TestUtils.callMethod("onInit", appmaster); + return appmaster; + } + + protected static class TestManagedContainerClusterAppmaster extends ManagedContainerClusterAppmaster { + + Map satisfyStateData = new HashMap(); + + public SatisfyStateData getSatisfyStateDataByCluster(String cluster) { + if (satisfyStateData == null) { + return null; + } + for (Entry entry : satisfyStateData.entrySet()) { + if (entry.getKey().getId().equals(cluster)) { + return entry.getValue(); + } + } + return null; + } + + public void resetTestData() { +// satisfyStateData = null; + satisfyStateData.clear();; + } + +// @Override +// protected void handleSatisfyStateData(Map satisfyData) { +// satisfyStateData = satisfyData; +// } + + @Override + protected void handleSatisfyStateData(ContainerCluster cluster, SatisfyStateData satisfyData) { + satisfyStateData.put(cluster, satisfyData); + } + + } + + protected static class TestContainerAllocator implements ContainerAllocator { + + ArrayList containerAllocateData; + ArrayList releaseContainers; + + public void resetTestData() { + containerAllocateData = null; + releaseContainers = null; + } + + @Override + public void allocateContainers(int count) { + } + + @Override + public void allocateContainers(ContainerAllocateData containerAllocateData) { + if (this.containerAllocateData == null) { + this.containerAllocateData = new ArrayList(); + } + this.containerAllocateData.add(containerAllocateData); + } + + @Override + public void releaseContainers(List containers) { + if (this.releaseContainers == null) { + this.releaseContainers = new ArrayList(); + } + this.releaseContainers.addAll(containers); + } + + @Override + public void releaseContainer(ContainerId containerId) { + } + + @Override + public void addListener(ContainerAllocatorListener listener) { + } + + @Override + public void setProgress(float progress) { + } + + } + + protected static class TestContainerLauncher implements ContainerLauncher { + + Container container; + + public void resetTestData() { + container = null; + } + + @Override + public void launchContainer(Container container, List commands) { + this.container = container; + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterMultiTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterMultiTests.java new file mode 100644 index 000000000..d71b1c59f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterMultiTests.java @@ -0,0 +1,209 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; +import org.springframework.yarn.TestUtils; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.support.ProjectionData; + +/** + * Tests for {@link ManagedContainerClusterAppmaster} using multiple clusters. + * + * @author Janne Valkealahti + * + */ +public class ManagedContainerClusterAppmasterMultiTests extends AbstractManagedContainerClusterAppmasterTests { + + @Test + public void testCreateTwoClusters() throws Exception { + ManagedContainerClusterAppmaster appmaster = createManagedAppmaster(); + appmaster.setStateMachineFactory(stateMachineFactory); + + // cluster1 + ProjectionData projectionData1 = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("cluster1", projectionData1); + + // cluster2 + ProjectionData projectionData2 = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("cluster2", projectionData2); + +// assertSatisfyStateData(appmaster, 2, null, null, null); + } + + @Test + public void testCreateStartTwoClustersDoAllocation() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + + ProjectionData projectionData1 = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("cluster1", projectionData1); + appmaster.startContainerCluster("cluster1"); + + ProjectionData projectionData2 = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("cluster2", projectionData2); + appmaster.startContainerCluster("cluster2"); + + + // should get 1 any alloc and no container kills + TestUtils.callMethod("doTask", appmaster); + assertThat(appmaster.satisfyStateData.size(), is(2)); + + assertThat(appmaster.getSatisfyStateDataByCluster("cluster1").getAllocateData().getAny(), is(1)); + assertThat(appmaster.getSatisfyStateDataByCluster("cluster2").getAllocateData().getAny(), is(1)); + assertThat(appmaster.getSatisfyStateDataByCluster("cluster1").getRemoveData().size(), is(0)); + assertThat(appmaster.getSatisfyStateDataByCluster("cluster2").getRemoveData().size(), is(0)); + + // allocate 1 + appmaster.resetTestData(); + allocateContainer(appmaster, 1); + TestUtils.callMethod("doTask", appmaster); + assertThat(launcher.container, notNullValue()); + assertThat(allocator.containerAllocateData, nullValue()); + assertThat(allocator.releaseContainers, nullValue()); + launcher.resetTestData(); + allocator.resetTestData(); +// assertThat(appmaster.satisfyStateData.size(), is(2)); +// assertThat(appmaster.satisfyStateData.get(0), nullValue()); +// assertThat(appmaster.satisfyStateData.get(1), nullValue()); + + // allocate 2 + appmaster.resetTestData(); + allocateContainer(appmaster, 1); + TestUtils.callMethod("doTask", appmaster); + assertThat(launcher.container, notNullValue()); + assertThat(allocator.containerAllocateData, nullValue()); + assertThat(allocator.releaseContainers, nullValue()); + launcher.resetTestData(); + allocator.resetTestData(); +// assertThat(appmaster.satisfyStateData.size(), is(2)); +// assertThat(appmaster.satisfyStateData.get(0), nullValue()); +// assertThat(appmaster.satisfyStateData.get(1), nullValue()); + } + + @Test + public void testCreateStartModifyTwoAnyClusters() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + + // cluster1 + ProjectionData projectionData1 = new ProjectionData(1, null, null, "any", 0); + GridProjection projection1 = appmaster.createContainerCluster("cluster1", projectionData1).getGridProjection(); + + // cluster2 + ProjectionData projectionData2 = new ProjectionData(1, null, null, "any", 0); + GridProjection projection2 = appmaster.createContainerCluster("cluster2", projectionData2).getGridProjection(); + + appmaster.startContainerCluster("cluster1"); + assertDoTask(appmaster, 1, 0, 0, 0, "cluster1"); + assertDoTask(appmaster, null, null, null, null, "cluster2"); + + appmaster.startContainerCluster("cluster2"); + assertDoTask(appmaster, 1, 0, 0, 0, "cluster2"); + assertDoTask(appmaster, null, null, null, null, "cluster1"); + + // allocate container 1 + allocateContainer(appmaster, 1); + assertThat(projection1.getMembers().size() + projection2.getMembers().size(), is(1)); +// assertDoTask(appmaster, null, null, null, null, "cluster1"); + + allocateContainer(appmaster, 2); + assertThat(projection1.getMembers().size() + projection2.getMembers().size(), is(2)); +// assertDoTask(appmaster, null, null, null, null, "cluster1"); + + allocateContainer(appmaster, 3); + assertThat(projection1.getMembers().size(), is(1)); + assertThat(projection2.getMembers().size(), is(1)); + + // modify - ramp up to 2 + appmaster.modifyContainerCluster("cluster1", new ProjectionData(2)); + assertDoTask(appmaster, 1, 0, 0, 0, "cluster1"); + + + } + + @Test + public void testCreateStartModifyTwoHostsClusters() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + + // cluster1 + Map hosts1 = new HashMap(); + hosts1.put("host10", 1); + +// ProjectionData projectionData1 = new ProjectionData(0, hosts1, null); +// HostsGridProjection projection1 = new HostsGridProjection(); +// projection1.setProjectionData(projectionData1); + ProjectionData projectionData1 = new ProjectionData(0, hosts1, null, "hosts", 0); + + +// HostsGridProjection projection1 = new HostsGridProjection(hosts1); + GridProjection projection1 = appmaster.createContainerCluster("cluster1", projectionData1).getGridProjection(); + + // cluster2 + Map hosts2 = new HashMap(); + hosts2.put("host20", 1); +// ProjectionData projectionData2 = new ProjectionData(0, hosts2, null); +// HostsGridProjection projection2 = new HostsGridProjection(); +// projection2.setProjectionData(projectionData2); +// HostsGridProjection projection2 = new HostsGridProjection(hosts2); + ProjectionData projectionData2 = new ProjectionData(0, hosts2, null, "hosts", 0); + GridProjection projection2 = appmaster.createContainerCluster("cluster2", projectionData2).getGridProjection(); + + appmaster.startContainerCluster("cluster1"); + assertDoTask(appmaster, 0, 1, 0, 0, "cluster1"); + assertDoTask(appmaster, null, null, null, null, "cluster2"); + + appmaster.startContainerCluster("cluster2"); + assertDoTask(appmaster, 0, 1, 0, 0, "cluster2"); + assertDoTask(appmaster, null, null, null, null, "cluster1"); + + // allocate container 1 + allocateContainer(appmaster, 1, "host10"); + assertThat(projection1.getMembers().size() + projection2.getMembers().size(), is(1)); +// assertDoTask(appmaster, null, null, null, null, "cluster1"); + + allocateContainer(appmaster, 2, "host20"); + assertThat(projection1.getMembers().size() + projection2.getMembers().size(), is(2)); +// assertDoTask(appmaster, null, null, null, null, "cluster1"); + + allocateContainer(appmaster, 3, "hostX1"); + assertThat(projection1.getMembers().size(), is(1)); + assertThat(projection2.getMembers().size(), is(1)); + + // modify - ramp up to 2 +// appmaster.modifyContainerCluster("cluster1", new ProjectionData(2)); +// assertDoTask(appmaster, 1, 0, 0, 0, "cluster1"); + + + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterSingleTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterSingleTests.java new file mode 100644 index 000000000..aba961db6 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/cluster/ManagedContainerClusterAppmasterSingleTests.java @@ -0,0 +1,309 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.cluster; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.api.records.Container; +import org.junit.Test; +import org.springframework.yarn.TestUtils; +import org.springframework.yarn.am.grid.GridProjection; +import org.springframework.yarn.am.grid.support.ProjectionData; +import org.springframework.yarn.am.grid.support.SatisfyStateData; + +/** + * Tests for {@link ManagedContainerClusterAppmaster} using single cluster. + * + * @author Janne Valkealahti + * + */ +public class ManagedContainerClusterAppmasterSingleTests extends AbstractManagedContainerClusterAppmasterTests { + + @Test + public void testInitialState() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + assertDoTask(appmaster, 0, 0, 0, 0, "foo", true); + } + + @Test + public void testCreateOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", null); + appmaster.createContainerCluster("cluster", projectionData); + assertDoTask(appmaster, 0, 0, 0, 0, "foo", true); + } + + @Test + public void testCreateStartOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", null); + appmaster.createContainerCluster("foo", projectionData); + appmaster.startContainerCluster("foo"); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + } + + @Test + public void testCreateStartOneClusterDoAllocation() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + + appmaster.createContainerCluster("foo", projectionData); + appmaster.startContainerCluster("foo"); + + // should get 1 any alloc and no container kills + TestUtils.callMethod("doTask", appmaster); + + assertThat(appmaster.getSatisfyStateDataByCluster("foo").getAllocateData(), notNullValue()); + assertThat(appmaster.getSatisfyStateDataByCluster("foo").getAllocateData().getAny(), is(1)); + assertThat(appmaster.getSatisfyStateDataByCluster("foo").getRemoveData(), notNullValue()); + + // this is garbage, track dirty + appmaster.resetTestData(); + TestUtils.callMethod("doTask", appmaster); + assertThat(appmaster.getSatisfyStateDataByCluster("foo"), nullValue()); + + allocateContainer(appmaster, 1); + + // should get 0 any alloc and no container kills +// assertSatisfyStateData(appmaster, 1, null, null, null); + + assertThat(launcher.container, notNullValue()); + assertThat(allocator.containerAllocateData, nullValue()); + assertThat(allocator.releaseContainers, nullValue()); + } + + @Test + public void testCreateStartStopOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + + // create + GridProjection projection = appmaster.createContainerCluster("foo", projectionData).getGridProjection(); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // start + appmaster.startContainerCluster("foo"); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + + // allocate container 1 + allocateContainer(appmaster, 1); + assertThat(projection.getMembers().size(), is(1)); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // stop + appmaster.resetTestData(); + appmaster.stopContainerCluster("foo"); + assertDoTask(appmaster, null, null, null, 1, "foo"); + } + + @Test + public void testCreateStartModifyOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + + // create + GridProjection projection = appmaster.createContainerCluster("foo", projectionData).getGridProjection(); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // start + appmaster.startContainerCluster("foo"); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + + // allocate container 1 + Container container1 = allocateContainer(appmaster, 1); + assertThat(projection.getMembers().size(), is(1)); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // modify - ramp up to 2 + appmaster.modifyContainerCluster("foo", new ProjectionData(2)); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + + // allocate container 2 + allocateContainer(appmaster, 2); + assertThat(projection.getMembers().size(), is(2)); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // modify - ramp up to 4 + appmaster.modifyContainerCluster("foo", new ProjectionData(4)); + assertDoTask(appmaster, 2, 0, 0, 0, "foo"); + + // allocate container 3 and 4 + allocateContainer(appmaster, 3); + allocateContainer(appmaster, 4); + assertThat(projection.getMembers().size(), is(4)); + assertDoTask(appmaster, null, null, null, null, "foo"); + + releaseContainer(appmaster, container1); + assertThat(projection.getMembers().size(), is(3)); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + } + + @Test + public void testReleaseShouldReAllocate() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + + // create and start + GridProjection projection = appmaster.createContainerCluster("foo", projectionData).getGridProjection(); + appmaster.startContainerCluster("foo"); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + + // allocate container 1 + Container container1 = allocateContainer(appmaster, 1); + assertThat(projection.getMembers().size(), is(1)); + appmaster.resetTestData(); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // release, should re-allocate + releaseContainer(appmaster, container1); + assertThat(projection.getMembers().size(), is(0)); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + } + + @Test + public void testCreateStartStopDestroyOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("foo", projectionData); + appmaster.startContainerCluster("foo"); + appmaster.stopContainerCluster("foo"); + assertDoTask(appmaster, 0, 0, 0, 0, "foo"); + appmaster.destroyContainerCluster("foo"); + assertThat(appmaster.getContainerClusters().size(), is(0)); + } + + @Test + public void testCreateStartStopStartOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("foo", projectionData); + appmaster.startContainerCluster("foo"); + assertThat(appmaster.getContainerClusters().get("foo").getStateMachine().getState().getId(), is(ClusterState.RUNNING)); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + appmaster.stopContainerCluster("foo"); + assertThat(appmaster.getContainerClusters().get("foo").getStateMachine().getState().getId(), is(ClusterState.STOPPED)); + assertDoTask(appmaster, 0, 0, 0, 0, "foo"); + appmaster.startContainerCluster("foo"); + assertThat(appmaster.getContainerClusters().get("foo").getStateMachine().getState().getId(), is(ClusterState.RUNNING)); + assertDoTask(appmaster, 1, 0, 0, 0, "foo"); + appmaster.destroyContainerCluster("foo"); + assertThat(appmaster.getContainerClusters().size(), is(1)); + } + + @Test + public void testCreateDestroyOneCluster() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + ProjectionData projectionData = new ProjectionData(1, null, null, "any", 0); + appmaster.createContainerCluster("foo", projectionData); + assertThat(appmaster.getContainerClusters().size(), is(1)); + appmaster.destroyContainerCluster("foo"); + assertThat(appmaster.getContainerClusters().size(), is(0)); + } + + @Test + public void testModifyClusterWithDifferentConfig() throws Exception { + TestContainerAllocator allocator = new TestContainerAllocator(); + TestContainerLauncher launcher = new TestContainerLauncher(); + TestManagedContainerClusterAppmaster appmaster = createTestAppmaster(allocator, launcher); + appmaster.setStateMachineFactory(stateMachineFactory); + Map hosts = new HashMap(); + hosts.put("host1", 2); + hosts.put("host2", 2); + ProjectionData projectionData = new ProjectionData(0, hosts, null, "hosts", 0); + + // create + GridProjection projection = appmaster.createContainerCluster("foo", projectionData).getGridProjection(); + assertDoTask(appmaster, null, null, null, null, "foo"); + + // start + appmaster.startContainerCluster("foo"); + assertDoTask(appmaster, 0, 2, 0, 0, "foo"); + + // allocate container 1 + allocateContainer(appmaster, 1, "host1"); + assertThat(projection.getMembers().size(), is(1)); + + // allocate container 2 + allocateContainer(appmaster, 2, "host1"); + assertThat(projection.getMembers().size(), is(2)); + + // allocate container 3 + allocateContainer(appmaster, 3, "host2"); + assertThat(projection.getMembers().size(), is(3)); + + // allocate container 4 + allocateContainer(appmaster, 4, "host2"); + assertThat(projection.getMembers().size(), is(4)); + + assertDoTask(appmaster, null, null, null, null, "foo"); + + // modify - set only host1 + hosts.clear(); + hosts.put("host1", 1); + appmaster.modifyContainerCluster("foo", new ProjectionData(0,hosts,null)); + + SatisfyStateData satisfyState = appmaster.getContainerClusters().get("foo").getGridProjection().getSatisfyState(); + assertThat(satisfyState.getRemoveData().size(), is(3)); + +// Map hostsfromprojection = appmaster.getContainerClusters().get("foo").getGridProjection().getProjectionData().getHosts(); +// assertThat(hostsfromprojection.get("host1"), is(1)); +// assertThat(hostsfromprojection.get("host2"), is(1)); +// +// assertDoTask(appmaster, 0, 2, 0, 0, "foo"); +// +// releaseContainer(appmaster, container1); +// assertThat(projection.getMembers().size(), is(1)); +// assertDoTask(appmaster, 0, 0, 0, 0, "foo"); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/GridTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/GridTests.java new file mode 100644 index 000000000..c7664dede --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/GridTests.java @@ -0,0 +1,41 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.junit.Test; +import org.springframework.yarn.MockUtils; +import org.springframework.yarn.am.grid.support.DefaultGrid; +import org.springframework.yarn.am.grid.support.DefaultGridMember; + +public class GridTests { + + @Test + public void testSimpleOperations() { + DefaultGrid grid = new DefaultGrid(); + ContainerId id = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 0); + Container container = MockUtils.getMockContainer(id, null, null, null); + DefaultGridMember member = new DefaultGridMember(container); + grid.addMember(member); + assertThat(grid.getMembers().size(), is(1)); + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/AnyGridProjectionTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/AnyGridProjectionTests.java new file mode 100644 index 000000000..a89f7a75f --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/AnyGridProjectionTests.java @@ -0,0 +1,122 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.junit.Test; +import org.springframework.yarn.MockUtils; +import org.springframework.yarn.TestUtils; +import org.springframework.yarn.am.grid.GridProjection; + +public class AnyGridProjectionTests { + + @Test + public void testDefaults() { + GridProjection projection = new AnyGridProjection(); + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData(), notNullValue()); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + + @Test + public void testAddOne() { + GridProjection projection = new AnyGridProjection(); + projection.setProjectionData(new ProjectionData(1)); + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(1)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData(), notNullValue()); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + + @Test + public void testAddOneAddMember() { + AnyGridProjection projection = new AnyGridProjection(); + projection.setProjectionData(new ProjectionData(1)); + projection.setPriority(0); + + ContainerId id = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 0); + Priority priority = MockUtils.getMockPriority(0); + Container container = MockUtils.getMockContainer(id, null, null, priority); + projection.acceptMember(new DefaultGridMember(container)); + + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData(), notNullValue()); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + + @Test + public void testAddTwoAddMembersRemoveMember() throws Exception { + AnyGridProjection projection = new AnyGridProjection(); + projection.setPriority(0); + projection.setProjectionData(new ProjectionData(2)); + int count = TestUtils.readField("count", projection); + assertThat(count, is(2)); + + Priority priority = MockUtils.getMockPriority(0); + + ContainerId id1 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 1); + Container container1 = MockUtils.getMockContainer(id1, null, null, priority); + DefaultGridMember member1 = new DefaultGridMember(container1); + projection.acceptMember(member1); + + ContainerId id2 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 2); + Container container2 = MockUtils.getMockContainer(id2, null, null, priority); + DefaultGridMember member2 = new DefaultGridMember(container2); + projection.acceptMember(member2); + + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(0)); + + projection.setProjectionData(new ProjectionData(1)); + count = TestUtils.readField("count", projection); + assertThat(count, is(1)); + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(1)); + + projection.removeMember(member2); + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(0)); + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/DefaultProjectedGridTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/DefaultProjectedGridTests.java new file mode 100644 index 000000000..76adc80fb --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/DefaultProjectedGridTests.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.junit.Test; +import org.springframework.yarn.MockUtils; + +public class DefaultProjectedGridTests { + + @Test + public void testSimpleOperations() { + DefaultGrid grid = new DefaultGrid(); + DefaultProjectedGrid projectedGrid = new DefaultProjectedGrid(grid); + AnyGridProjection projection = new AnyGridProjection(2); + projection.setPriority(0); + + projectedGrid.addProjection(projection); + + ContainerId id = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 0); + Priority priority = MockUtils.getMockPriority(0); + Container container = MockUtils.getMockContainer(id, null, null, priority); + DefaultGridMember member = new DefaultGridMember(container); + grid.addMember(member); + +// GridProjection projection2 = projectedGrid.getProjection("test1"); + assertThat(projection, notNullValue()); + + assertThat(projection.getMembers().size(), is(1)); + + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getAny(), is(1)); + } + + + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/HostsGridProjectionTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/HostsGridProjectionTests.java new file mode 100644 index 000000000..f330910be --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/HostsGridProjectionTests.java @@ -0,0 +1,171 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.junit.Test; +import org.springframework.yarn.MockUtils; +import org.springframework.yarn.am.grid.GridProjection; + +/** + * Tests for {@link HostsGridProjection}. + * + * @author Janne Valkealahti + * + */ +public class HostsGridProjectionTests { + + @Test + public void testDefaults() { + GridProjection projection = new HostsGridProjection(); + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData(), notNullValue()); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + + @Test + public void testAddHosts() throws Exception { + HostsGridProjection projection = new HostsGridProjection(); + + ProjectionData projectionData = new ProjectionData(); + projectionData.setHost("host1", 2); + projectionData.setHost("host2", 2); + projectionData.setHost("host3", 2); + projection.setProjectionData(projectionData); + projection.setPriority(0); + + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(3)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(0)); + + ContainerId id1 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 1); + NodeId nodeId1 = MockUtils.getMockNodeId("host1", 0); + Priority priority = MockUtils.getMockPriority(0); + Container container1 = MockUtils.getMockContainer(id1, nodeId1, null, priority); + DefaultGridMember member1 = new DefaultGridMember(container1); + boolean accepted = projection.acceptMember(member1); + assertThat(accepted, is(true)); + + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(3)); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), is(1)); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), is(2)); + assertThat(satisfyState.getAllocateData().getHosts().get("host3"), is(2)); + + projectionData = new ProjectionData(); + projectionData.setHost("host1", 0); + projectionData.setHost("host2", 0); + projectionData.setHost("host3", 0); + projection.setProjectionData(projectionData); + + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(3)); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host3"), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(1)); + + projection.removeMember(member1); + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + + @Test + public void testRampDown() throws Exception { + HostsGridProjection projection = new HostsGridProjection(); + + ProjectionData projectionData = new ProjectionData(); + projectionData.setHost("host1", 2); + projectionData.setHost("host2", 2); + projection.setProjectionData(projectionData); + projection.setPriority(0); + + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(2)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(0)); + + ContainerId id1 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 1); + NodeId nodeId1 = MockUtils.getMockNodeId("host1", 0); + Priority priority1 = MockUtils.getMockPriority(0); + Container container1 = MockUtils.getMockContainer(id1, nodeId1, null, priority1); + DefaultGridMember member1 = new DefaultGridMember(container1); + assertThat(projection.acceptMember(member1), is(true)); + + ContainerId id2 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 2); + NodeId nodeId2 = MockUtils.getMockNodeId("host1", 0); + Priority priority2 = MockUtils.getMockPriority(0); + Container container2 = MockUtils.getMockContainer(id2, nodeId2, null, priority2); + DefaultGridMember member2 = new DefaultGridMember(container2); + assertThat(projection.acceptMember(member2), is(true)); + + ContainerId id3 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 3); + NodeId nodeId3 = MockUtils.getMockNodeId("host2", 0); + Priority priority3 = MockUtils.getMockPriority(0); + Container container3 = MockUtils.getMockContainer(id3, nodeId3, null, priority3); + DefaultGridMember member3 = new DefaultGridMember(container3); + assertThat(projection.acceptMember(member3), is(true)); + + ContainerId id4 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 4); + NodeId nodeId4 = MockUtils.getMockNodeId("host2", 0); + Priority priority4 = MockUtils.getMockPriority(0); + Container container4 = MockUtils.getMockContainer(id4, nodeId4, null, priority4); + DefaultGridMember member4 = new DefaultGridMember(container4); + assertThat(projection.acceptMember(member4), is(true)); + + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(2)); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), is(0)); + + projectionData = new ProjectionData(); + projectionData.setHost("host1", 1); + projectionData.setHost("host2", 2); + projection.setProjectionData(projectionData); + + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(2)); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(1)); + + projection.removeMember(member2); + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/RacksGridProjectionTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/RacksGridProjectionTests.java new file mode 100644 index 000000000..cd75c8aa2 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/RacksGridProjectionTests.java @@ -0,0 +1,131 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.am.grid.support; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertThat; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.util.RackResolver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.yarn.MockUtils; +import org.springframework.yarn.TestUtils; +import org.springframework.yarn.am.grid.GridProjection; + +/** + * Tests for {@link RacksGridProjection}. + * + * @author Janne Valkealahti + * + */ +public class RacksGridProjectionTests { + + @Before + public void setup() throws Exception { + // try to fix some hadoop static init usage + // do it again after a test + TestUtils.setField("initCalled", new RackResolver(), false); + } + + @After + public void clean() throws Exception { + TestUtils.setField("initCalled", new RackResolver(), false); + } + + @Test + public void testDefaults() { + GridProjection projection = new RacksGridProjection(); + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(0)); + assertThat(satisfyState.getRemoveData(), notNullValue()); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + + @Test + public void testAddRacks() throws Exception { + Configuration configuration = new Configuration(); + configuration.set("net.topology.node.switch.mapping.impl", "org.springframework.yarn.am.grid.support.TestDNSToSwitchMapping"); + RacksGridProjection projection = new RacksGridProjection(configuration); + projection.setPriority(0); + + ProjectionData projectionData = new ProjectionData(); + projectionData.setRack("/rack1", 2); + projectionData.setRack("/rack2", 2); + projectionData.setRack("/rack3", 2); + projection.setProjectionData(projectionData); + + SatisfyStateData satisfyState = projection.getSatisfyState(); + assertThat(satisfyState, notNullValue()); + assertThat(satisfyState.getAllocateData(), notNullValue()); + assertThat(satisfyState.getAllocateData().getAny(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().size(), is(3)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack1"), is(2)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack2"), is(2)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack3"), is(2)); + assertThat(satisfyState.getRemoveData().size(), is(0)); + + ContainerId id1 = MockUtils.getMockContainerId(MockUtils.getMockApplicationAttemptId(0, 0), 1); + NodeId nodeId1 = MockUtils.getMockNodeId("host1", 0); + Priority priority = MockUtils.getMockPriority(0); + Container container1 = MockUtils.getMockContainer(id1, nodeId1, null, priority); + DefaultGridMember member1 = new DefaultGridMember(container1); + boolean accepted = projection.acceptMember(member1); + assertThat(accepted, is(true)); + + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), nullValue()); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), nullValue()); + assertThat(satisfyState.getAllocateData().getHosts().get("host3"), nullValue()); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack1"), is(1)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack2"), is(2)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack3"), is(2)); + + projectionData = new ProjectionData(); + projectionData.setRack("/rack1", 0); + projectionData.setRack("/rack2", 0); + projectionData.setRack("/rack3", 0); + projection.setProjectionData(projectionData); + + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getAllocateData().getHosts().size(), is(0)); + assertThat(satisfyState.getAllocateData().getHosts().get("host1"), nullValue()); + assertThat(satisfyState.getAllocateData().getHosts().get("host2"), nullValue()); + assertThat(satisfyState.getAllocateData().getHosts().get("host3"), nullValue()); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack1"), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack2"), is(0)); + assertThat(satisfyState.getAllocateData().getRacks().get("/rack3"), is(0)); + assertThat(satisfyState.getRemoveData().size(), is(1)); + + projection.removeMember(member1); + satisfyState = projection.getSatisfyState(); + assertThat(satisfyState.getRemoveData().size(), is(0)); + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/TestDNSToSwitchMapping.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/TestDNSToSwitchMapping.java new file mode 100644 index 000000000..c8c6d5e68 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/am/grid/support/TestDNSToSwitchMapping.java @@ -0,0 +1,29 @@ +package org.springframework.yarn.am.grid.support; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.net.DNSToSwitchMapping; + +public class TestDNSToSwitchMapping implements DNSToSwitchMapping { + + @Override + public List resolve(List names) { + List racks = new ArrayList(); + for (String name : names) { + if (name.startsWith("host1")) { + racks.add("/rack1"); + } else if (name.startsWith("host2")) { + racks.add("/rack2"); + } else if (name.startsWith("host3")) { + racks.add("/rack3"); + } + } + return racks; + } + + @Override + public void reloadCachedMappings() { + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/config/annotation/SpringYarnConfigurationMasterAllocatorTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/config/annotation/SpringYarnConfigurationMasterAllocatorTests.java new file mode 100644 index 000000000..36c8f02f1 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/config/annotation/SpringYarnConfigurationMasterAllocatorTests.java @@ -0,0 +1,74 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.yarn.config.annotation; + +import static org.junit.Assert.assertNotNull; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.yarn.TestUtils; +import org.springframework.yarn.am.YarnAppmaster; +import org.springframework.yarn.am.allocate.ContainerAllocator; +import org.springframework.yarn.config.annotation.EnableYarn.Enable; +import org.springframework.yarn.config.annotation.builders.YarnAppmasterConfigurer; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(loader=AnnotationConfigContextLoader.class) +public class SpringYarnConfigurationMasterAllocatorTests { + + @Autowired + private ApplicationContext ctx; + + @Test + public void testSimpleConfig() throws Exception { + YarnAppmaster master = (YarnAppmaster) ctx.getBean("yarnAppmaster"); + assertNotNull(master); + + ContainerAllocator allocator = TestUtils.callMethod("getAllocator", master); + assertNotNull(allocator); + } + + @Configuration + @EnableYarn(enable=Enable.APPMASTER) + static class Config extends SpringYarnConfigurerAdapter { + + // TODO: need to fake appmaster for it to not try to connect hadoop + @Override + public void configure(YarnAppmasterConfigurer master) throws Exception { + master + .withContainerAllocator() + .priority(1) + .memory(64) + .virtualCores(1) + .locality(true) + .withCollection("cluster1") + .priority(2) + .and() + .withCollection("cluster2") + .priority(3) + .memory(64) + .virtualCores(1); + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/AbstractStateMachineTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/AbstractStateMachineTests.java new file mode 100644 index 000000000..9b59567a4 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/AbstractStateMachineTests.java @@ -0,0 +1,14 @@ +package org.springframework.yarn.support.statemachine; + +public class AbstractStateMachineTests { + + protected enum TestStates { + SI,S1,S2,S3,S4 + } + + protected enum TestEvents { + E1,E2,E3,E4 + } + + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ActionTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ActionTests.java new file mode 100644 index 000000000..82823437d --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ActionTests.java @@ -0,0 +1,117 @@ +package org.springframework.yarn.support.statemachine; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.config.EnableStateMachine; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +public class ActionTests extends AbstractStateMachineTests { + + @SuppressWarnings({ "unchecked" }) + @Test + public void testTransitionActions() { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(Config1.class); + assertTrue(ctx.containsBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE)); + EnumStateMachine machine = + ctx.getBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE, EnumStateMachine.class); + TestCountAction testAction1 = ctx.getBean("testAction1", TestCountAction.class); + TestCountAction testAction2 = ctx.getBean("testAction2", TestCountAction.class); + TestCountAction testAction3 = ctx.getBean("testAction3", TestCountAction.class); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E2).build()); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E3).build()); + assertThat(testAction1.count, is(1)); + assertThat(testAction2.count, is(1)); + assertThat(testAction3.count, is(1)); + ctx.close(); + } + + @Test + public void testEventFromAction() { + + } + + private static class TestCountAction implements Action { + + int count = 0; + + @Override + public void execute(MessageHeaders headers) { + count++; + } + + } + + @Configuration + @EnableStateMachine + public static class Config1 extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(TestStates.S1) + .state(TestStates.S1) + .state(TestStates.S2) + .state(TestStates.S3) + .state(TestStates.S4); + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + transitions + .withExternal() + .source(TestStates.S1) + .target(TestStates.S2) + .event(TestEvents.E1) + .action(testAction1()) + .and() + .withExternal() + .source(TestStates.S2) + .target(TestStates.S3) + .event(TestEvents.E2) + .action(testAction2()) + .and() + .withExternal() + .source(TestStates.S3) + .target(TestStates.S4) + .event(TestEvents.E3) + .action(testAction3()); + } + + @Bean + public TestCountAction testAction1() { + return new TestCountAction(); + } + + @Bean + public TestCountAction testAction2() { + return new TestCountAction(); + } + + @Bean + public TestCountAction testAction3() { + return new TestCountAction(); + } + + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ConfigurationTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ConfigurationTests.java new file mode 100644 index 000000000..a9004f740 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/ConfigurationTests.java @@ -0,0 +1,96 @@ +package org.springframework.yarn.support.statemachine; + +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import java.util.EnumSet; + +import org.junit.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.messaging.MessageHeaders; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.config.EnableStateMachine; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +public class ConfigurationTests extends AbstractStateMachineTests { + + @SuppressWarnings({ "unchecked" }) + @Test + public void testStates() { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(Config1.class); + assertTrue(ctx.containsBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE)); + EnumStateMachine machine = + ctx.getBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE, EnumStateMachine.class); + assertThat(machine, notNullValue()); + TestAction testAction = ctx.getBean("testAction", TestAction.class); + assertThat(testAction, notNullValue()); + ctx.close(); + } + + private static class TestAction implements Action { + + @Override + public void execute(MessageHeaders headers) { + } + + } + + @Configuration + @EnableStateMachine + public static class Config1 extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(TestStates.S1) + .state(TestStates.S1) + .state(TestStates.S2) + .state(TestStates.S3) + .state(TestStates.S4); + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + transitions + .withExternal() + .source(TestStates.S1) + .target(TestStates.S2) + .event(TestEvents.E1) + .action(testAction()); + } + + @Bean + public TestAction testAction() { + return new TestAction(); + } + + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + + } + + @Configuration + @EnableStateMachine + public static class Config2 extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(TestStates.S1) + .states(EnumSet.allOf(TestStates.class)); + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/EnumStateMachineTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/EnumStateMachineTests.java new file mode 100644 index 000000000..5d2684d6d --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/EnumStateMachineTests.java @@ -0,0 +1,195 @@ +package org.springframework.yarn.support.statemachine; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.state.EnumState; +import org.springframework.yarn.support.statemachine.state.State; +import org.springframework.yarn.support.statemachine.transition.DefaultExternalTransition; +import org.springframework.yarn.support.statemachine.transition.DefaultInternalTransition; +import org.springframework.yarn.support.statemachine.transition.Transition; + +public class EnumStateMachineTests extends AbstractStateMachineTests { + + @Test + public void testSimpleStateSwitch() { + + State stateSI = new EnumState(TestStates.SI); + State stateS1 = new EnumState(TestStates.S1); + State stateS2 = new EnumState(TestStates.S2); + State stateS3 = new EnumState(TestStates.S3); + + Collection> states = new ArrayList>(); + states.add(stateSI); + states.add(stateS1); + states.add(stateS2); + states.add(stateS3); + + Collection> transitions = new ArrayList>(); + + Collection actionsFromSIToS1 = new ArrayList(); + actionsFromSIToS1.add(new LoggingAction("actionsFromSIToS1")); + DefaultExternalTransition transitionFromSIToS1 = + new DefaultExternalTransition(stateSI, stateS1, actionsFromSIToS1, TestEvents.E1); + + Collection actionsFromS1ToS2 = new ArrayList(); + actionsFromS1ToS2.add(new LoggingAction("actionsFromS1ToS2")); + DefaultExternalTransition transitionFromS1ToS2 = + new DefaultExternalTransition(stateS1, stateS2, actionsFromS1ToS2, TestEvents.E2); + + Collection actionsFromS2ToS3 = new ArrayList(); + actionsFromS1ToS2.add(new LoggingAction("actionsFromS2ToS3")); + DefaultExternalTransition transitionFromS2ToS3 = + new DefaultExternalTransition(stateS2, stateS3, actionsFromS2ToS3, TestEvents.E3); + + transitions.add(transitionFromSIToS1); + transitions.add(transitionFromS1ToS2); + transitions.add(transitionFromS2ToS3); + + SyncTaskExecutor taskExecutor = new SyncTaskExecutor(); + EnumStateMachine machine = new EnumStateMachine(states, transitions, stateSI); + machine.setTaskExecutor(taskExecutor); + machine.start(); + + State initialState = machine.getInitialState(); + assertThat(initialState, is(stateSI)); + + State state = machine.getState(); + assertThat(state, is(stateSI)); + + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + state = machine.getState(); + assertThat(state, is(stateS1)); + + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E2).build()); + state = machine.getState(); + assertThat(state, is(stateS2)); + + // not processed + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + state = machine.getState(); + assertThat(state, is(stateS2)); + + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E3).build()); + state = machine.getState(); + assertThat(state, is(stateS3)); + } + + @Test + public void testDeferredEvents() { + + Collection deferred = new ArrayList(); + deferred.add(TestEvents.E2); + deferred.add(TestEvents.E3); + + // states + State stateSI = new EnumState(TestStates.SI, deferred); + State stateS1 = new EnumState(TestStates.S1); + State stateS2 = new EnumState(TestStates.S2); + State stateS3 = new EnumState(TestStates.S3); + + Collection> states = new ArrayList>(); + states.add(stateSI); + states.add(stateS1); + states.add(stateS2); + states.add(stateS3); + + // transitions + Collection> transitions = new ArrayList>(); + + Collection actionsFromSIToS1 = new ArrayList(); + actionsFromSIToS1.add(new LoggingAction("actionsFromSIToS1")); + DefaultExternalTransition transitionFromSIToS1 = + new DefaultExternalTransition(stateSI, stateS1, actionsFromSIToS1, TestEvents.E1); + + Collection actionsFromS1ToS2 = new ArrayList(); + actionsFromS1ToS2.add(new LoggingAction("actionsFromS1ToS2")); + DefaultExternalTransition transitionFromS1ToS2 = + new DefaultExternalTransition(stateS1, stateS2, actionsFromS1ToS2, TestEvents.E2); + + Collection actionsFromS2ToS3 = new ArrayList(); + actionsFromS1ToS2.add(new LoggingAction("actionsFromS2ToS3")); + DefaultExternalTransition transitionFromS2ToS3 = + new DefaultExternalTransition(stateS2, stateS3, actionsFromS2ToS3, TestEvents.E3); + + transitions.add(transitionFromSIToS1); + transitions.add(transitionFromS1ToS2); + transitions.add(transitionFromS2ToS3); + + // create machine + SyncTaskExecutor taskExecutor = new SyncTaskExecutor(); + EnumStateMachine machine = new EnumStateMachine(states, transitions, stateSI); +// StateMachine, TestEvents> machine2 = new EnumStateMachine(states, transitions, stateSI); + machine.setTaskExecutor(taskExecutor); + machine.start(); + + State initialState = machine.getInitialState(); + assertThat(initialState, is(stateSI)); + + State state = machine.getState(); + assertThat(state, is(stateSI)); + + + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E2).build()); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E3).build()); + state = machine.getState(); + assertThat(state, is(stateSI)); + + + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + state = machine.getState(); + assertThat(state, is(stateS3)); + } + + @Test + public void testInternalTransitions() { + State stateSI = new EnumState(TestStates.SI); + + Collection> states = new ArrayList>(); + states.add(stateSI); + + Collection actionsInSI = new ArrayList(); + actionsInSI.add(new LoggingAction("actionsInSI")); + DefaultInternalTransition transitionInternalSI = + new DefaultInternalTransition(stateSI, actionsInSI, TestEvents.E1); + + // transitions + Collection> transitions = new ArrayList>(); + transitions.add(transitionInternalSI); + + SyncTaskExecutor taskExecutor = new SyncTaskExecutor(); + EnumStateMachine machine = new EnumStateMachine(states, transitions, stateSI); + machine.setTaskExecutor(taskExecutor); + machine.start(); + + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + } + + private static class LoggingAction implements Action { + + private static final Log log = LogFactory.getLog(LoggingAction.class); + + private String message; + + public LoggingAction(String message) { + this.message = message; + } + + @Override + public void execute(MessageHeaders headers) { + log.info("Hello from LoggingAction " + message); + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineFactoryTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineFactoryTests.java new file mode 100644 index 000000000..f568b1d86 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineFactoryTests.java @@ -0,0 +1,66 @@ +package org.springframework.yarn.support.statemachine; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +import org.junit.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.yarn.support.statemachine.config.EnableStateMachineFactory; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineFactory; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; +import org.springframework.yarn.support.statemachine.state.State; + +public class StateMachineFactoryTests extends AbstractStateMachineTests { + + @SuppressWarnings({ "unchecked" }) + @Test + public void testMachineFromFactory() { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(Config.class); + + EnumStateMachineFactory stateMachineFactory = + ctx.getBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINEFACTORY, EnumStateMachineFactory.class); + + StateMachine, TestEvents> machine = stateMachineFactory.getStateMachine(); + assertThat(machine.getState().getId(), is(TestStates.S1)); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + assertThat(machine.getState().getId(), is(TestStates.S2)); + ctx.close(); + } + + @Configuration + @EnableStateMachineFactory + static class Config extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(TestStates.S1) + .state(TestStates.S1) + .state(TestStates.S2); + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + transitions + .withExternal() + .source(TestStates.S1) + .target(TestStates.S2) + .event(TestEvents.E1); + } + + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineTests.java new file mode 100644 index 000000000..d9fa0b546 --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/StateMachineTests.java @@ -0,0 +1,112 @@ +package org.springframework.yarn.support.statemachine; + +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.yarn.support.statemachine.action.Action; +import org.springframework.yarn.support.statemachine.config.EnableStateMachine; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +public class StateMachineTests extends AbstractStateMachineTests { + + @Test + public void test1() { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(Config.class); + assertTrue(ctx.containsBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE)); + @SuppressWarnings("unchecked") + EnumStateMachine machine = + ctx.getBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE, EnumStateMachine.class); + assertThat(machine, notNullValue()); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).setHeader("foo", "jee1").build()); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E2).setHeader("foo", "jee2").build()); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E4).setHeader("foo", "jee2").build()); + ctx.close(); + } + + private static class LoggingAction implements Action { + + private static final Log log = LogFactory.getLog(StateMachineTests.LoggingAction.class); + + private String message; + + public LoggingAction(String message) { + this.message = message; + } + + @Override + public void execute(MessageHeaders headers) { + log.info("Hello from LoggingAction " + message + " foo=" + headers.get("foo")); + } + + } + + @Configuration + @EnableStateMachine + static class Config extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(TestStates.S1) + .state(TestStates.S1) + .state(TestStates.S2) + .state(TestStates.S3, TestEvents.E4) + .state(TestStates.S4); + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + transitions + .withExternal() + .source(TestStates.S1) + .target(TestStates.S2) + .event(TestEvents.E1) + .action(loggingAction()) + .action(loggingAction()) + .and() + .withExternal() + .source(TestStates.S2) + .target(TestStates.S3) + .event(TestEvents.E2) + .action(loggingAction()) + .and() + .withExternal() + .source(TestStates.S3) + .target(TestStates.S4) + .event(TestEvents.E3) + .action(loggingAction()) + .and() + .withExternal() + .source(TestStates.S4) + .target(TestStates.S3) + .event(TestEvents.E4) + .action(loggingAction()); + } + + @Bean + public LoggingAction loggingAction() { + return new LoggingAction("as bean"); + } + + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + + } + +} diff --git a/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/TransitionTests.java b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/TransitionTests.java new file mode 100644 index 000000000..8265b469e --- /dev/null +++ b/spring-yarn/spring-yarn-core/src/test/java/org/springframework/yarn/support/statemachine/TransitionTests.java @@ -0,0 +1,68 @@ +package org.springframework.yarn.support.statemachine; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import java.util.EnumSet; + +import org.junit.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.yarn.support.statemachine.config.EnableStateMachine; +import org.springframework.yarn.support.statemachine.config.EnumStateMachineConfigurerAdapter; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineStateConfigurer; +import org.springframework.yarn.support.statemachine.config.builders.StateMachineTransitionConfigurer; + +public class TransitionTests extends AbstractStateMachineTests { + + @SuppressWarnings({ "unchecked" }) + @Test + public void testTriggerlessTransition() { + AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(Config1.class); + assertTrue(ctx.containsBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE)); + EnumStateMachine machine = + ctx.getBean(StateMachineSystemConstants.DEFAULT_ID_STATEMACHINE, EnumStateMachine.class); + assertThat(machine.getState().getId(), is(TestStates.S1)); + machine.sendEvent(MessageBuilder.withPayload(TestEvents.E1).build()); + assertThat(machine.getState().getId(), is(TestStates.S3)); + ctx.close(); + } + + @Configuration + @EnableStateMachine + public static class Config1 extends EnumStateMachineConfigurerAdapter { + + @Override + public void configure(StateMachineStateConfigurer states) throws Exception { + states + .withStates() + .initial(TestStates.S1) + .states(EnumSet.allOf(TestStates.class)); + } + + @Override + public void configure(StateMachineTransitionConfigurer transitions) throws Exception { + transitions + .withExternal() + .source(TestStates.S1) + .target(TestStates.S2) + .event(TestEvents.E1) + .and() + .withExternal() + .source(TestStates.S2) + .target(TestStates.S3); + } + + @Bean + public TaskExecutor taskExecutor() { + return new SyncTaskExecutor(); + } + + } + +}