Skip to content
Permalink
Browse files
[JCLOUDS-1430] Aliyun ECS
- add instance API
- add compute abstraction
- add validation for vpc and vSwitch IDs
- add builders for Image and Instance
- add unit tests for compute/functions
- add pagination to instanceStatus api
- rename provider id
- clean up code
- add network apis
- vpc api + tests
- vswitch api + tests
- improve CreateResourcesThenCreateNodes
- create default vpc and vswitch in case needed
- fix InstanceApiLiveTest
- add ECSDependencyViolationRetryHandler
- add ErrorRetryHandler
- fix ListImagesOptions.imageId
- fix enums in Instance and EIPAddress
  • Loading branch information
andreaturli committed Sep 12, 2018
1 parent a5dbf00 commit 2c7db7e809085a7738d76f5714663231dff0a9f3
Showing 100 changed files with 13,485 additions and 113 deletions.
@@ -0,0 +1,43 @@
alibaba Elastic Compute Service Provider
==========================

# How to use it

alibaba ECS provider works exactly as any other jclouds provider.
Notice that as alibaba supports dozens of locations and to limit the scope of some operations, one may want to use:

and
```bash
jclouds.regions
```
which is by default `null`. If you want to target only the `north europe` region, you can use

```bash
jclouds.regions="eu-central-1"
```

# Setting Up Test Environment

Get or create the `User Access Key` and `Access Key Secret` for your account at `https://usercenter.console.alibaba.com/#/manage/ak`

# Run Live Tests

Use the following to run one live test:

```bash
mvn -Dtest=<name of the live test> \
-Dtest.alibaba-ecs.identity="<AccessKey ID>" \
-Dtest.alibaba-ecs.credential="<Access Key Secret>"
integration-test -Plive
```

Use the following to run all the live tests:

```bash
mvn clean verify -Plive \
-Dtest.alibaba-ecs.identity="<AccessKey ID>" \
-Dtest.alibaba-ecs.credential="<Access Key Secret>"
```


@@ -32,8 +32,6 @@

<properties>
<test.aliyun-ecs.endpoint>https://ecs.aliyuncs.com/</test.aliyun-ecs.endpoint>
<test.aliyun-ecs.api-version></test.aliyun-ecs.api-version>
<test.aliyun-ecs.build-version/>
<test.aliyun-ecs.identity>FIXME_IDENTITY</test.aliyun-ecs.identity>
<test.aliyun-ecs.credential>FIXME_CREDENTIALS</test.aliyun-ecs.credential>
<test.aliyun-ecs.template/>
@@ -142,6 +140,5 @@
</profile>
</profiles>


</project>

@@ -17,10 +17,13 @@
package org.jclouds.aliyun.ecs;

import org.jclouds.aliyun.ecs.features.ImageApi;
import org.jclouds.aliyun.ecs.features.InstanceApi;
import org.jclouds.aliyun.ecs.features.RegionAndZoneApi;
import org.jclouds.aliyun.ecs.features.SecurityGroupApi;
import org.jclouds.aliyun.ecs.features.SshKeyPairApi;
import org.jclouds.aliyun.ecs.features.TagApi;
import org.jclouds.aliyun.ecs.features.VPCApi;
import org.jclouds.aliyun.ecs.features.VSwitchApi;
import org.jclouds.rest.annotations.Delegate;

import java.io.Closeable;
@@ -42,4 +45,12 @@ public interface ECSComputeServiceApi extends Closeable {
@Delegate
TagApi tagApi();

@Delegate
InstanceApi instanceApi();

@Delegate
VPCApi vpcApi();

@Delegate
VSwitchApi vSwitchApi();
}
@@ -51,8 +51,8 @@ public Builder toBuilder() {
public static class Builder extends BaseProviderMetadata.Builder {

protected Builder() {
id("aliyun-ecs")
.name("Alibaba Elastic Compute Service")
id("alibaba-ecs")
.name("Alibaba Cloud Elastic Compute Service")
.apiMetadata(new ECSServiceApiMetadata())
.homepage(URI.create("https://www.alibabacloud.com"))
.console(URI.create("https://ecs.console.aliyun.com"))
@@ -18,6 +18,7 @@

import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
import org.jclouds.aliyun.ecs.compute.config.ECSServiceContextModule;
import org.jclouds.aliyun.ecs.config.ECSComputeServiceHttpApiModule;
import org.jclouds.aliyun.ecs.config.ECSComputeServiceParserModule;
import org.jclouds.apis.ApiMetadata;
@@ -46,7 +47,7 @@ protected ECSServiceApiMetadata(Builder builder) {

public static Properties defaultProperties() {
Properties properties = BaseHttpApiMetadata.defaultProperties();
properties.put(TEMPLATE, "osFamily=CENTOS,os64Bit=true,osVersionMatches=7.4");
properties.put(TEMPLATE, "osFamily=CENTOS,os64Bit=true,osVersionMatches=7.*");
properties.put(TIMEOUT_NODE_RUNNING, 900000); // 15 mins
properties.put(TIMEOUT_NODE_SUSPENDED, 900000); // 15 mins
return properties;
@@ -60,7 +61,7 @@ public Builder toBuilder() {
public static class Builder extends BaseHttpApiMetadata.Builder<ECSComputeServiceApi, Builder> {

protected Builder() {
id("aliyun-ecs")
id("alibaba-ecs")
.name("Alibaba Elastic Compute Service API")
.identityName("user name")
.credentialName("user password")
@@ -72,6 +73,7 @@ protected Builder() {
.defaultModules(ImmutableSet.<Class<? extends Module>>builder()
.add(ECSComputeServiceHttpApiModule.class)
.add(ECSComputeServiceParserModule.class)
.add(ECSServiceContextModule.class)
.build());
}

@@ -0,0 +1,154 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jclouds.aliyun.ecs.compute;

import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Supplier;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ListeningExecutorService;
import org.jclouds.Constants;
import org.jclouds.aliyun.ecs.ECSComputeServiceApi;
import org.jclouds.aliyun.ecs.compute.strategy.CleanupResources;
import org.jclouds.aliyun.ecs.domain.SecurityGroup;
import org.jclouds.aliyun.ecs.domain.VSwitch;
import org.jclouds.aliyun.ecs.domain.options.ListVSwitchesOptions;
import org.jclouds.aliyun.ecs.domain.regionscoped.RegionAndId;
import org.jclouds.collect.Memoized;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.callables.RunScriptOnNode;
import org.jclouds.compute.domain.Hardware;
import org.jclouds.compute.domain.Image;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.TemplateBuilder;
import org.jclouds.compute.extensions.ImageExtension;
import org.jclouds.compute.extensions.SecurityGroupExtension;
import org.jclouds.compute.extensions.internal.DelegatingImageExtension;
import org.jclouds.compute.internal.BaseComputeService;
import org.jclouds.compute.internal.PersistNodeCredentials;
import org.jclouds.compute.options.TemplateOptions;
import org.jclouds.compute.strategy.CreateNodesInGroupThenAddToSet;
import org.jclouds.compute.strategy.DestroyNodeStrategy;
import org.jclouds.compute.strategy.GetImageStrategy;
import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
import org.jclouds.compute.strategy.InitializeRunScriptOnNodeOrPlaceInBadMap;
import org.jclouds.compute.strategy.ListNodesStrategy;
import org.jclouds.compute.strategy.RebootNodeStrategy;
import org.jclouds.compute.strategy.ResumeNodeStrategy;
import org.jclouds.compute.strategy.SuspendNodeStrategy;
import org.jclouds.domain.Credentials;
import org.jclouds.domain.Location;
import org.jclouds.scriptbuilder.functions.InitAdminAccess;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Provider;
import javax.inject.Singleton;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;

import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_RUNNING;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_SUSPENDED;
import static org.jclouds.compute.config.ComputeServiceProperties.TIMEOUT_NODE_TERMINATED;

@Singleton
public class ECSComputeService extends BaseComputeService {
private final CleanupResources cleanupResources;

@Inject
protected ECSComputeService(ComputeServiceContext context, Map<String, Credentials> credentialStore,
@Memoized Supplier<Set<? extends Image>> images, @Memoized Supplier<Set<? extends Hardware>> sizes,
@Memoized Supplier<Set<? extends Location>> locations, ListNodesStrategy listNodesStrategy,
GetImageStrategy getImageStrategy, GetNodeMetadataStrategy getNodeMetadataStrategy,
CreateNodesInGroupThenAddToSet runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy,
DestroyNodeStrategy destroyNodeStrategy, ResumeNodeStrategy startNodeStrategy,
SuspendNodeStrategy stopNodeStrategy, Provider<TemplateBuilder> templateBuilderProvider,
@Named("DEFAULT") Provider<TemplateOptions> templateOptionsProvider,
@Named(TIMEOUT_NODE_RUNNING) Predicate<AtomicReference<NodeMetadata>> nodeRunning,
@Named(TIMEOUT_NODE_TERMINATED) Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
@Named(TIMEOUT_NODE_SUSPENDED) Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
RunScriptOnNode.Factory runScriptOnNodeFactory, InitAdminAccess initAdminAccess,
PersistNodeCredentials persistNodeCredentials,
@Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor,
CleanupResources cleanupResources, Optional<ImageExtension> imageExtension,
Optional<SecurityGroupExtension> securityGroupExtension,
DelegatingImageExtension.Factory delegatingImageExtension) {
super(context, credentialStore, images, sizes, locations, listNodesStrategy, getImageStrategy,
getNodeMetadataStrategy, runNodesAndAddToSetStrategy, rebootNodeStrategy, destroyNodeStrategy,
startNodeStrategy, stopNodeStrategy, templateBuilderProvider, templateOptionsProvider, nodeRunning,
nodeTerminated, nodeSuspended, initScriptRunnerFactory, initAdminAccess, runScriptOnNodeFactory,
persistNodeCredentials, userExecutor, imageExtension, securityGroupExtension, delegatingImageExtension);
this.cleanupResources = cleanupResources;
}

@Override
protected void cleanUpIncidentalResourcesOfDeadNodes(Set<? extends NodeMetadata> deadNodes) {
for (NodeMetadata deadNode : deadNodes) {
RegionAndId regionAndId = RegionAndId.fromSlashEncoded(deadNode.getId());
Set<String> tags = deadNode.getTags();
String vSwitchId = extractVSwitchId(tags);
VSwitch vSwitch = context.unwrapApi(ECSComputeServiceApi.class).vSwitchApi().list(deadNode.getLocation().getId(), ListVSwitchesOptions.Builder.vSwitchId(vSwitchId)).first().orNull();
String vpcId = vSwitch.vpcId();

try {
cleanupResources.cleanupNode(regionAndId);
} catch (Exception ex) {
logger.warn(ex, "Error cleaning up resources for node %s", deadNode);
}

List<SecurityGroup> securityGroups = cleanupResources.findOrphanedSecurityGroups(regionAndId.regionId(), deadNode.getGroup());
for (SecurityGroup securityGroup : securityGroups) {
logger.debug(">> destroying security group %s ...", securityGroup.id());
if (cleanupResources.cleanupSecurityGroupIfOrphaned(regionAndId.regionId(), securityGroup.id())) {
logger.debug(">> security group: (%s) has been deleted.", securityGroup.id());
} else {
logger.warn(">> security group: (%s) has not been deleted.", securityGroup.id());
}
}

// FIXME not sure it is correct to always delete vSwitch and VPC_PREFIX
logger.debug(">> destroying vSwitch %s ...", vSwitchId);
if (cleanupResources.cleanupVSwitchIfOrphaned(regionAndId.regionId(), vSwitchId)) {
logger.debug(">> vSwitch: (%s) has been deleted.", vSwitchId);
} else {
logger.warn(">> vSwitch: (%s) has not been deleted.", vSwitchId);
}

logger.debug(">> destroying vpc %s ...", vpcId);
try {
cleanupResources.cleanupVPCIfOrphaned(regionAndId.regionId(), vpcId);
logger.debug(">> VPC_PREFIX: (%s) has been deleted.", vpcId);
} catch (IllegalArgumentException e) {
logger.warn(">> VPC_PREFIX: (%s) has not been deleted.", vpcId);
}
}
}

private String extractVSwitchId(Set<String> tags) {
String vSwitchIdTag = Iterables.tryFind(tags, new Predicate<String>() {
@Override
public boolean apply(@Nullable String input) {
return input.startsWith("vsw-");
}
}).orNull();
return vSwitchIdTag;
}
}

0 comments on commit 2c7db7e

Please sign in to comment.