From 6d7765b8cf023030de766bcb11f551765450ad46 Mon Sep 17 00:00:00 2001 From: Valentin Aitken Date: Thu, 28 Jul 2016 03:13:18 +0300 Subject: [PATCH 1/2] Effector for opening inbound ports in security group - Live tests examining security groups (the effector will return error when called before machine location is provisioned) --- .../networking/NetworkingEffectors.java | 100 ++++++++++++++++ software/base/pom.xml | 12 +- .../entity/software/base/SoftwareProcess.java | 3 + .../software/base/SoftwareProcessImpl.java | 4 + .../NetworkEffectorsEc2LiveTests.java | 45 ++++++++ .../NetworkingEffectorsLiveTests.java | 108 ++++++++++++++++++ 6 files changed, 265 insertions(+), 7 deletions(-) create mode 100644 locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java create mode 100644 software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkEffectorsEc2LiveTests.java create mode 100644 software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkingEffectorsLiveTests.java diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java new file mode 100644 index 0000000000..72441237cb --- /dev/null +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.location.jclouds.networking; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Range; +import com.google.common.reflect.TypeToken; +import org.apache.brooklyn.api.effector.Effector; +import org.apache.brooklyn.api.location.Location; +import org.apache.brooklyn.config.ConfigKey; +import org.apache.brooklyn.core.config.ConfigKeys; +import org.apache.brooklyn.core.effector.EffectorBody; +import org.apache.brooklyn.core.effector.Effectors; +import org.apache.brooklyn.location.jclouds.JcloudsMachineLocation; +import org.apache.brooklyn.util.collections.MutableList; +import org.apache.brooklyn.util.core.config.ConfigBag; +import org.apache.brooklyn.util.net.Cidr; +import org.apache.brooklyn.util.net.Networking; +import org.jclouds.net.domain.IpPermission; +import org.jclouds.net.domain.IpProtocol; + +import java.util.List; + +import static com.google.common.base.Predicates.instanceOf; +import static com.google.common.collect.Iterables.tryFind; +import static org.apache.brooklyn.core.location.Locations.getLocationsCheckingAncestors; + +public class NetworkingEffectors { + // Intentionally not use CloudLocationConfig.INBOUND_PORTS to make richer syntax and rename it to differ it from the first in a ConfigBag + public static final ConfigKey> INBOUND_PORTS_LIST = ConfigKeys.newConfigKey(new TypeToken>() {}, "inbound.ports.list", + "Ports to open from the effector", ImmutableList.of()); + public static final ConfigKey INBOUND_PORTS_LIST_PROTOCOL = ConfigKeys.newConfigKey(new TypeToken() {}, "inbound.ports.list.protocol", + "Protocol for ports to open. Possible values: TCP, UDP, ICMP, ALL.", IpProtocol.TCP); + + public static final ConfigKey JCLOUDS_MACHINE_LOCATIN = ConfigKeys.newConfigKey(JcloudsMachineLocation.class, "jcloudsMachineLocation"); + + @SuppressWarnings("unchecked") + public static final Effector> OPEN_INBOUND_PORTS_IN_SECURITY_GROUP_EFFECTOR = (Effector>)(Effector)Effectors.effector(Iterable.class, "openPortsInSecurityGroup") + .parameter(INBOUND_PORTS_LIST) + .parameter(INBOUND_PORTS_LIST_PROTOCOL) + .description("Open ports in Cloud Security Group. If called before machine location is provisioned, it will fail.") + .impl(new OpenPortsInSecurityGroupBody()) + .build(); + + @SuppressWarnings("rawtypes") + private static class OpenPortsInSecurityGroupBody extends EffectorBody { + @Override + public Iterable call(ConfigBag parameters) { + List rawPortRules = parameters.get(INBOUND_PORTS_LIST); + IpProtocol ipProtocol = parameters.get(INBOUND_PORTS_LIST_PROTOCOL); + JcloudsMachineLocation jcloudsMachineLocation = parameters.get(JCLOUDS_MACHINE_LOCATIN); + Preconditions.checkNotNull(ipProtocol, INBOUND_PORTS_LIST_PROTOCOL.getName() + " cannot be null"); + Preconditions.checkNotNull(rawPortRules, INBOUND_PORTS_LIST.getName() + " cannot be null"); + MutableList.Builder ipPermissionsBuilder = MutableList.builder(); + for (Range portRule : Networking.portRulesToRanges(rawPortRules).asRanges()) { + ipPermissionsBuilder.add( + IpPermission.builder() + .ipProtocol(ipProtocol) + .fromPort(portRule.lowerEndpoint()) + .toPort(portRule.upperEndpoint()) + .cidrBlock(Cidr.UNIVERSAL.toString()) + .build()); + } + JcloudsLocationSecurityGroupCustomizer customizer = JcloudsLocationSecurityGroupCustomizer.getInstance(entity()); + + if (jcloudsMachineLocation == null) { + Optional jcloudsMachineLocationOptional = tryFind( + (Iterable) getLocationsCheckingAncestors(null, entity()), + instanceOf(JcloudsMachineLocation.class)); + if (!jcloudsMachineLocationOptional.isPresent()) { + throw new IllegalArgumentException("Tried to execute open ports effector on an entity with no JcloudsMachineLocation"); + } else { + jcloudsMachineLocation = (JcloudsMachineLocation)jcloudsMachineLocationOptional.get(); + } + } + Iterable ipPermissionsToAdd = ipPermissionsBuilder.build(); + customizer.addPermissionsToLocation(jcloudsMachineLocation, ipPermissionsToAdd); + return ipPermissionsToAdd; + } + } + +} diff --git a/software/base/pom.xml b/software/base/pom.xml index 7e0d4fda14..daec1dc5d3 100644 --- a/software/base/pom.xml +++ b/software/base/pom.xml @@ -70,6 +70,11 @@ brooklyn-policy ${project.version} + + org.apache.brooklyn + brooklyn-locations-jclouds + ${project.version} + org.freemarker @@ -111,13 +116,6 @@ ${project.version} test - - - org.apache.brooklyn - brooklyn-locations-jclouds - ${project.version} - test - ${jclouds.groupId} jclouds-core diff --git a/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcess.java b/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcess.java index 133ced5347..05bdd15946 100644 --- a/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcess.java +++ b/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcess.java @@ -147,6 +147,9 @@ public interface SoftwareProcess extends Entity, Startable { @SetFromFlag("runDir") AttributeSensorAndConfigKey RUN_DIR = BrooklynConfigKeys.RUN_DIR; + ConfigKey ADD_OPEN_INBOUND_PORTS_EFFECTOR = ConfigKeys.newBooleanConfigKey("effector.add.openInboundPorts", + "Flag which adds effector for opening ports through Cloud security groups", false); + ConfigKey OPEN_IPTABLES = ConfigKeys.newBooleanConfigKey("openIptables", "Whether to open the INBOUND_PORTS via iptables rules; " + "if true then ssh in to run iptables commands, as part of machine provisioning", false); diff --git a/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcessImpl.java b/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcessImpl.java index 5abc8e96f5..0d4dbdaae8 100644 --- a/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcessImpl.java +++ b/software/base/src/main/java/org/apache/brooklyn/entity/software/base/SoftwareProcessImpl.java @@ -28,6 +28,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; +import org.apache.brooklyn.location.jclouds.networking.NetworkingEffectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,6 +131,9 @@ protected MachineLocation getMachineOrNull() { public void init() { super.init(); getLifecycleEffectorTasks().attachLifecycleEffectors(this); + if (Boolean.TRUE.equals(getConfig(ADD_OPEN_INBOUND_PORTS_EFFECTOR))) { + getMutableEntityType().addEffector(NetworkingEffectors.OPEN_INBOUND_PORTS_IN_SECURITY_GROUP_EFFECTOR); + } } @Override diff --git a/software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkEffectorsEc2LiveTests.java b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkEffectorsEc2LiveTests.java new file mode 100644 index 0000000000..bf7a67b7dc --- /dev/null +++ b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkEffectorsEc2LiveTests.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.entity.software.base.location; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +public class NetworkEffectorsEc2LiveTests extends NetworkingEffectorsLiveTests { + public static final String PROVIDER = "aws-ec2"; + public static final String REGION_NAME = "us-east-1"; + public static final String LOCATION_SPEC = PROVIDER + (REGION_NAME == null ? "" : ":" + REGION_NAME); + + @Test(groups = "Live") + public void testPassSecurityGroupParameters() { + super.testPassSecurityGroupParameters(); + } + + @Override + public String getLocationSpec() { + return LOCATION_SPEC; + } + + @Override + public Map getLocationProperties() { + return ImmutableMap.of("imageId", "us-east-1/ami-a96b01c0", "hardwareId", "t1.micro"); + } +} diff --git a/software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkingEffectorsLiveTests.java b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkingEffectorsLiveTests.java new file mode 100644 index 0000000000..482432c15d --- /dev/null +++ b/software/base/src/test/java/org/apache/brooklyn/entity/software/base/location/NetworkingEffectorsLiveTests.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.entity.software.base.location; + +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import org.apache.brooklyn.api.effector.Effector; +import org.apache.brooklyn.api.entity.EntitySpec; +import org.apache.brooklyn.api.location.Location; +import org.apache.brooklyn.api.mgmt.Task; +import org.apache.brooklyn.core.mgmt.internal.EffectorUtils; +import org.apache.brooklyn.core.test.BrooklynAppLiveTestSupport; +import org.apache.brooklyn.entity.software.base.EmptySoftwareProcess; +import org.apache.brooklyn.entity.software.base.SoftwareProcess; +import org.apache.brooklyn.location.jclouds.JcloudsLocation; +import org.apache.brooklyn.location.jclouds.JcloudsMachineLocation; +import org.jclouds.compute.ComputeService; +import org.jclouds.compute.domain.SecurityGroup; +import org.jclouds.compute.extensions.SecurityGroupExtension; +import org.jclouds.net.domain.IpPermission; +import org.jclouds.net.domain.IpProtocol; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Set; + +import static org.apache.brooklyn.location.jclouds.networking.NetworkingEffectors.INBOUND_PORTS_LIST; +import static org.apache.brooklyn.location.jclouds.networking.NetworkingEffectors.INBOUND_PORTS_LIST_PROTOCOL; +import static org.apache.brooklyn.test.Asserts.assertTrue; +import static org.jclouds.net.domain.IpProtocol.TCP; +import static org.jclouds.net.domain.IpProtocol.UDP; + +public abstract class NetworkingEffectorsLiveTests extends BrooklynAppLiveTestSupport { + @Test(groups = "Live") + public void testPassSecurityGroupParameters() { + EmptySoftwareProcess emptySoftwareProcess = app.createAndManageChild(EntitySpec.create(EmptySoftwareProcess.class) + .configure(SoftwareProcess.ADD_OPEN_INBOUND_PORTS_EFFECTOR, true)); + + JcloudsLocation jcloudsLocation = (JcloudsLocation)mgmt.getLocationRegistry().getLocationManaged(getLocationSpec(), getLocationProperties()); + app.start(ImmutableList.of(jcloudsLocation)); + + Optional jcloudsMachineLocation = Iterables.tryFind(emptySoftwareProcess.getLocations(), Predicates.instanceOf(JcloudsMachineLocation.class)); + if (!jcloudsMachineLocation.isPresent()) { + throw new IllegalArgumentException("Tried to execute open ports effector on an entity with no JcloudsMachineLocation"); + } + ComputeService computeService = ((JcloudsMachineLocation)jcloudsMachineLocation.get()).getParent().getComputeService(); + String nodeId = ((JcloudsMachineLocation)jcloudsMachineLocation.get()).getNode().getId(); + final SecurityGroupExtension securityApi = computeService.getSecurityGroupExtension().get(); + + Effector> openPortsInSecurityGroup = (Effector>)EffectorUtils.findEffectorDeclared(emptySoftwareProcess, "openPortsInSecurityGroup").get(); + Task> task = EffectorUtils.invokeEffectorAsync(emptySoftwareProcess, openPortsInSecurityGroup, + ImmutableMap.of(INBOUND_PORTS_LIST.getName(), "234,324,550-1050")); + Iterable effectorResult = task.getUnchecked(); + for (Predicate ipPermissionPredicate : ImmutableList.of(ruleExistsPredicate(234, 234, TCP), ruleExistsPredicate(324, 324, TCP), ruleExistsPredicate(550, 1050, TCP))) { + assertTrue(Iterables.tryFind(effectorResult, ipPermissionPredicate).isPresent()); + } + + task = EffectorUtils.invokeEffectorAsync(emptySoftwareProcess, openPortsInSecurityGroup, + ImmutableMap.of(INBOUND_PORTS_LIST.getName(), "234,324,550-1050", INBOUND_PORTS_LIST_PROTOCOL.getName(), "UDP")); + effectorResult = task.getUnchecked(); + for (Predicate ipPermissionPredicate : ImmutableList.of(ruleExistsPredicate(234, 234, UDP), ruleExistsPredicate(324, 324, UDP), ruleExistsPredicate(550, 1050, UDP))) { + assertTrue(Iterables.tryFind(effectorResult, ipPermissionPredicate).isPresent()); + } + + Set groupsOnNode = securityApi.listSecurityGroupsForNode(nodeId); + SecurityGroup securityGroup = Iterables.getOnlyElement(groupsOnNode); + + assertTrue(Iterables.tryFind(securityGroup.getIpPermissions(), ruleExistsPredicate(234, 234, TCP)).isPresent()); + assertTrue(Iterables.tryFind(securityGroup.getIpPermissions(), ruleExistsPredicate(324, 324, TCP)).isPresent()); + assertTrue(Iterables.tryFind(securityGroup.getIpPermissions(), ruleExistsPredicate(550, 1050, TCP)).isPresent()); + + assertTrue(Iterables.tryFind(securityGroup.getIpPermissions(), ruleExistsPredicate(234, 234, UDP)).isPresent()); + assertTrue(Iterables.tryFind(securityGroup.getIpPermissions(), ruleExistsPredicate(324, 324, UDP)).isPresent()); + assertTrue(Iterables.tryFind(securityGroup.getIpPermissions(), ruleExistsPredicate(550, 1050, UDP)).isPresent()); + } + + protected Predicate ruleExistsPredicate(final int fromPort, final int toPort, final IpProtocol ipProtocol) { + return new Predicate() { + public boolean apply(IpPermission ipPermission) { + return ipPermission.getFromPort() == fromPort && ipPermission.getToPort() == toPort && ipPermission.getIpProtocol() == ipProtocol; + } + }; + } + + public abstract String getLocationSpec(); + + public abstract Map getLocationProperties(); +} From 2e736083e55a273c3485ab3f9cb2b033156e091d Mon Sep 17 00:00:00 2001 From: Valentin Aitken Date: Tue, 23 Aug 2016 15:24:59 +0300 Subject: [PATCH 2/2] InboundPortsJcloudsLocationCustomizer --- ...InboundPortsJcloudsLocationCustomizer.java | 68 +++++++++++++++++++ .../networking/NetworkingEffectors.java | 2 + 2 files changed, 70 insertions(+) create mode 100644 locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/InboundPortsJcloudsLocationCustomizer.java diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/InboundPortsJcloudsLocationCustomizer.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/InboundPortsJcloudsLocationCustomizer.java new file mode 100644 index 0000000000..11b01eee0f --- /dev/null +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/InboundPortsJcloudsLocationCustomizer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.brooklyn.location.jclouds.networking; + +import com.google.common.annotations.Beta; +import org.apache.brooklyn.api.entity.Entity; +import org.apache.brooklyn.api.mgmt.TaskAdaptable; +import org.apache.brooklyn.core.effector.Effectors; +import org.apache.brooklyn.core.location.cloud.CloudLocationConfig; +import org.apache.brooklyn.location.jclouds.BasicJcloudsLocationCustomizer; +import org.apache.brooklyn.location.jclouds.JcloudsLocation; +import org.apache.brooklyn.location.jclouds.JcloudsMachineLocation; +import org.apache.brooklyn.util.collections.MutableMap; +import org.apache.brooklyn.util.core.task.DynamicTasks; +import org.jclouds.compute.ComputeService; +import org.jclouds.net.domain.IpPermission; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.brooklyn.location.jclouds.networking.NetworkingEffectors.*; + +@Beta +public class InboundPortsJcloudsLocationCustomizer extends BasicJcloudsLocationCustomizer { + public static final Logger LOG = LoggerFactory.getLogger(InboundPortsJcloudsLocationCustomizer.class); + + private String inboundPortsList; + private String inboundPortsListProtocol; + + @Override + public void customize(JcloudsLocation location, ComputeService computeService, JcloudsMachineLocation machine) { + Object callerContext = machine.getConfig(CloudLocationConfig.CALLER_CONTEXT); + Entity entity; + if (callerContext instanceof Entity) { + entity = (Entity)callerContext; + } else { + throw new IllegalArgumentException("customizer should be called on against a jcloudsLocation which has callerContext of type Entity. Location " + location); + } + TaskAdaptable> taskAdaptable = Effectors.invocation(entity, + NetworkingEffectors.OPEN_INBOUND_PORTS_IN_SECURITY_GROUP_EFFECTOR, + MutableMap.of(INBOUND_PORTS_LIST, inboundPortsList, INBOUND_PORTS_LIST_PROTOCOL, inboundPortsListProtocol, JCLOUDS_MACHINE_LOCATIN, machine)); + Iterable result = DynamicTasks.submit(taskAdaptable.asTask(), entity).getUnchecked(); + LOG.info("Opened ports for " + entity + " " + result); + } + + public void setInboundPortsList(String inboundPortsList) { + this.inboundPortsList = inboundPortsList; + } + + public void setInboundPortsListProtocol(String inboundPortsListProtocol) { + this.inboundPortsListProtocol = inboundPortsListProtocol; + } +} diff --git a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java index 72441237cb..aba97530d3 100644 --- a/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java +++ b/locations/jclouds/src/main/java/org/apache/brooklyn/location/jclouds/networking/NetworkingEffectors.java @@ -18,6 +18,7 @@ */ package org.apache.brooklyn.location.jclouds.networking; +import com.google.common.annotations.Beta; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -52,6 +53,7 @@ public class NetworkingEffectors { public static final ConfigKey JCLOUDS_MACHINE_LOCATIN = ConfigKeys.newConfigKey(JcloudsMachineLocation.class, "jcloudsMachineLocation"); + @Beta @SuppressWarnings("unchecked") public static final Effector> OPEN_INBOUND_PORTS_IN_SECURITY_GROUP_EFFECTOR = (Effector>)(Effector)Effectors.effector(Iterable.class, "openPortsInSecurityGroup") .parameter(INBOUND_PORTS_LIST)