WHIRR-49 Allow Whirr to use Chef for configuration management

git-svn-id: https://svn.apache.org/repos/asf/whirr/trunk@1190104 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -8,6 +8,8 @@ Trunk (unreleased changes)
 
     WHIRR-386. Remove references to the Apache Incubator (asavu)
 
+    WHIRR-49.  Allow Whirr to use Chef for configuration management
+
   IMPROVEMENTS
 
     WHIRR-116. Site should have docs for each released version. (tomwhite)

cli/pom.xml

@@ -84,6 +84,11 @@
       <artifactId>whirr-puppet</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>whirr-chef</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

core/src/main/java/org/apache/whirr/ClusterController.java

@@ -37,6 +37,7 @@
 import org.jclouds.compute.domain.ExecResponse;
 import org.jclouds.compute.domain.NodeMetadata;
 import org.jclouds.compute.domain.NodeState;
+import org.jclouds.compute.options.RunScriptOptions;
 import org.jclouds.domain.Credentials;
 import org.jclouds.scriptbuilder.domain.Statement;
 import org.slf4j.Logger;
@@ -151,12 +152,34 @@ public ClusterStateStore getClusterStateStore(ClusterSpec clusterSpec) {
   public Map<? extends NodeMetadata, ExecResponse> runScriptOnNodesMatching(ClusterSpec spec,
         Predicate<NodeMetadata> condition, Statement statement) throws IOException, RunScriptOnNodesException {
 
-    Credentials credentials = new Credentials(spec.getClusterUser(), spec.getPrivateKey());
-    ComputeServiceContext context = getCompute().apply(spec);
+    return runScriptOnNodesMatching(spec, condition, statement, null);
+  }
+
+  public Map<? extends NodeMetadata, ExecResponse> runScriptOnNodesMatching(
+      ClusterSpec spec, Predicate<NodeMetadata> condition, Statement statement,
+      RunScriptOptions options) throws IOException, RunScriptOnNodesException {
+
+    Credentials credentials = new Credentials(spec.getClusterUser(),
+        spec.getPrivateKey());
 
-    condition = Predicates.and(runningInGroup(spec.getClusterName()), condition);
+    if (options == null) {
+      options = defaultRunScriptOptionsForSpec(spec);
+    } else if (options.getOverridingCredentials() == null) {
+      options = options.overrideCredentialsWith(credentials);
+    }
+    condition = Predicates
+        .and(runningInGroup(spec.getClusterName()), condition);
+
+    ComputeServiceContext context = getCompute().apply(spec);
     return context.getComputeService().runScriptOnNodesMatching(condition,
-      statement, overrideCredentialsWith(credentials).wrapInInitScript(false).runAsRoot(false));
+        statement, options);
+  }
+
+  public RunScriptOptions defaultRunScriptOptionsForSpec(ClusterSpec spec) {
+    Credentials credentials = new Credentials(spec.getClusterUser(),
+        spec.getPrivateKey());
+    return overrideCredentialsWith(credentials).wrapInInitScript(false)
+        .runAsRoot(false);
   }
 
   @Deprecated

pom.xml

@@ -50,6 +50,7 @@
     <module>services/elasticsearch</module>
     <module>services/hama</module>
     <module>services/puppet</module>
+    <module>services/chef</module>
   </modules>
 
   <properties>

services/chef/pom.xml

@@ -0,0 +1,78 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.whirr</groupId>
+    <artifactId>whirr</artifactId>
+    <version>0.7.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <artifactId>whirr-chef</artifactId>
+  <packaging>jar</packaging>
+  <name>Apache Whirr Chef</name>
+  <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>whirr-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>whirr-core</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-configuration</groupId>
+      <artifactId>commons-configuration</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.jcraft</groupId>
+      <artifactId>jsch</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <scope>test</scope>
+    </dependency>  
+  </dependencies>
+</project>

services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandler.java

@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.chef;
+
+import static org.apache.whirr.service.chef.ChefClusterActionHandlerFactory.CHEF_ROLE_PREFIX;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+import java.io.IOException;
+
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+
+/**
+ * Installs chef-solo provisioned services, chef itself and its dependencies.
+ * 
+ */
+public class ChefClusterActionHandler extends ClusterActionHandlerSupport {
+
+  public static final String CHEF_SLEEP_AFTER_RECIPE = "whirr.chef.recipe.sleep";
+  public static final long CHEF_SLEEP_AFTER_RECIPE_DEFAULT = 2000L;
+
+  private String role;
+  private String cookbook;
+  private String recipe;
+  private long sleepAfterConfigure;
+
+  public ChefClusterActionHandler(String role) {
+    this.role = role;
+    parseCookbookAndRecipe();
+  }
+
+  @Override
+  public String getRole() {
+    return role;
+  }
+
+  @Override
+  protected void beforeBootstrap(ClusterActionEvent event) throws IOException,
+      InterruptedException {
+    if (isFirstChefRoleIn(event.getInstanceTemplate().getRoles()).apply(role)) {
+      addInstallChefStatements(event);
+      // for some reason even non-running recipes like ant are sometimes not
+      // immediately available after install. Jclouds seems to be behaving fine
+      // and file system stores should be atomic wrt to visibility, but there
+      // might be refresh or memory visibility issues wrt to how path context is
+      // built or maintained. Until the issue is dug up a small sleep time
+      // should take care of it
+      sleepAfterConfigure = event.getClusterSpec().getConfiguration()
+          .getLong(CHEF_SLEEP_AFTER_RECIPE, CHEF_SLEEP_AFTER_RECIPE_DEFAULT);
+    }
+  }
+
+  @Override
+  protected void beforeConfigure(ClusterActionEvent event) throws IOException,
+      InterruptedException {
+    // if the role is an exact match to the prefix then there is nothing to
+    // do (chef only installation)
+    if (role.equals("")) {
+      return;
+    }
+    addStatement(event, new Recipe(cookbook, recipe, event.getClusterSpec()
+        .getConfigurationForKeysWithPrefix(cookbook)));
+  }
+
+  private void addInstallChefStatements(ClusterActionEvent event) {
+    // install ruby and ruby-gems in the nodes
+    addStatement(event, call("install_ruby"));
+    // install git
+    addStatement(event, call("install_git"));
+    // install chef-solo
+    addStatement(event, call("install_chef"));
+  }
+
+  @Override
+  protected void afterConfigure(ClusterActionEvent event) throws IOException,
+      InterruptedException {
+    Thread.sleep(sleepAfterConfigure);
+  }
+
+  private void parseCookbookAndRecipe() {
+    String[] both = this.role.trim().split(":");
+    if (both.length < 1 || both.length > 3) {
+      throw new IllegalArgumentException(
+          "Chef roles must be specified the following way: \"chef\" to install chef with no recipes,"
+              + " \"chef:cookbook_name\" to install the default recipe in the provided cookbook,"
+              + " or \"chef:cookbook_name:recipe_name\" to  install a particular recipe [was: "
+              + role + "]");
+    }
+    switch (both.length) {
+    case 3:
+      recipe = both[2];
+    case 2:
+      cookbook = both[1];
+    }
+  }
+
+  private static Predicate<String> isFirstChefRoleIn(
+      final Iterable<String> roles) {
+    return new Predicate<String>() {
+      @Override
+      public boolean apply(String arg0) {
+        return Iterables.get(
+            Iterables.filter(roles,
+                Predicates.containsPattern("^" + CHEF_ROLE_PREFIX + arg0)), 0)
+            .equals(CHEF_ROLE_PREFIX + arg0);
+      }
+
+      @Override
+      public String toString() {
+        return "isFirstChefRoleIn(" + roles + ")";
+
+      }
+    };
+  }
+
+}

services/chef/src/main/java/org/apache/whirr/service/chef/ChefClusterActionHandlerFactory.java

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.chef;
+
+import org.apache.whirr.service.ClusterActionHandler;
+import org.apache.whirr.service.ClusterActionHandlerFactory;
+
+public class ChefClusterActionHandlerFactory extends
+    ClusterActionHandlerFactory {
+
+  public static final String CHEF_ROLE_PREFIX = "chef";
+
+  @Override
+  public String getRolePrefix() {
+    return CHEF_ROLE_PREFIX;
+  }
+
+  @Override
+  public ClusterActionHandler create(String roleName) {
+    return new ChefClusterActionHandler(roleName);
+  }
+
+}