Added support for vRealize Operations Cloud (#53)

* Added support for vR Ops Cloud

* Bumped version. Added documentation

README.md

@@ -101,27 +101,35 @@ chmod +x exporttool.sh
 ## Command syntax
 
 ```
-usage: exporttool [-d <arg>] [-e <arg>] [-F <arg>] [-H <arg>] [-h] [-i]
-       [-l <arg>] [-m <arg>] [-n <arg>] [-o <arg>] [-P <arg>] [-p <arg>]
-       [-q] [-R <arg>] [-s <arg>] [-S] [-t <arg>] [-T <arg>] [--trustpass
-       <arg>] [-u <arg>] [-v]
-
+usage: exporttool [-d <arg>] [--dumprest] [-e <arg>] [-F <arg>] [-G <arg>]
+       [-H <arg>] [-h] [-i] [-l <arg>] [-m <arg>] [-n <arg>]
+       [--no-sniextension] [-o <arg>] [-P <arg>] [-p <arg>] [-q] [-r
+       <arg>] [-R <arg>] [--resfetch <arg>] [-s <arg>] [-S] [-t <arg>] [-T
+       <arg>] [--trustpass <arg>] [-u <arg>] [-v]
+Exports vRealize Operations Metrics
  -d,--definition <arg>       Path to definition file
+    --dumprest               Dump rest calls to output
  -e,--end <arg>              Time period end (date format in definition
                              file)
  -F,--list-fields <arg>      Print name and keys of all fields to stdout
+ -G,--generate <arg>         Generate template definition for resource
+                             type
  -H,--host <arg>             URL to vRealize Operations Host
  -h,--help                   Print a short help text
  -i,--ignore-cert            Trust any cert (DEPRECATED!)
  -l,--lookback <arg>         Lookback time
- -m,--max-rows <arg>         Maximum number of rows to fetch 
+ -m,--max-rows <arg>         Maximum number of rows to fetch
                              (default=1000*thread count)
  -n,--namequery <arg>        Name query
+    --no-sniextension        Disable SNI extension. May be needed for very
+                             old SSL implementations
  -o,--output <arg>           Output file
  -P,--parent <arg>           Parent resource (ResourceKind:resourceName)
  -p,--password <arg>         Password
  -q,--quiet                  Quiet mode (no progress counter)
+ -r,--refreshtoken <arg>     Refresh token
  -R,--resource-kinds <arg>   List resource kinds
+    --resfetch <arg>         Resource fetch count (default=1000)
  -s,--start <arg>            Time period start (date format in definition
                              file)
  -S,--streaming              True streaming processing. Faster but less
@@ -141,12 +149,32 @@ it encounters an untrusted certificate. If the user chooses to trust the certifi
 truststore and reused next time the tool is executed against that host. By default, the trusted certs are stored in
 $HOME/.vropsexport/truststore, but the location can be overridden using the -T flag.
 
+### TLS handshake issues
+
+In some cases, especially with very old SSL/TLS implementations, you may see name verification errors. To remedy this,
+please try the --no-sniextension command-line argument.
+
 ### Authentication
 
 The vrops-export tool uses the local authentication source by default. Other sources can be specified by using the "
 username@source" syntax. Be aware that in the case of e.g. Active Directory, the string after the @-sign should be the
 name of the *authentication source* and not that of the Active Directory domain.
 
+### Support for vRealize Operations Cloud
+
+As of version 3.2.0, vRealize Operations Cloud is supported. To export data from a cloud instance, use the following API
+endpoind URLs as arguments to the `-H` option:
+
+United States:
+`https://www.mgmt.cloud.vmware.com/vrops-cloud/suite-api`
+
+All other countries:
+`https://<country>.www.mgmt.cloud.vmware.com/vrops-cloud/suite-api`
+Where `country` is the country code of your API endpoint. Should match the country code in your browser URL when
+interacting with vRealize Operations Cloud.
+
+Use the `-r` option to log in using a refresh token/API token.
+
 ### Notes:
 
 * Start and end dates will use the date format specified in the definition file. Since dates tend to contain spaces and

pom.xml

@@ -17,7 +17,7 @@
             </snapshots>
         </repository>
     </repositories>
-    <version>3.0.1</version>
+    <version>3.2.0</version>
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>

src/bin/exporttool.bat

@@ -25,4 +25,4 @@ SET CP=%DIR%\vrops-export-%VERSION%.jar
 IF [%JDBC_JAR%] == [] GOTO :NO_JDBC_JAR
 SET CP=%CP%;%JDBC_JAR%
 :NO_JDBC_JAR
-JAVA -cp %CP% -Djsse.enableSNIExtension=false com.vmware.vropsexport.Main %*
+JAVA -cp %CP% com.vmware.vropsexport.Main %*

src/bin/exporttool.sh

@@ -28,4 +28,4 @@ if [ -n "$JDBC_JAR" ]
 then
 	CP=$CP:$JDBC_JAR
 fi
-$JAVA -cp $CP -Djsse.enableSNIExtension=false com.vmware.vropsexport.Main "$@"
+$JAVA -cp $CP com.vmware.vropsexport.Main "$@"

src/main/java/com/vmware/vropsexport/Client.java

@@ -22,23 +22,14 @@
 import com.vmware.vropsexport.exceptions.ExporterException;
 import com.vmware.vropsexport.models.AuthRequest;
 import com.vmware.vropsexport.models.AuthResponse;
+import com.vmware.vropsexport.models.TokenAuthResponse;
 import com.vmware.vropsexport.security.ExtendableTrustStrategy;
 import com.vmware.vropsexport.security.RecoverableCertificateException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.security.KeyManagementException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.X509Certificate;
-import java.util.List;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLHandshakeException;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpException;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
@@ -54,6 +45,18 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLHandshakeException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+import java.util.List;
+
 @SuppressWarnings("WeakerAccess")
 public class Client {
   private static final Logger log = LogManager.getLogger(Client.class);
@@ -68,16 +71,15 @@ public class Client {
 
   private final String urlBase;
 
-  private final String authToken;
+  private String authToken;
 
   private final boolean dumpRest;
 
-  public Client(
-      final String urlBase,
-      String username,
-      final String password,
-      final KeyStore extendedTrust,
-      final boolean dumpRest)
+  private String tokenPrefix;
+
+  private final ExtendableTrustStrategy trustStrategy;
+
+  public Client(final String urlBase, final KeyStore extendedTrust, final boolean dumpRest)
       throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException, IOException,
           HttpException, ExporterException {
     this.urlBase = urlBase;
@@ -89,12 +91,12 @@ public Client(
             .setConnectTimeout(CONNTECTION_TIMEOUT_MS)
             .setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MS)
             .setSocketTimeout(SOCKET_TIMEOUT_MS)
+            .setCookieSpec(CookieSpecs.STANDARD)
             .build();
 
-    final ExtendableTrustStrategy extendedTrustStrategy =
-        new ExtendableTrustStrategy(extendedTrust);
+    trustStrategy = new ExtendableTrustStrategy(extendedTrust);
     final SSLContext sslContext =
-        SSLContexts.custom().loadTrustMaterial(null, extendedTrustStrategy).build();
+        SSLContexts.custom().loadTrustMaterial(null, trustStrategy).build();
     final SSLConnectionSocketFactory sslf =
         new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
     final Registry<ConnectionSocketFactory> socketFactoryRegistry =
@@ -108,41 +110,67 @@ public Client(
             .setSSLSocketFactory(sslf)
             .setConnectionManager(cm)
             .setDefaultRequestConfig(requestConfig)
-            .
-            // setRetryHandler(new DefaultHttpRequestRetryHandler(3, false)).
-            setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+            .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
             .build();
 
-    // Authenticate
-    //
     try {
-      // User may be in a non-local auth source.
-      //
-      int p = username.indexOf('\\');
-      String authSource = null;
-      if (p != -1) {
-        authSource = username.substring(0, p);
-        username = username.substring(p + 1);
-      } else {
-        p = username.indexOf('@');
-        if (p != -1) {
-          authSource = username.substring(p + 1);
-          username = username.substring(0, p);
-        }
-      }
-      final AuthRequest rq = new AuthRequest(authSource, username, password);
-      final AuthResponse response =
-          postJsonReturnJson("/suite-api/api/auth/token/acquire", rq, AuthResponse.class);
-      authToken = response.getToken();
+      // Make a dummy API call to make sure we have our certs in order
+      log.debug("Hitting dummy URL to get certs");
+      getStream("/suite-api/api/resources");
     } catch (final SSLHandshakeException e) {
       // If we captured a cert, it's recoverable by asking the user to trust it.
       //
-      final X509Certificate[] cc = extendedTrustStrategy.getCapturedCerts();
+      final X509Certificate[] cc = trustStrategy.getCapturedCerts();
       if (cc == null) {
         throw e;
       }
       throw new RecoverableCertificateException(cc, e);
+    } catch (final HttpException e) {
+      // Ignore all other exceptions. We'll probably get a 401 here.
+    }
+  }
+
+  public Client login(final String apiToken) throws HttpException, IOException {
+    final HttpPost post =
+        new HttpPost(
+            "https://console.cloud.vmware.com/csp/gateway/am/api/auth/api-tokens/authorize");
+    post.addHeader("Accept", "application/json");
+    post.addHeader("Accept-Encoding", "gzip");
+    post.addHeader("Content-Type", "application/x-www-form-urlencoded");
+    post.setEntity(new StringEntity("refresh_token=" + apiToken));
+
+    final HttpResponse resp = client.execute(post);
+    checkResponse(resp);
+    final TokenAuthResponse tokenResp =
+        getObjectMapper().readValue(resp.getEntity().getContent(), TokenAuthResponse.class);
+    authToken = tokenResp.getAccess_token();
+    tokenPrefix = "CSPToken ";
+    return this;
+  }
+
+  public Client login(String username, final String password)
+      throws HttpException, IOException, RecoverableCertificateException {
+
+    // User may be in a non-local auth source.
+    //
+    int p = username.indexOf('\\');
+    String authSource = null;
+    if (p != -1) {
+      authSource = username.substring(0, p);
+      username = username.substring(p + 1);
+    } else {
+      p = username.indexOf('@');
+      if (p != -1) {
+        authSource = username.substring(p + 1);
+        username = username.substring(0, p);
+      }
     }
+    final AuthRequest rq = new AuthRequest(authSource, username, password);
+    final AuthResponse response =
+        postJsonReturnJson("/suite-api/api/auth/token/acquire", rq, AuthResponse.class);
+    authToken = response.getToken();
+    tokenPrefix = "vRealizeOpsToken ";
+    return this;
   }
 
   public <T> T getJson(final String uri, final Class<T> responseClass, final String... queries)
@@ -168,7 +196,7 @@ private HttpResponse innerGet(String uri, final String... queries)
     get.addHeader("Accept", "application/json");
     get.addHeader("Accept-Encoding", "gzip");
     if (authToken != null) {
-      get.addHeader("Authorization", "vRealizeOpsToken " + authToken + "");
+      get.addHeader("Authorization", tokenPrefix + authToken);
     }
     final HttpResponse resp = client.execute(get);
     checkResponse(resp);
@@ -183,7 +211,7 @@ public InputStream postJsonReturnStream(final String uri, final Object payload)
     post.addHeader("Content-Type", "application/json");
     post.addHeader("Accept-Encoding", "gzip");
     if (authToken != null) {
-      post.addHeader("Authorization", "vRealizeOpsToken " + authToken + "");
+      post.addHeader("Authorization", tokenPrefix + authToken + "");
     }
     if (dumpRest) {
       log.debug("POST " + urlBase + uri);

src/main/java/com/vmware/vropsexport/Exporter.java

@@ -18,43 +18,8 @@
 package com.vmware.vropsexport;
 
 import com.vmware.vropsexport.exceptions.ExporterException;
-import com.vmware.vropsexport.models.MetricsRequest;
-import com.vmware.vropsexport.models.NamedResource;
-import com.vmware.vropsexport.models.PageOfResources;
-import com.vmware.vropsexport.models.PropertiesResponse;
-import com.vmware.vropsexport.models.ResourceKind;
-import com.vmware.vropsexport.models.ResourceKindResponse;
-import com.vmware.vropsexport.models.ResourceStatKeysResponse;
-import com.vmware.vropsexport.models.StatKeysResponse;
-import com.vmware.vropsexport.processors.CSVPrinter;
-import com.vmware.vropsexport.processors.ElasticSearchIndexer;
-import com.vmware.vropsexport.processors.JsonPrinter;
-import com.vmware.vropsexport.processors.SQLDumper;
-import com.vmware.vropsexport.processors.WavefrontPusher;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
-import java.security.KeyManagementException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.stream.Collectors;
+import com.vmware.vropsexport.models.*;
+import com.vmware.vropsexport.processors.*;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpException;
 import org.apache.http.NoHttpResponseException;
@@ -67,6 +32,16 @@
 import org.yaml.snakeyaml.nodes.Tag;
 import org.yaml.snakeyaml.representer.Representer;
 
+import java.io.*;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
+
 @SuppressWarnings("SameParameterValue")
 public class Exporter implements DataProvider {
   private static class Progress implements ProgressMonitor {
@@ -131,19 +106,14 @@ public static boolean isProducingOutput(final Config conf) {
   }
 
   public Exporter(
-      final String urlBase,
-      final String username,
-      final String password,
+      final Client client,
       final int threads,
       final Config conf,
       final boolean verbose,
-      final boolean dumpRest,
       final boolean useTempFile,
       final int maxRows,
-      final int maxResourceFetch,
-      final KeyStore extendedTrust)
-      throws IOException, HttpException, KeyStoreException, NoSuchAlgorithmException,
-          KeyManagementException, ExporterException {
+      final int maxResourceFetch)
+      throws ExporterException {
     if (conf != null) {
       rspFactory = rspFactories.get(conf.getOutputFormat());
       if (rspFactory == null) {
@@ -156,7 +126,7 @@ public Exporter(
     this.conf = conf;
     this.maxRows = maxRows;
     this.maxResourceFetch = maxResourceFetch;
-    client = new Client(urlBase, username, password, extendedTrust, dumpRest);
+    this.client = client;
 
     executor =
         new ThreadPoolExecutor(