ItUtils.java

/*
 * SonarQube
 * Copyright (C) 2009-2017 SonarSource SA
 * mailto:info AT sonarsource DOT com
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 3 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
package util;

import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.sonar.orchestrator.Orchestrator;
import com.sonar.orchestrator.build.BuildResult;
import com.sonar.orchestrator.build.SonarRunner;
import com.sonar.orchestrator.container.Server;
import com.sonar.orchestrator.locator.FileLocation;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.io.FileUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.sonar.wsclient.base.HttpException;
import org.sonar.wsclient.issue.Issue;
import org.sonar.wsclient.issue.IssueClient;
import org.sonar.wsclient.issue.IssueQuery;
import org.sonarqube.ws.WsComponents.Component;
import org.sonarqube.ws.WsMeasures;
import org.sonarqube.ws.WsMeasures.Measure;
import org.sonarqube.ws.client.GetRequest;
import org.sonarqube.ws.client.HttpConnector;
import org.sonarqube.ws.client.WsClient;
import org.sonarqube.ws.client.WsClientFactories;
import org.sonarqube.ws.client.component.ShowWsRequest;
import org.sonarqube.ws.client.measure.ComponentWsRequest;
import org.sonarqube.ws.client.organization.OrganizationService;
import org.sonarqube.ws.client.organization.SearchWsRequest;
import org.sonarqube.ws.client.qualityprofile.RestoreWsRequest;
import org.sonarqube.ws.client.setting.ResetRequest;
import org.sonarqube.ws.client.setting.SetRequest;

import static com.google.common.base.Preconditions.checkState;
import static com.sonar.orchestrator.container.Server.ADMIN_LOGIN;
import static com.sonar.orchestrator.container.Server.ADMIN_PASSWORD;
import static java.lang.Double.parseDouble;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static java.util.Locale.ENGLISH;
import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;

public class ItUtils {
  public static final Splitter LINE_SPLITTER = Splitter.on(System.getProperty("line.separator"));

  private ItUtils() {
  }

  public static FileLocation xooPlugin() {
    return FileLocation.byWildcardMavenFilename(new File("../../plugins/sonar-xoo-plugin/target"), "sonar-xoo-plugin-*.jar");
  }

  public static List<Issue> getAllServerIssues(Orchestrator orchestrator) {
    IssueClient issueClient = orchestrator.getServer().wsClient().issueClient();
    return issueClient.find(IssueQuery.create()).list();
  }

  public static WsClient newAdminWsClient(Orchestrator orchestrator) {
    return newUserWsClient(orchestrator, ADMIN_LOGIN, ADMIN_PASSWORD);
  }

  public static WsClient newWsClient(Orchestrator orchestrator) {
    return newUserWsClient(orchestrator, null, null);
  }

  public static WsClient newUserWsClient(Orchestrator orchestrator, @Nullable String login, @Nullable String password) {
    Server server = orchestrator.getServer();
    return WsClientFactories.getDefault().newClient(HttpConnector.newBuilder()
      .url(server.getUrl())
      .credentials(login, password)
      .build());
  }

  /**
   * Locate the directory of sample project
   *
   * @param relativePath path related to the directory it/it-projects, for example "qualitygate/xoo-sample"
   */
  public static File projectDir(String relativePath) {
    File dir = new File("../it-projects/" + relativePath);
    if (!dir.exists() || !dir.isDirectory()) {
      throw new IllegalStateException("Directory does not exist: " + dir.getAbsolutePath());
    }
    return dir;
  }

  /**
   * Locate the artifact of a fake plugin stored in it/it-plugins.
   *
   * @param dirName the directory of it/it-plugins, for example "sonar-fake-plugin".
   *                     It assumes that version is 1.0-SNAPSHOT
   */
  public static FileLocation pluginArtifact(String dirName) {
    return FileLocation.byWildcardMavenFilename(new File("../it-plugins/" + dirName + "/target"), dirName + "-*.jar");
  }

  /**
   * Locate the pom file of a sample project
   *
   * @param projectName project path related to the directory it/it-projects, for example "qualitygate/xoo-sample"
   */
  public static File projectPom(String projectName) {
    File pom = new File(projectDir(projectName), "pom.xml");
    if (!pom.exists() || !pom.isFile()) {
      throw new IllegalStateException("pom file does not exist: " + pom.getAbsolutePath());
    }
    return pom;
  }

  public static String sanitizeTimezones(String s) {
    return s.replaceAll("[\\+\\-]\\d\\d\\d\\d", "+0000");
  }

  public static JSONObject getJSONReport(BuildResult result) {
    Pattern pattern = Pattern.compile("Export issues to (.*?).json");
    Matcher m = pattern.matcher(result.getLogs());
    if (m.find()) {
      String s = m.group(1);
      File path = new File(s + ".json");
      assertThat(path).exists();
      try {
        return (JSONObject) JSONValue.parse(FileUtils.readFileToString(path));
      } catch (IOException e) {
        throw new RuntimeException("Unable to read JSON report", e);
      }
    }
    fail("Unable to locate json report");
    return null;
  }

  public static int countIssuesInJsonReport(BuildResult result, boolean onlyNews) {
    JSONObject obj = getJSONReport(result);
    JSONArray issues = (JSONArray) obj.get("issues");
    int count = 0;
    for (Object issue : issues) {
      JSONObject jsonIssue = (JSONObject) issue;
      if (!onlyNews || (Boolean) jsonIssue.get("isNew")) {
        count++;
      }
    }
    return count;
  }

  public static void assertIssuesInJsonReport(BuildResult result, int newIssues, int resolvedIssues, int existingIssues) {
    JSONObject obj = getJSONReport(result);
    JSONArray issues = (JSONArray) obj.get("issues");
    int countNew = 0;
    int countResolved = 0;
    int countExisting = 0;

    for (Object issue : issues) {
      JSONObject jsonIssue = (JSONObject) issue;

      if ((Boolean) jsonIssue.get("isNew")) {
        countNew++;
      } else if (jsonIssue.get("resolution") != null) {
        countResolved++;
      } else {
        countExisting++;
      }
    }
    assertThat(countNew).isEqualTo(newIssues);
    assertThat(countResolved).isEqualTo(resolvedIssues);
    assertThat(countExisting).isEqualTo(existingIssues);
  }

  public static SonarRunner runVerboseProjectAnalysis(Orchestrator orchestrator, String projectRelativePath, String... properties) {
    return runProjectAnalysis(orchestrator, projectRelativePath, true, properties);
  }

  public static SonarRunner runProjectAnalysis(Orchestrator orchestrator, String projectRelativePath, String... properties) {
    return runProjectAnalysis(orchestrator, projectRelativePath, false, properties);
  }

  private static SonarRunner runProjectAnalysis(Orchestrator orchestrator, String projectRelativePath, boolean enableDebugLogs, String... properties) {
    SonarRunner sonarRunner = SonarRunner.create(projectDir(projectRelativePath));
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    for (int i = 0; i < properties.length; i += 2) {
      builder.put(properties[i], properties[i + 1]);
    }
    SonarRunner scan = sonarRunner.setDebugLogs(enableDebugLogs).setProperties(builder.build());
    orchestrator.executeBuild(scan);
    return scan;
  }

  public static void setServerProperty(Orchestrator orchestrator, String key, @Nullable String value) {
    setServerProperty(orchestrator, null, key, value);
  }

  public static void setServerProperty(Orchestrator orchestrator, @Nullable String componentKey, String key, @Nullable String value) {
    if (value == null) {
      newAdminWsClient(orchestrator).settingsService().reset(ResetRequest.builder().setKeys(key).setComponent(componentKey).build());
    } else {
      newAdminWsClient(orchestrator).settingsService().set(SetRequest.builder().setKey(key).setValue(value).setComponent(componentKey).build());
    }
  }

  public static void setServerProperties(Orchestrator orchestrator, @Nullable String componentKey, String... properties) {
    for (int i = 0; i < properties.length; i += 2) {
      setServerProperty(orchestrator, componentKey, properties[i], properties[i + 1]);
    }
  }

  public static void resetSettings(Orchestrator orchestrator, @Nullable String componentKey, String... keys) {
    if (keys.length > 0) {
      newAdminWsClient(orchestrator).settingsService().reset(ResetRequest.builder().setKeys(keys).setComponent(componentKey).build());
    }
  }

  public static void resetEmailSettings(Orchestrator orchestrator) {
    resetSettings(orchestrator, null, "email.smtp_host.secured", "email.smtp_port.secured", "email.smtp_secure_connection.secured", "email.smtp_username.secured",
      "email.smtp_password.secured", "email.from", "email.prefix");
  }

  public static void resetPeriod(Orchestrator orchestrator) {
    resetSettings(orchestrator, null, "sonar.leak.period");
  }

  @CheckForNull
  public static Measure getMeasure(Orchestrator orchestrator, String componentKey, String metricKey) {
    return getMeasuresByMetricKey(orchestrator, componentKey, metricKey).get(metricKey);
  }

  @CheckForNull
  public static Double getMeasureAsDouble(Orchestrator orchestrator, String componentKey, String metricKey) {
    Measure measure = getMeasure(orchestrator, componentKey, metricKey);
    return (measure == null) ? null : Double.parseDouble(measure.getValue());
  }

  public static Map<String, Measure> getMeasuresByMetricKey(Orchestrator orchestrator, String componentKey, String... metricKeys) {
    return getStreamMeasures(orchestrator, componentKey, metricKeys)
      .filter(Measure::hasValue)
      .collect(Collectors.toMap(Measure::getMetric, Function.identity()));
  }

  public static Map<String, Double> getMeasuresAsDoubleByMetricKey(Orchestrator orchestrator, String componentKey, String... metricKeys) {
    return getStreamMeasures(orchestrator, componentKey, metricKeys)
      .filter(Measure::hasValue)
      .collect(Collectors.toMap(Measure::getMetric, measure -> parseDouble(measure.getValue())));
  }

  private static Stream<Measure> getStreamMeasures(Orchestrator orchestrator, String componentKey, String... metricKeys) {
    return newWsClient(orchestrator).measures().component(new ComponentWsRequest()
      .setComponentKey(componentKey)
      .setMetricKeys(asList(metricKeys)))
      .getComponent().getMeasuresList()
      .stream();
  }

  @CheckForNull
  public static Measure getMeasureWithVariation(Orchestrator orchestrator, String componentKey, String metricKey) {
    WsMeasures.ComponentWsResponse response = newWsClient(orchestrator).measures().component(new ComponentWsRequest()
      .setComponentKey(componentKey)
      .setMetricKeys(singletonList(metricKey))
      .setAdditionalFields(singletonList("periods")));
    List<Measure> measures = response.getComponent().getMeasuresList();
    return measures.size() == 1 ? measures.get(0) : null;
  }

  @CheckForNull
  public static Map<String, Measure> getMeasuresWithVariationsByMetricKey(Orchestrator orchestrator, String componentKey, String... metricKeys) {
    return newWsClient(orchestrator).measures().component(new ComponentWsRequest()
      .setComponentKey(componentKey)
      .setMetricKeys(asList(metricKeys))
      .setAdditionalFields(singletonList("periods"))).getComponent().getMeasuresList()
      .stream()
      .collect(Collectors.toMap(Measure::getMetric, Function.identity()));
  }

  /**
   * Return leak period value
   */
  @CheckForNull
  public static Double getLeakPeriodValue(Orchestrator orchestrator, String componentKey, String metricKey) {
    List<WsMeasures.PeriodValue> periodsValueList = getMeasureWithVariation(orchestrator, componentKey, metricKey).getPeriods().getPeriodsValueList();
    return periodsValueList.size() > 0 ? Double.parseDouble(periodsValueList.get(0).getValue()) : null;
  }

  @CheckForNull
  public static Component getComponent(Orchestrator orchestrator, String componentKey) {
    try {
      return newWsClient(orchestrator).components().show(new ShowWsRequest().setKey((componentKey))).getComponent();
    } catch (org.sonarqube.ws.client.HttpException e) {
      if (e.code() == 404) {
        return null;
      }
      throw new IllegalStateException(e);
    }
  }

  @CheckForNull
  public static ComponentNavigation getComponentNavigation(Orchestrator orchestrator, String componentKey) {
    // Waiting for SONAR-7745 to have version in api/components/show, we use internal api/navigation/component WS to get the component
    // version
    String content = newWsClient(orchestrator).wsConnector().call(new GetRequest("api/navigation/component").setParam("componentKey", componentKey)).failIfNotSuccessful()
      .content();
    return ComponentNavigation.parse(content);
  }

  public static void restoreProfile(Orchestrator orchestrator, URL resource) {
    restoreProfile(orchestrator, resource, null);
  }

  public static void restoreProfile(Orchestrator orchestrator, URL resource, String organization) {
    URI uri;
    try {
      uri = resource.toURI();
    } catch (URISyntaxException e) {
      throw new IllegalArgumentException("Cannot find quality profile xml file '" + resource + "' in classpath");
    }
    newAdminWsClient(orchestrator)
      .qualityProfiles()
      .restoreProfile(
        RestoreWsRequest.builder()
          .setBackup(new File(uri))
          .setOrganization(organization)
          .build());
  }

  public static String newOrganizationKey() {
    return randomAlphabetic(32).toLowerCase(ENGLISH);
  }

  public static String newProjectKey() {
    return "key-" + randomAlphabetic(200);
  }

  public static void deleteOrganizationsIfExists(Orchestrator orchestrator, String... organizationKeys) {
    OrganizationService adminOrganizationService = newAdminWsClient(orchestrator).organizations();
    adminOrganizationService.search(SearchWsRequest.builder().setOrganizations(organizationKeys).build()).getOrganizationsList()
      .forEach(organization -> adminOrganizationService.delete(organization.getKey()));
  }

  public static class ComponentNavigation {
    private String version;
    private String snapshotDate;

    public String getVersion() {
      return version;
    }

    public Date getDate() {
      return toDatetime(snapshotDate);
    }

    public static ComponentNavigation parse(String json) {
      Gson gson = new Gson();
      return gson.fromJson(json, ComponentNavigation.class);
    }
  }

  /**
   * Concatenates a vararg to a String array.
   *
   * Useful when using {@link #runVerboseProjectAnalysis(Orchestrator, String, String...)}, eg.:
   * <pre>
   * ItUtils.runProjectAnalysis(orchestrator, "project_name",
   *    ItUtils.concat(properties, "sonar.scm.disabled", "false")
   *    );
   * </pre>
   */
  public static String[] concat(String[] properties, String... str) {
    if (properties == null || properties.length == 0) {
      return str;
    }
    return Stream.concat(Arrays.stream(properties), Arrays.stream(str))
      .toArray(String[]::new);
  }

  public static void verifyHttpException(Exception e, int expectedCode) {
    assertThat(e).isInstanceOf(HttpException.class);
    HttpException exception = (HttpException) e;
    assertThat(exception.status()).isEqualTo(expectedCode);
  }

  public static Date toDate(String sDate) {
    try {
      SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
      return sdf.parse(sDate);
    } catch (ParseException e) {
      throw new RuntimeException(e);
    }
  }

  public static Date toDatetime(String sDate) {
    try {
      SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
      return sdf.parse(sDate);
    } catch (ParseException e) {
      throw new RuntimeException(e);
    }
  }

  public static String formatDate(Date d) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
    return sdf.format(d);
  }

  public static String extractCeTaskId(BuildResult buildResult) {
    List<String> taskIds = extractCeTaskIds(buildResult);
    checkState(taskIds.size() == 1, "More than one task id retrieved from logs");
    return taskIds.iterator().next();
  }

  public static List<String> extractCeTaskIds(BuildResult buildResult) {
    String logs = buildResult.getLogs();
    return StreamSupport.stream(LINE_SPLITTER.split(logs).spliterator(), false)
      .filter(s -> s.contains("More about the report processing at"))
      .map(s -> s.substring(s.length() - 20, s.length()))
      .collect(Collectors.toList());
  }

  public static Map<String, Object> jsonToMap(String json) {
    Gson gson = new Gson();
    Type type = new TypeToken<Map<String, Object>>() {
    }.getType();
    return gson.fromJson(json, type);
  }

  public static Response call(String url, String... headers) {
    Request.Builder requestBuilder = new Request.Builder().get().url(url);
    for (int i = 0; i < headers.length; i += 2) {
      String headerName = headers[i];
      String headerValue = headers[i + 1];
      if (headerValue != null) {
        requestBuilder.addHeader(headerName, headerValue);
      }
    }
    try {
      return new OkHttpClient.Builder()
        .connectTimeout(30, TimeUnit.SECONDS)
        .readTimeout(30, TimeUnit.SECONDS)
        .build()
        .newCall(requestBuilder.build())
        .execute();
    } catch (IOException e) {
      throw Throwables.propagate(e);
    }
  }
}