Skip to content

Commit

Permalink
Move tests persistence from batch to compute - SONAR-6255
Browse files Browse the repository at this point in the history
- define index and check the 3 use cases
- define batch report format
- create file_sources.test_data
  • Loading branch information
teryk committed Apr 16, 2015
1 parent 8ad882a commit 598bce5
Show file tree
Hide file tree
Showing 47 changed files with 6,236 additions and 180 deletions.
Expand Up @@ -20,9 +20,8 @@
package org.sonar.server.activity.index;

import com.google.common.collect.ImmutableMap;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.sonar.api.config.Settings;
import org.sonar.process.ProcessProperties;
import org.sonar.server.es.EsUtils;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.NewIndex;

Expand Down Expand Up @@ -50,16 +49,9 @@ public ActivityIndexDefinition(Settings settings) {
@Override
public void define(IndexDefinitionContext context) {
NewIndex index = context.create(INDEX);
index.getSettings().put("index.refresh_interval", "-1");
index.getSettings().put("analysis.analyzer.default.type", "keyword");

// shards
boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
if (clusterMode) {
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4);
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
// else keep defaults (one shard)
}
EsUtils.refreshHandledByIndexer(index);
EsUtils.setShards(index, settings);

// type "activity"
NewIndex.NewIndexType mapping = index.createType(TYPE);
Expand Down
Expand Up @@ -28,7 +28,7 @@ public BaseIndex(EsClient client) {
this.client = client;
}

public EsClient getClient() {
protected EsClient getClient() {
return client;
}

Expand Down
16 changes: 16 additions & 0 deletions server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
Expand Up @@ -21,10 +21,13 @@

import com.google.common.base.Function;
import com.google.common.collect.Lists;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.joda.time.format.ISODateTimeFormat;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.sonar.api.config.Settings;
import org.sonar.process.ProcessProperties;
import org.sonar.server.search.BaseDoc;

import javax.annotation.CheckForNull;
Expand Down Expand Up @@ -79,4 +82,17 @@ public static String formatDateTime(@Nullable Date date) {
}
return null;
}

public static void setShards(NewIndex index, Settings settings) {
boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
if (clusterMode) {
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4);
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
// else keep defaults (one shard)
}
}

public static void refreshHandledByIndexer(NewIndex index) {
index.getSettings().put("index.refresh_interval", "-1");
}
}
Expand Up @@ -27,7 +27,6 @@
import org.sonar.server.search.IndexField;

import javax.annotation.CheckForNull;

import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
Expand Down Expand Up @@ -72,6 +71,10 @@ public StringFieldBuilder stringFieldBuilder(String fieldName) {
return new StringFieldBuilder(this, fieldName);
}

public NestedObjectBuilder nestedObjectBuilder(String fieldName, NewIndexType nestedMapping) {
return new NestedObjectBuilder(this, nestedMapping, fieldName);
}

public NewIndexType createBooleanField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "boolean"));
}
Expand All @@ -97,7 +100,7 @@ public NewIndexType createLongField(String fieldName) {
}

public NewIndexType createDynamicNestedField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true"));
return setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true", "include_in_parent", "true"));
}

public NewIndexType createShortField(String fieldName) {
Expand All @@ -121,6 +124,33 @@ public Object getProperty(String key) {
}
}

public static class NestedObjectBuilder {
private final NewIndexType indexType;
private final NewIndexType nestedType;
private final String fieldName;
private boolean dynamic = false;

public NestedObjectBuilder(NewIndexType indexType, NewIndexType nestedType, String fieldName) {
this.indexType = indexType;
this.nestedType = nestedType;
this.fieldName = fieldName;
}

public NestedObjectBuilder dynamic() {
this.dynamic = true;
return this;
}

public void build() {
if (dynamic) {
indexType.setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true"));
} else {
nestedType.setAttribute("type", "nested");
indexType.setProperty(fieldName, nestedType.attributes);
}
}
}

/**
* Helper to define a string field in mapping of index type
*/
Expand Down
Expand Up @@ -20,9 +20,8 @@
package org.sonar.server.issue.index;

import com.google.common.collect.ImmutableMap;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.sonar.api.config.Settings;
import org.sonar.process.ProcessProperties;
import org.sonar.server.es.EsUtils;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.NewIndex;

Expand Down Expand Up @@ -93,16 +92,8 @@ public IssueIndexDefinition(Settings settings) {
public void define(IndexDefinitionContext context) {
NewIndex index = context.create(INDEX);

// refresh is handled by IssueIndexer
index.getSettings().put("index.refresh_interval", "-1");

// shards
boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
if (clusterMode) {
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4);
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
// else keep defaults (one shard)
}
EsUtils.refreshHandledByIndexer(index);
EsUtils.setShards(index, settings);

// type "authorization"
NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION);
Expand Down
Expand Up @@ -20,9 +20,8 @@
package org.sonar.server.source.index;

import com.google.common.collect.ImmutableMap;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.sonar.api.config.Settings;
import org.sonar.process.ProcessProperties;
import org.sonar.server.es.EsUtils;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.NewIndex;

Expand Down Expand Up @@ -61,16 +60,8 @@ public SourceLineIndexDefinition(Settings settings) {
public void define(IndexDefinitionContext context) {
NewIndex index = context.create(INDEX);

// refresh is always handled by SourceLineIndexer
index.getSettings().put("index.refresh_interval", "-1");

// shards
boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
if (clusterMode) {
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4);
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
// else keep defaults (one shard)
}
EsUtils.refreshHandledByIndexer(index);
EsUtils.setShards(index, settings);

// type "sourceline"
NewIndex.NewIndexType mapping = index.createType(TYPE);
Expand Down
@@ -0,0 +1,105 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.server.test.index;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import org.sonar.server.search.BaseDoc;

import java.util.List;
import java.util.Map;

import static org.sonar.server.test.index.TestIndexDefinition.*;

public class TestDoc extends BaseDoc {
public TestDoc(Map<String, Object> fields) {
super(fields);
}

@VisibleForTesting
TestDoc() {
super(Maps.<String, Object>newHashMapWithExpectedSize(7));
}

public String uuid() {
return getField(FIELD_UUID);
}

public TestDoc setUuid(String uuid) {
setField(FIELD_UUID, uuid);
return this;
}

public String name() {
return getField(FIELD_NAME);
}

public TestDoc setName(String name) {
setField(FIELD_NAME, name);
return this;
}

public String status() {
return getField(FIELD_STATUS);
}

public TestDoc setStatus(String status) {
setField(FIELD_STATUS, status);
return this;
}

public String message() {
return getField(FIELD_MESSAGE);
}

public TestDoc setMessage(String message) {
setField(FIELD_MESSAGE, message);
return this;
}

public String stackTrace() {
return getField(FIELD_STACKTRACE);
}

public TestDoc setStackTrace(String stackTrace) {
setField(FIELD_STACKTRACE, stackTrace);
return this;
}

public String type() {
return getField(FIELD_TYPE);
}

public TestDoc setType(String type) {
setField(FIELD_TYPE, type);
return this;
}

// TODO TBE - it should be a CoverageBlockDoc list
public List<Map<String, Object>> coverageBlocks() {
return getField(FIELD_COVERAGE_BLOCKS);
}

public TestDoc setCoverageBlocks(List<Map<String, Object>> coverageBlocks) {
setField(FIELD_COVERAGE_BLOCKS, coverageBlocks);
return this;
}
}
@@ -0,0 +1,85 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.server.test.index;

import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.sonar.server.es.BaseIndex;
import org.sonar.server.es.EsClient;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import static org.sonar.server.test.index.TestIndexDefinition.*;

public class TestIndex extends BaseIndex {
public TestIndex(EsClient client) {
super(client);
}

public List<Map<String, Object>> coveredLines(String testFileUuid, String methodName) {
List<Map<String, Object>> coverageBlocks = new ArrayList<>();

for (SearchHit hit : getClient().prepareSearch(TestIndexDefinition.INDEX)
.setTypes(TestIndexDefinition.TYPE)
.setSize(1)
.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.boolFilter()
.must(FilterBuilders.termFilter(FIELD_UUID, testFileUuid).cache(false))
.must(FilterBuilders.termFilter(TestIndexDefinition.FIELD_NAME, methodName).cache(false))))
.get().getHits().getHits()) {
coverageBlocks.addAll(new TestDoc(hit.sourceAsMap()).coverageBlocks());
}

return coverageBlocks;
}

public List<TestDoc> testMethods(String testFileUuid) {
List<TestDoc> testDocs = new ArrayList<>();

for (SearchHit hit : getClient().prepareSearch(TestIndexDefinition.INDEX)
.setTypes(TestIndexDefinition.TYPE)
.setSize(10_000)
.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termFilter(FIELD_UUID, testFileUuid)))
.get().getHits().getHits()) {
testDocs.add(new TestDoc(hit.sourceAsMap()));
}

return testDocs;
}

public List<TestDoc> testsCovering(String mainFileUuid, int line) {
List<TestDoc> testDocs = new ArrayList<>();

for (SearchHit hit : getClient().prepareSearch(TestIndexDefinition.INDEX)
.setTypes(TestIndexDefinition.TYPE)
.setSize(10_000)
.setQuery(QueryBuilders.nestedQuery(FIELD_COVERAGE_BLOCKS, FilterBuilders.boolFilter()
.must(FilterBuilders.termFilter(FIELD_COVERAGE_BLOCKS + "." + FIELD_COVERAGE_BLOCK_UUID, mainFileUuid).cache(false))
.must(FilterBuilders.termFilter(FIELD_COVERAGE_BLOCKS + "." + FIELD_COVERAGE_BLOCK_LINES, line).cache(false))))
.get().getHits().getHits()) {
testDocs.add(new TestDoc(hit.sourceAsMap()));
}

return testDocs;
}
}

0 comments on commit 598bce5

Please sign in to comment.