Skip to content

Commit

Permalink
SONAR-6258 Remove hack setting updated_at to 0 by only indexing file …
Browse files Browse the repository at this point in the history 
…sources of current project
  • Loading branch information
@julienlancelot
julienlancelot committed Apr 17, 2015
1 parent ae3b5f5 commit 803a343
Show file tree
Hide file tree
Showing 17 changed files with 277 additions and 101 deletions.
2 changes: 1 addition & 1 deletion ...nar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java
View file
Expand Up @@ -73,7 +73,7 @@ private void scrollRows() throws SQLException {

try {
long start = System.currentTimeMillis();
SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L);
SourceFileResultSetIterator it = SourceFileResultSetIterator.create(dbClient, connection, 0L, null);
while (it.hasNext()) {
SourceFileResultSetIterator.Row row = it.next();
assertThat(row.getLineUpdateRequests().size()).isEqualTo(NUMBER_OF_LINES);
Expand Down
21 changes: 13 additions & 8 deletions server/sonar-server/src/main/java/org/sonar/server/computation/source/SymbolsLineReader.java
View file
Expand Up @@ -24,11 +24,16 @@
import org.sonar.server.source.db.FileSourceDb;

import java.io.Serializable;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;

public class SymbolsLineReader implements LineReader {

Expand All @@ -39,7 +44,7 @@ public SymbolsLineReader(List<BatchReport.Symbols.Symbol> symbols) {
this.symbols = newArrayList(symbols);
// Sort symbols to have deterministic results and avoid false variation that would lead to an unnecessary update of the source files
// data
Collections.sort(this.symbols, new SymbolsDuplication());
Collections.sort(this.symbols, new SymbolsComparator());

this.idsBySymbol = createIdsBySymbolMap(this.symbols);
}
Expand Down Expand Up @@ -78,8 +83,8 @@ private void appendSymbol(StringBuilder lineSymbol, BatchReport.Range range, int
}

private List<BatchReport.Symbols.Symbol> findSymbolsMatchingLine(int line) {
List<BatchReport.Symbols.Symbol> lineSymbols = newArrayList();
Set<BatchReport.Symbols.Symbol> symbolsIndex = newHashSet();
List<BatchReport.Symbols.Symbol> lineSymbols = new ArrayList<>();
Set<BatchReport.Symbols.Symbol> symbolsIndex = new HashSet<>();
for (BatchReport.Symbols.Symbol symbol : symbols) {
if (matchLine(symbol.getDeclaration(), line) && !symbolsIndex.contains(symbol)) {
lineSymbols.add(symbol);
Expand All @@ -101,7 +106,7 @@ private static boolean matchLine(BatchReport.Range range, int line) {
}

private Map<BatchReport.Symbols.Symbol, Integer> createIdsBySymbolMap(List<BatchReport.Symbols.Symbol> symbols) {
Map<BatchReport.Symbols.Symbol, Integer> map = newHashMap();
Map<BatchReport.Symbols.Symbol, Integer> map = new HashMap<>();
int symbolId = 1;
for (BatchReport.Symbols.Symbol symbol : symbols) {
map.put(symbol, symbolId);
Expand All @@ -110,7 +115,7 @@ private Map<BatchReport.Symbols.Symbol, Integer> createIdsBySymbolMap(List<Batch
return map;
}

private static class SymbolsDuplication implements Comparator<BatchReport.Symbols.Symbol>, Serializable {
private static class SymbolsComparator implements Comparator<BatchReport.Symbols.Symbol>, Serializable {
@Override
public int compare(BatchReport.Symbols.Symbol o1, BatchReport.Symbols.Symbol o2) {
if (o1.getDeclaration().getStartLine() == o2.getDeclaration().getStartLine()) {
Expand Down
25 changes: 2 additions & 23 deletions ...er/sonar-server/src/main/java/org/sonar/server/computation/step/IndexSourceLinesStep.java
View file
Expand Up @@ -20,22 +20,14 @@
package org.sonar.server.computation.step;

import org.sonar.api.resources.Qualifiers;
import org.sonar.api.utils.System2;
import org.sonar.core.persistence.DbSession;
import org.sonar.core.persistence.MyBatis;
import org.sonar.server.computation.ComputationContext;
import org.sonar.server.db.DbClient;
import org.sonar.server.source.index.SourceLineIndexer;

public class IndexSourceLinesStep implements ComputationStep {

private final DbClient dbClient;
private final System2 system2;
private final SourceLineIndexer indexer;

public IndexSourceLinesStep(DbClient dbClient, System2 system2, SourceLineIndexer indexer) {
this.dbClient = dbClient;
this.system2 = system2;
public IndexSourceLinesStep(SourceLineIndexer indexer) {
this.indexer = indexer;
}

Expand All @@ -46,20 +38,7 @@ public String[] supportedProjectQualifiers() {

@Override
public void execute(ComputationContext context) {
updateSourceUpdateDate(context.getProject().uuid());
indexer.index();
}

// Temporary solution to only index in E/S updated sources from current project
// Should be removed when source wil be persisted in compute engine
private void updateSourceUpdateDate(String projectUuid){
DbSession session = dbClient.openSession(true);
try {
dbClient.fileSourceDao().updateDateWhenUpdatedDateIsZero(session, projectUuid, system2.now());
session.commit();
} finally {
MyBatis.closeQuietly(session);
}
indexer.index(context.getProject().uuid());
}

@Override
Expand Down
22 changes: 13 additions & 9 deletions .../sonar-server/src/main/java/org/sonar/server/computation/step/PersistFileSourcesStep.java
View file
Expand Up @@ -36,18 +36,24 @@
import org.sonar.core.persistence.MyBatis;
import org.sonar.core.source.db.FileSourceDto;
import org.sonar.server.computation.ComputationContext;
import org.sonar.server.computation.source.*;
import org.sonar.server.computation.source.ComputeFileSourceData;
import org.sonar.server.computation.source.CoverageLineReader;
import org.sonar.server.computation.source.DuplicationLineReader;
import org.sonar.server.computation.source.HighlightingLineReader;
import org.sonar.server.computation.source.LineReader;
import org.sonar.server.computation.source.ReportIterator;
import org.sonar.server.computation.source.ScmLineReader;
import org.sonar.server.computation.source.SymbolsLineReader;
import org.sonar.server.db.DbClient;
import org.sonar.server.source.db.FileSourceDb;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.google.common.collect.Lists.newArrayList;

public class PersistFileSourcesStep implements ComputationStep {

private final DbClient dbClient;
Expand Down Expand Up @@ -131,8 +137,7 @@ private void persistSource(FileSourcesContext fileSourcesContext, ComputeFileSou
.setDataHash(dataHash)
.setLineHashes(lineHashes)
.setCreatedAt(system2.now())
// TODO set current date here when indexing sources in E/S will be done in this class
.setUpdatedAt(0L);
.setUpdatedAt(system2.now());
dbClient.fileSourceDao().insert(fileSourcesContext.session, dto);
fileSourcesContext.session.commit();
} else {
Expand All @@ -147,8 +152,7 @@ private void persistSource(FileSourcesContext fileSourcesContext, ComputeFileSou
.setLineHashes(lineHashes);
// Optimization only change updated at when updating binary data to avoid unecessary indexation by E/S
if (binaryDataUpdated) {
// TODO set current date here when indexing sources in E/S will be done in this class
previousDto.setUpdatedAt(0L);
previousDto.setUpdatedAt(system2.now());
}
dbClient.fileSourceDao().update(previousDto);
fileSourcesContext.session.commit();
Expand All @@ -169,8 +173,8 @@ public FileSourcesContext(DbSession session, ComputationContext context, Map<Str
}

private static class LineReaders {
private final List<LineReader> lineReaders = newArrayList();
private final List<ReportIterator> reportIterators = newArrayList();
private final List<LineReader> lineReaders = new ArrayList<>();
private final List<ReportIterator> reportIterators = new ArrayList<>();

LineReaders(BatchReportReader reportReader, int componentRef) {
File coverageFile = reportReader.readComponentCoverage(componentRef);
Expand Down
19 changes: 16 additions & 3 deletions server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
View file
Expand Up @@ -49,7 +49,7 @@ public abstract class BaseIndexer implements ServerComponent, Startable {
private boolean enabled = false;

protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName,
String dateFieldName) {
String dateFieldName) {
this.indexName = indexName;
this.typeName = typeName;
this.dateFieldName = dateFieldName;
Expand All @@ -58,14 +58,14 @@ protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String index
threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
}

public void index() {
public void index(final IndexerTask task) {
if (enabled) {
final long requestedAt = System.currentTimeMillis();
Future submit = executor.submit(new Runnable() {
@Override
public void run() {
if (requestedAt > lastUpdatedAt) {
long l = doIndex(lastUpdatedAt);
long l = task.index(lastUpdatedAt);
// l can be 0 if no documents were indexed
lastUpdatedAt = Math.max(l, lastUpdatedAt);
}
Expand All @@ -79,6 +79,15 @@ public void run() {
}
}

public void index() {
index(new IndexerTask() {
@Override
public long index(long lastUpdatedAt) {
return doIndex(lastUpdatedAt);
}
});
}

protected abstract long doIndex(long lastUpdatedAt);

public BaseIndexer setEnabled(boolean b) {
Expand All @@ -96,4 +105,8 @@ public void stop() {
executor.shutdown();
}

public interface IndexerTask {
long index(long lastUpdatedAt);
}

}
60 changes: 44 additions & 16 deletions ...sonar-server/src/main/java/org/sonar/server/source/index/SourceFileResultSetIterator.java
View file
Expand Up @@ -29,6 +29,8 @@
import org.sonar.server.es.EsUtils;
import org.sonar.server.source.db.FileSourceDb;

import javax.annotation.Nullable;

import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;
import java.sql.Connection;
Expand Down Expand Up @@ -79,23 +81,49 @@ public List<UpdateRequest> getLineUpdateRequests() {
"updated_at",
"binary_data"
};
private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from file_sources";
private static final String SQL_AFTER_DATE = SQL_ALL + " where updated_at>?";

public static SourceFileResultSetIterator create(DbClient dbClient, Connection connection, long afterDate) {
private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM file_sources ";
private static final String AFTER_DATE_FILTER = "updated_at>? ";
private static final String PROJECT_FILTER = "project_uuid=? ";

public static SourceFileResultSetIterator create(DbClient dbClient, Connection connection, long afterDate, @Nullable String projectUuid) {
try {
String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL;
String sql = createSQL(afterDate, projectUuid);
// rows are big, so they are scrolled once at a time (one row in memory at a time)
PreparedStatement stmt = dbClient.newScrollingSingleRowSelectStatement(connection, sql);
int index = 1;
if (afterDate > 0L) {
stmt.setLong(1, afterDate);
stmt.setLong(index, afterDate);
index++;
}
if (projectUuid != null) {
stmt.setString(index, projectUuid);
}
return new SourceFileResultSetIterator(stmt);
} catch (SQLException e) {
throw new IllegalStateException("Fail to prepare SQL request to select all file sources", e);
}
}

private static String createSQL(long afterDate, @Nullable String projectUuid) {
String sql = SQL_ALL;
if (afterDate > 0L || projectUuid != null) {
sql += "WHERE ";
boolean isFirst = true;
if (afterDate > 0L) {
sql += AFTER_DATE_FILTER;
isFirst = false;
}
if (projectUuid != null) {
if (!isFirst) {
sql += "AND ";
}
sql += PROJECT_FILTER;
}
}
return sql;
}

private SourceFileResultSetIterator(PreparedStatement stmt) throws SQLException {
super(stmt);
}
Expand Down Expand Up @@ -130,62 +158,62 @@ public static Row toRow(String projectUuid, String fileUuid, Date updatedAt, Fil

// unit tests
if (line.hasUtLineHits()) {
writer.prop(SourceLineIndexDefinition.FIELD_UT_LINE_HITS, line.getUtLineHits());
writer.prop(SourceLineIndexDefinition.FIELD_UT_LINE_HITS, line.getUtLineHits());
} else {
writer.name(SourceLineIndexDefinition.FIELD_UT_LINE_HITS).valueObject(null);
}
if (line.hasUtConditions()) {
writer.prop(SourceLineIndexDefinition.FIELD_UT_CONDITIONS, line.getUtConditions());
writer.prop(SourceLineIndexDefinition.FIELD_UT_CONDITIONS, line.getUtConditions());
} else {
writer.name(SourceLineIndexDefinition.FIELD_UT_CONDITIONS).valueObject(null);
}
if (line.hasUtCoveredConditions()) {
writer.prop(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, line.getUtCoveredConditions());
writer.prop(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS, line.getUtCoveredConditions());
} else {
writer.name(SourceLineIndexDefinition.FIELD_UT_COVERED_CONDITIONS).valueObject(null);
}

// IT
if (line.hasItLineHits()) {
writer.prop(SourceLineIndexDefinition.FIELD_IT_LINE_HITS, line.getItLineHits());
writer.prop(SourceLineIndexDefinition.FIELD_IT_LINE_HITS, line.getItLineHits());
} else {
writer.name(SourceLineIndexDefinition.FIELD_IT_LINE_HITS).valueObject(null);
}
if (line.hasItConditions()) {
writer.prop(SourceLineIndexDefinition.FIELD_IT_CONDITIONS, line.getItConditions());
writer.prop(SourceLineIndexDefinition.FIELD_IT_CONDITIONS, line.getItConditions());
} else {
writer.name(SourceLineIndexDefinition.FIELD_IT_CONDITIONS).valueObject(null);
}
if (line.hasItCoveredConditions()) {
writer.prop(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, line.getItCoveredConditions());
writer.prop(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS, line.getItCoveredConditions());
} else {
writer.name(SourceLineIndexDefinition.FIELD_IT_COVERED_CONDITIONS).valueObject(null);
}

// Overall coverage
if (line.hasOverallLineHits()) {
writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, line.getOverallLineHits());
writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS, line.getOverallLineHits());
} else {
writer.name(SourceLineIndexDefinition.FIELD_OVERALL_LINE_HITS).valueObject(null);
}
if (line.hasOverallConditions()) {
writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, line.getOverallConditions());
writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS, line.getOverallConditions());
} else {
writer.name(SourceLineIndexDefinition.FIELD_OVERALL_CONDITIONS).valueObject(null);
}
if (line.hasOverallCoveredConditions()) {
writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, line.getOverallCoveredConditions());
writer.prop(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS, line.getOverallCoveredConditions());
} else {
writer.name(SourceLineIndexDefinition.FIELD_OVERALL_COVERED_CONDITIONS).valueObject(null);
}

if (line.hasHighlighting()) {
writer.prop(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, line.getHighlighting());
writer.prop(SourceLineIndexDefinition.FIELD_HIGHLIGHTING, line.getHighlighting());
} else {
writer.name(SourceLineIndexDefinition.FIELD_HIGHLIGHTING).valueObject(null);
}
if (line.hasSymbols()) {
writer.prop(SourceLineIndexDefinition.FIELD_SYMBOLS, line.getSymbols());
writer.prop(SourceLineIndexDefinition.FIELD_SYMBOLS, line.getSymbols());
} else {
writer.name(SourceLineIndexDefinition.FIELD_SYMBOLS).valueObject(null);
}
Expand Down
17 changes: 16 additions & 1 deletion server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
View file
Expand Up @@ -29,6 +29,8 @@
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.EsClient;

import javax.annotation.Nullable;

import java.sql.Connection;
import java.util.Iterator;

Expand All @@ -48,15 +50,28 @@ public SourceLineIndexer(DbClient dbClient, EsClient esClient) {
this.dbClient = dbClient;
}

public void index(final String projectUuid){
super.index(new IndexerTask() {
@Override
public long index(long lastUpdatedAt) {
return doIndex(lastUpdatedAt, projectUuid);
}
});
}

@Override
protected long doIndex(long lastUpdatedAt) {
return doIndex(lastUpdatedAt, null);
}

private long doIndex(long lastUpdatedAt, @Nullable String projectUuid) {
final BulkIndexer bulk = new BulkIndexer(esClient, SourceLineIndexDefinition.INDEX);
bulk.setLarge(lastUpdatedAt == 0L);

DbSession dbSession = dbClient.openSession(false);
Connection dbConnection = dbSession.getConnection();
try {
SourceFileResultSetIterator rowIt = SourceFileResultSetIterator.create(dbClient, dbConnection, lastUpdatedAt);
SourceFileResultSetIterator rowIt = SourceFileResultSetIterator.create(dbClient, dbConnection, lastUpdatedAt, projectUuid);
long maxUpdatedAt = doIndex(bulk, rowIt);
rowIt.close();
return maxUpdatedAt;
Expand Down

0 comments on commit 803a343

Please sign in to comment.