Skip to content

Commit

Permalink
SONAR-10670 Count ncloc of whole instance with biggest long-living br…
Browse files Browse the repository at this point in the history 
…anch
  • Loading branch information
@teryk
teryk authored and SonarTech committed May 24, 2018
1 parent 4c9d0c7 commit 812cf12
Show file tree
Hide file tree
Showing 23 changed files with 180 additions and 109 deletions.
14 changes: 14 additions & 0 deletions server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureDao.java
View file
Expand Up @@ -29,9 +29,12 @@
import org.sonar.core.util.Uuids;
import org.sonar.db.Dao;
import org.sonar.db.DbSession;
import org.sonar.db.component.BranchType;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.component.KeyType;

import static java.util.Collections.singletonList;
import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
import static org.sonar.db.DatabaseUtils.executeLargeInputs;

public class LiveMeasureDao implements Dao {
Expand Down Expand Up @@ -78,6 +81,17 @@ public void selectTreeByQuery(DbSession dbSession, ComponentDto baseComponent, M
mapper(dbSession).selectTreeByQuery(query, baseComponent.uuid(), query.getUuidPath(baseComponent), resultHandler);
}

/**
* Example:
* If Main Branch = 0 LOCs (provisioned but never analyzed) and the "largest long-lived branch" is 120 LOCs, I'm expecting to consider the value 120.
* If Main Branch = 100 LOCs and the "largest long-lived branch" is 120 LOCs, I'm expecting to consider the value 120.
* If Main Branch = 100 LOCs and the "largest long-lived branch" is 80 LOCs, I'm expecting to consider the value 100.
*/
public long sumNclocOfBiggestLongLivingBranch(DbSession dbSession) {
Long ncloc = mapper(dbSession).sumNclocOfBiggestLongLivingBranch(NCLOC_KEY, KeyType.BRANCH, BranchType.LONG);
return ncloc == null ? 0L : ncloc;
}

public void insert(DbSession dbSession, LiveMeasureDto dto) {
mapper(dbSession).insert(dto, Uuids.create(), null, system2.now());
}
Expand Down
7 changes: 7 additions & 0 deletions server/sonar-db-dao/src/main/java/org/sonar/db/measure/LiveMeasureMapper.java
View file
Expand Up @@ -24,6 +24,8 @@
import javax.annotation.Nullable;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.session.ResultHandler;
import org.sonar.db.component.BranchType;
import org.sonar.db.component.KeyType;

public interface LiveMeasureMapper {

Expand All @@ -41,6 +43,11 @@ void selectTreeByQuery(
@Param("baseUuidPath") String baseUuidPath,
ResultHandler<LiveMeasureDto> resultHandler);

Long sumNclocOfBiggestLongLivingBranch(
@Param("ncloc") String nclocKey,
@Param("branch") KeyType branchOrPullRequest,
@Param("branchType") BranchType branchType);

void insert(
@Param("dto") LiveMeasureDto dto,
@Param("uuid") String uuid,
Expand Down
19 changes: 19 additions & 0 deletions server/sonar-db-dao/src/main/resources/org/sonar/db/measure/LiveMeasureMapper.xml
View file
Expand Up @@ -34,6 +34,25 @@
</foreach>
</select>

<select id="sumNclocOfBiggestLongLivingBranch" parameterType="map" resultType="long">
select sum(sumncloc.maxncloc) from (
select b.project_uuid as projectUuid, max(lm.value) as maxncloc
from live_measures lm
inner join metrics m on m.id = lm.metric_id
inner join projects p on p.uuid = lm.component_uuid
inner join project_branches b on b.uuid = p.uuid
where
m.name = #{ncloc, jdbcType=VARCHAR}
and p.enabled = ${_true}
and p.scope = 'PRJ'
and p.qualifier = 'TRK'
and p.copy_component_uuid is null
and b.branch_type = #{branchType, jdbcType=VARCHAR}
and b.key_type = #{branch, jdbcType=VARCHAR}
group by b.project_uuid
) sumncloc
</select>

<insert id="insert" parameterType="map" useGeneratedKeys="false">
insert into live_measures (
uuid,
Expand Down
44 changes: 40 additions & 4 deletions server/sonar-db-dao/src/test/java/org/sonar/db/measure/LiveMeasureDaoTest.java
View file
Expand Up @@ -27,15 +27,18 @@
import org.junit.Test;
import org.sonar.api.utils.System2;
import org.sonar.db.DbTester;
import org.sonar.db.component.BranchType;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.metric.MetricDto;
import org.sonar.db.organization.OrganizationDto;

import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.groups.Tuple.tuple;
import static org.sonar.api.measures.Metric.ValueType.INT;
import static org.sonar.db.component.ComponentTesting.newFileDto;
import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
import static org.sonar.db.measure.MeasureTesting.newLiveMeasure;
Expand All @@ -54,7 +57,7 @@ public void setUp() throws Exception {
}

@Test
public void test_selectByComponentUuidsAndMetricIds() {
public void selectByComponentUuidsAndMetricIds() {
LiveMeasureDto measure1 = newLiveMeasure().setMetricId(metric.getId());
LiveMeasureDto measure2 = newLiveMeasure().setMetricId(metric.getId());
underTest.insert(db.getSession(), measure1);
Expand Down Expand Up @@ -94,7 +97,7 @@ public void selectByComponentUuidsAndMetricIds_returns_empty_list_if_component_d
}

@Test
public void test_selectByComponentUuidsAndMetricKeys() {
public void selectByComponentUuidsAndMetricKeys() {
LiveMeasureDto measure1 = newLiveMeasure().setMetricId(metric.getId());
LiveMeasureDto measure2 = newLiveMeasure().setMetricId(metric.getId());
underTest.insert(db.getSession(), measure1);
Expand Down Expand Up @@ -133,7 +136,7 @@ public void selectByComponentUuidsAndMetricKeys_returns_empty_list_if_component_
}

@Test
public void test_selectMeasure() {
public void selectMeasure() {
MetricDto metric = db.measures().insertMetric();
LiveMeasureDto stored = newLiveMeasure().setMetricId(metric.getId());
underTest.insert(db.getSession(), stored);
Expand Down Expand Up @@ -198,6 +201,39 @@ public void selectMeasure_map_fields() {
.contains(project.uuid(), file.uuid(), metric.getId(), 3.14, 0.1, "text_value", "text_value");
}

@Test
public void countNcloc() {
OrganizationDto organization = db.organizations().insert();
MetricDto ncloc = db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
MetricDto lines = db.measures().insertMetric(m -> m.setKey("lines").setValueType(INT.toString()));

ComponentDto simpleProject = db.components().insertMainBranch(organization);
db.measures().insertLiveMeasure(simpleProject, ncloc, m -> m.setValue(10d));

ComponentDto projectWithBiggerLongLivingBranch = db.components().insertMainBranch(organization);
ComponentDto bigLongLivingBranch = db.components().insertProjectBranch(projectWithBiggerLongLivingBranch, b -> b.setBranchType(BranchType.LONG));
db.measures().insertLiveMeasure(projectWithBiggerLongLivingBranch, ncloc, m -> m.setValue(100d));
db.measures().insertLiveMeasure(bigLongLivingBranch, ncloc, m -> m.setValue(200d));

ComponentDto projectWithLinesButNoLoc = db.components().insertMainBranch(organization);
db.measures().insertLiveMeasure(projectWithLinesButNoLoc, lines, m -> m.setValue(365d));
db.measures().insertLiveMeasure(projectWithLinesButNoLoc, ncloc, m -> m.setValue(0d));

long result = underTest.sumNclocOfBiggestLongLivingBranch(db.getSession());

assertThat(result).isEqualTo(10L + 200L);
}

@Test
public void countNcloc_empty() {
db.measures().insertMetric(m -> m.setKey("ncloc").setValueType(INT.toString()));
db.measures().insertMetric(m -> m.setKey("lines").setValueType(INT.toString()));

long result = underTest.sumNclocOfBiggestLongLivingBranch(db.getSession());

assertThat(result).isEqualTo(0L);
}

@Test
public void insert_data() {
byte[] data = "text_value".getBytes(StandardCharsets.UTF_8);
Expand All @@ -215,7 +251,7 @@ public void insert_data() {
}

@Test
public void test_insertOrUpdate() {
public void insertOrUpdate() {
// insert
LiveMeasureDto dto = newLiveMeasure();
underTest.insertOrUpdate(db.getSession(), dto, "foo");
Expand Down
15 changes: 11 additions & 4 deletions ...src/test/java/util/XooProjectBuilder.java → .../sonarqube/qa/util/XooProjectBuilder.java
View file
Expand Up @@ -17,7 +17,7 @@
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package util;
package org.sonarqube.qa.util;

import java.io.File;
import java.io.IOException;
Expand All @@ -34,6 +34,7 @@ public class XooProjectBuilder {
private final String key;
private final List<String> moduleKeys = new ArrayList<>();
private int filesPerModule = 1;
private Properties projectProperties = new Properties();

public XooProjectBuilder(String projectKey) {
this.key = projectKey;
Expand All @@ -50,13 +51,19 @@ public XooProjectBuilder setFilesPerModule(int i) {
return this;
}

public XooProjectBuilder addProjectProperties(String... keyValueProperties) {
for (int i = 0; i < keyValueProperties.length; i += 2) {
this.projectProperties.setProperty(keyValueProperties[i], keyValueProperties[i + 1]);
}
return this;
}

public File build(File dir) {
for (String moduleKey : moduleKeys) {
generateModule(moduleKey, new File(dir, moduleKey), new Properties());
}
Properties additionalProps = new Properties();
additionalProps.setProperty("sonar.modules", StringUtils.join(moduleKeys, ","));
generateModule(key, dir, additionalProps);
projectProperties.setProperty("sonar.modules", StringUtils.join(moduleKeys, ","));
generateModule(key, dir, projectProperties);
return dir;
}

Expand Down
27 changes: 16 additions & 11 deletions server/sonar-server/src/main/java/org/sonar/server/edition/ws/FormDataAction.java
View file
Expand Up @@ -23,20 +23,22 @@
import org.sonar.api.server.ws.Request;
import org.sonar.api.server.ws.Response;
import org.sonar.api.server.ws.WebService;
import org.sonar.server.measure.index.ProjectMeasuresIndex;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.server.user.UserSession;
import org.sonar.server.ws.WsUtils;
import org.sonarqube.ws.Editions.FormDataResponse;

import static org.sonar.server.ws.WsUtils.writeProtobuf;

public class FormDataAction implements EditionsWsAction {
private final UserSession userSession;
private final Server server;
private final ProjectMeasuresIndex measuresIndex;
private final DbClient dbClient;

public FormDataAction(UserSession userSession, Server server, ProjectMeasuresIndex measuresIndex) {
public FormDataAction(UserSession userSession, Server server, DbClient dbClient) {
this.userSession = userSession;
this.server = server;
this.measuresIndex = measuresIndex;
this.dbClient = dbClient;
}

@Override
Expand All @@ -56,13 +58,16 @@ public void handle(Request request, Response response) throws Exception {
.checkLoggedIn()
.checkIsSystemAdministrator();

String serverId = server.getId();
long nloc = measuresIndex.searchTelemetryStatistics().getNcloc();

FormDataResponse responsePayload = FormDataResponse.newBuilder()
.setNcloc(nloc)
.setServerId(serverId)
.setNcloc(computeNcloc())
.setServerId(server.getId())
.build();
WsUtils.writeProtobuf(responsePayload, request, response);
writeProtobuf(responsePayload, request, response);
}

private long computeNcloc() {
try (DbSession dbSession = dbClient.openSession(false)) {
return dbClient.liveMeasureDao().sumNclocOfBiggestLongLivingBranch(dbSession);
}
}
}
12 changes: 5 additions & 7 deletions server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
View file
Expand Up @@ -75,7 +75,6 @@
import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY;
import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY;
import static org.sonar.api.measures.CoreMetrics.DUPLICATED_LINES_DENSITY_KEY;
import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;
import static org.sonar.api.measures.CoreMetrics.NEW_COVERAGE_KEY;
import static org.sonar.api.measures.CoreMetrics.NEW_DUPLICATED_LINES_DENSITY_KEY;
Expand Down Expand Up @@ -105,8 +104,8 @@
import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_TAGS;
import static org.sonarqube.ws.client.project.ProjectsWsParameters.MAX_PAGE_SIZE;

@ComputeEngineSide
@ServerSide
@ComputeEngineSide
public class ProjectMeasuresIndex {

public static final List<String> SUPPORTED_FACETS = ImmutableList.of(
Expand Down Expand Up @@ -203,16 +202,15 @@ public ProjectMeasuresStatistics searchTelemetryStatistics() {
.order(Terms.Order.count(false))
.subAggregation(sum(FIELD_DISTRIB_NCLOC).field(FIELD_DISTRIB_NCLOC))));

Stream.of(LINES_KEY, NCLOC_KEY)
.forEach(metric -> request.addAggregation(AggregationBuilders.nested(metric, FIELD_MEASURES)
.subAggregation(AggregationBuilders.filter(metric + "_filter", termQuery(FIELD_MEASURES_KEY, metric))
.subAggregation(sum(metric + "_filter_sum").field(FIELD_MEASURES_VALUE)))));
request.addAggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES)
.subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_KEY, NCLOC_KEY))
.subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_VALUE))));

ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder();

SearchResponse response = request.get();
statistics.setProjectCount(response.getHits().getTotalHits());
Stream.of(LINES_KEY, NCLOC_KEY)
Stream.of(NCLOC_KEY)
.map(metric -> (Nested) response.getAggregations().get(metric))
.map(nested -> (Filter) nested.getAggregations().get(nested.getName() + "_filter"))
.map(filter -> (Sum) filter.getAggregations().get(filter.getName() + "_sum"))
Expand Down
26 changes: 8 additions & 18 deletions .../sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresStatistics.java
View file
Expand Up @@ -22,19 +22,16 @@
import java.util.Map;

import static java.util.Objects.requireNonNull;
import static org.sonar.api.measures.CoreMetrics.LINES_KEY;
import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY;

public class ProjectMeasuresStatistics {
private final long projectCount;
private final long lines;
private final long ncloc;
private final Map<String, Long> projectCountByLanguage;
private final Map<String, Long> nclocByLanguage;

private ProjectMeasuresStatistics(Builder builder) {
projectCount = builder.projectCount;
lines = builder.lines;
ncloc = builder.ncloc;
projectCountByLanguage = builder.projectCountByLanguage;
nclocByLanguage = builder.nclocByLanguage;
Expand All @@ -44,10 +41,10 @@ public long getProjectCount() {
return projectCount;
}

public long getLines() {
return lines;
}

/**
* @deprecated since 7.2 Global Ncloc computation should rely on org.sonar.db.measure.LiveMeasureDao#countNcloc(org.sonar.db.DbSession)
*/
@Deprecated
public long getNcloc() {
return ncloc;
}
Expand All @@ -66,7 +63,6 @@ public static Builder builder() {

public static class Builder {
private Long projectCount;
private Long lines;
private Long ncloc;
private Map<String, Long> projectCountByLanguage;
private Map<String, Long> nclocByLanguage;
Expand All @@ -81,15 +77,10 @@ public Builder setProjectCount(long projectCount) {
}

public Builder setSum(String metric, long value) {
switch (metric) {
case LINES_KEY:
this.lines = value;
break;
case NCLOC_KEY:
this.ncloc = value;
break;
default:
throw new IllegalStateException("Metric not supported: " + metric);
if (NCLOC_KEY.equals(metric)) {
this.ncloc = value;
} else {
throw new IllegalStateException("Metric not supported: " + metric);
}
return this;
}
Expand All @@ -105,7 +96,6 @@ public Builder setNclocByLanguage(Map<String, Long> nclocByLanguage) {

public ProjectMeasuresStatistics build() {
requireNonNull(projectCount);
requireNonNull(lines);
requireNonNull(ncloc);
requireNonNull(projectCountByLanguage);
requireNonNull(nclocByLanguage);
Expand Down

0 comments on commit 812cf12

Please sign in to comment.