SONAR-6993 Search duplication candidates by hashes

SonarSource · Nov 12, 2015 · 1d0f9fc · 1d0f9fc
1 parent c203430
commit 1d0f9fc
Show file tree

Hide file tree

Showing 8 changed files with 111 additions and 111 deletions.
diff --git a/sonar-db/src/main/java/org/sonar/db/duplication/DuplicationDao.java b/sonar-db/src/main/java/org/sonar/db/duplication/DuplicationDao.java
@@ -19,46 +19,37 @@
  */
 package org.sonar.db.duplication;
 
+import com.google.common.base.Function;
 import java.util.Collection;
 import java.util.List;
-import org.apache.ibatis.session.SqlSession;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import org.sonar.db.Dao;
+import org.sonar.db.DatabaseUtils;
 import org.sonar.db.DbSession;
-import org.sonar.db.MyBatis;
 
 public class DuplicationDao implements Dao {
 
-  private final MyBatis mybatis;
+  /**
-
+   * @param projectSnapshotId snapshot id of the project from the previous analysis (islast=true)
-  public DuplicationDao(MyBatis mybatis) {
+   */
-    this.mybatis = mybatis;
+  public List<DuplicationUnitDto> selectCandidates(final DbSession session, @Nullable final Long projectSnapshotId, final String language, Collection<String> hashes) {
-  }
+    return DatabaseUtils.executeLargeInputs(hashes, new Function<List<String>, List<DuplicationUnitDto>>() {
-
+      @Override
-  public List<DuplicationUnitDto> selectCandidates(int resourceSnapshotId, Integer lastSnapshotId, String language) {
+      public List<DuplicationUnitDto> apply(@Nonnull List<String> partition) {
-    SqlSession session = mybatis.openSession(false);
+        return session.getMapper(DuplicationMapper.class).selectCandidates(projectSnapshotId, language, partition);
-    try {
+      }
-      DuplicationMapper mapper = session.getMapper(DuplicationMapper.class);
+    });
-      return mapper.selectCandidates(resourceSnapshotId, lastSnapshotId, language);
-    } finally {
-      MyBatis.closeQuietly(session);
-    }
   }
 
   /**
    * Insert rows in the table DUPLICATIONS_INDEX.
    * Note that generated ids are not returned.
    */
-  public void insert(Collection<DuplicationUnitDto> units) {
+  public void insert(DbSession session, Collection<DuplicationUnitDto> units) {
-    DbSession session = mybatis.openSession(true);
+    DuplicationMapper mapper = session.getMapper(DuplicationMapper.class);
-    try {
+    for (DuplicationUnitDto unit : units) {
-      DuplicationMapper mapper = session.getMapper(DuplicationMapper.class);
+      mapper.batchInsert(unit);
-      for (DuplicationUnitDto unit : units) {
-        mapper.batchInsert(unit);
-      }
-      session.commit();
-
-    } finally {
-      MyBatis.closeQuietly(session);
     }
   }
 

diff --git a/sonar-db/src/main/java/org/sonar/db/duplication/DuplicationMapper.java b/sonar-db/src/main/java/org/sonar/db/duplication/DuplicationMapper.java
@@ -19,15 +19,17 @@
  */
 package org.sonar.db.duplication;
 
+import java.util.Collection;
 import java.util.List;
+import javax.annotation.Nullable;
 import org.apache.ibatis.annotations.Param;
 
 public interface DuplicationMapper {
 
   List<DuplicationUnitDto> selectCandidates(
-    @Param("resource_snapshot_id") int resourceSnapshotId,
+    @Nullable @Param("projectSnapshotId") Long projectSnapshotId,
-    @Param("last_project_snapshot_id") Integer lastSnapshotId,
+    @Param("language") String language,
-    @Param("language") String language);
+    @Param("hashes") Collection<String> hashes);
 
   void batchInsert(DuplicationUnitDto unit);
 

diff --git a/sonar-db/src/main/java/org/sonar/db/duplication/DuplicationUnitDto.java b/sonar-db/src/main/java/org/sonar/db/duplication/DuplicationUnitDto.java
@@ -19,97 +19,85 @@
  */
 package org.sonar.db.duplication;
 
-/**
- * A simple DTO (Data Transfer Object) class that provides the mapping of data to a table.
- */
 public final class DuplicationUnitDto {
 
-  private Long id;
+  private long id;
-  private Integer snapshotId;
+  private long snapshotId;
-  private Integer projectSnapshotId;
+  private long projectSnapshotId;
 
   private String hash;
   private int indexInFile;
   private int startLine;
   private int endLine;
 
-  private String resourceKey;
+  // Return by join
-
+  private String componentKey;
-  public DuplicationUnitDto() {
-  }
 
-  public DuplicationUnitDto(Integer projectSnapshotId, Integer snapshotId, String hash, Integer indexInFile, Integer startLine, Integer endLine) {
+  public long getId() {
-    this.projectSnapshotId = projectSnapshotId;
-    this.snapshotId = snapshotId;
-    this.hash = hash;
-    this.indexInFile = indexInFile;
-    this.startLine = startLine;
-    this.endLine = endLine;
-  }
-
-  public Long getId() {
     return id;
   }
 
-  public DuplicationUnitDto setId(Long id) {
+  public DuplicationUnitDto setId(long id) {
     this.id = id;
     return this;
   }
 
-  public Integer getSnapshotId() {
+  public long getSnapshotId() {
     return snapshotId;
   }
 
-  public void setSnapshotId(Integer snapshotId) {
+  public DuplicationUnitDto setSnapshotId(long snapshotId) {
     this.snapshotId = snapshotId;
+    return this;
   }
 
-  public Integer getProjectSnapshotId() {
+  public long getProjectSnapshotId() {
     return projectSnapshotId;
   }
 
-  public void setProjectSnapshotId(Integer projectSnapshotId) {
+  public DuplicationUnitDto setProjectSnapshotId(long projectSnapshotId) {
     this.projectSnapshotId = projectSnapshotId;
+    return this;
   }
 
   public String getHash() {
     return hash;
   }
 
-  public void setHash(String hash) {
+  public DuplicationUnitDto setHash(String hash) {
     this.hash = hash;
+    return this;
   }
 
   public int getIndexInFile() {
     return indexInFile;
   }
 
-  public void setIndexInFile(int indexInFile) {
+  public DuplicationUnitDto setIndexInFile(int indexInFile) {
     this.indexInFile = indexInFile;
+    return this;
   }
 
   public int getStartLine() {
     return startLine;
   }
 
-  public void setStartLine(int startLine) {
+  public DuplicationUnitDto setStartLine(int startLine) {
     this.startLine = startLine;
+    return this;
   }
 
   public int getEndLine() {
     return endLine;
   }
 
-  public void setEndLine(int endLine) {
+  public DuplicationUnitDto setEndLine(int endLine) {
     this.endLine = endLine;
+    return this;
   }
 
-  public String getResourceKey() {
+  public String getComponentKey() {
-    return resourceKey;
+    return componentKey;
-  }
-
-  public void setResourceKey(String resourceKey) {
-    this.resourceKey = resourceKey;
   }
 
 }
diff --git a/sonar-db/src/main/resources/org/sonar/db/duplication/DuplicationMapper.xml b/sonar-db/src/main/resources/org/sonar/db/duplication/DuplicationMapper.xml
@@ -4,22 +4,30 @@
 <mapper namespace="org.sonar.db.duplication.DuplicationMapper">
 
   <select id="selectCandidates" parameterType="map" resultType="DuplicationUnit">
-    SELECT DISTINCT to_blocks.hash as hash, res.kee as resourceKey, to_blocks.index_in_file as indexInFile,
+    SELECT DISTINCT
-    to_blocks.start_line as startLine, to_blocks.end_line as endLine
+    duplication_block.id as id,
-    FROM duplications_index to_blocks, duplications_index from_blocks, snapshots snapshot, projects res
+    duplication_block.snapshot_id as snapshotId,
-    WHERE from_blocks.snapshot_id = #{resource_snapshot_id}
+    duplication_block.project_snapshot_id as projectSnapshotId,
-    AND to_blocks.hash = from_blocks.hash
+    duplication_block.hash as hash,
-    AND to_blocks.snapshot_id = snapshot.id
+    duplication_block.index_in_file as indexInFile,
-    AND snapshot.islast = ${_true}
+    duplication_block.start_line as startLine,
-    AND snapshot.project_id = res.id
+    duplication_block.end_line as endLine,
-    AND res.language = #{language}
+    file.kee as componentKey
-    <if test="last_project_snapshot_id != null">
+    FROM duplications_index duplication_block
-      AND to_blocks.project_snapshot_id != #{last_project_snapshot_id}
+    INNER JOIN snapshots snapshot ON duplication_block.snapshot_id=snapshot.id AND snapshot.islast=${_true}
-    </if>
+    INNER JOIN projects file ON file.id=snapshot.project_id AND file.language=#{language} AND file.enabled=${_true}
+    <where>
+      AND duplication_block.hash in
+      <foreach collection="hashes" open="(" close=")" item="hash" separator=",">#{hash}</foreach>
+      <if test="projectSnapshotId != null">
+        AND duplication_block.project_snapshot_id &lt;&gt; #{projectSnapshotId}
+      </if>
+    </where>
   </select>
 
   <insert id="batchInsert" parameterType="DuplicationUnit" useGeneratedKeys="false">
     INSERT INTO duplications_index (snapshot_id, project_snapshot_id, hash, index_in_file, start_line, end_line)
     VALUES (#{snapshotId}, #{projectSnapshotId}, #{hash}, #{indexInFile}, #{startLine}, #{endLine})
   </insert>
+
 </mapper>
diff --git a/sonar-db/src/test/java/org/sonar/db/duplication/DuplicationDaoTest.java b/sonar-db/src/test/java/org/sonar/db/duplication/DuplicationDaoTest.java
@@ -19,52 +19,63 @@
  */
 package org.sonar.db.duplication;
 
-import java.util.Arrays;
 import java.util.List;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.sonar.api.utils.System2;
+import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
 import org.sonar.test.DbTests;
 
-import static org.hamcrest.Matchers.is;
+import static java.util.Collections.singletonList;
-import static org.junit.Assert.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
 
 @Category(DbTests.class)
 public class DuplicationDaoTest {
 
   @Rule
   public DbTester db = DbTester.create(System2.INSTANCE);
 
+  DbSession dbSession = db.getSession();
+
   DuplicationDao dao = db.getDbClient().duplicationDao();
 
   @Test
-  public void shouldGetByHash() {
+  public void select_candidates() {
-    db.prepareDbUnit(getClass(), "shouldGetByHash.xml");
+    db.prepareDbUnit(getClass(), "select_candidates.xml");
+    dbSession.commit();
 
-    List<DuplicationUnitDto> blocks = dao.selectCandidates(10, 7, "java");
+    List<DuplicationUnitDto> blocks = dao.selectCandidates(dbSession, 7L, "java", singletonList("aa"));
-    assertThat(blocks.size(), is(1));
+    assertThat(blocks).hasSize(1);
 
     DuplicationUnitDto block = blocks.get(0);
-    assertThat("block resourceId", block.getResourceKey(), is("bar-last"));
+    assertThat(block.getComponentKey()).isEqualTo("bar-last");
-    assertThat("block hash", block.getHash(), is("aa"));
+    assertThat(block.getHash()).isEqualTo("aa");
-    assertThat("block index in file", block.getIndexInFile(), is(0));
+    assertThat(block.getIndexInFile()).isEqualTo(0);
-    assertThat("block start line", block.getStartLine(), is(1));
+    assertThat(block.getStartLine()).isEqualTo(1);
-    assertThat("block end line", block.getEndLine(), is(2));
+    assertThat(block.getEndLine()).isEqualTo(2);
 
     // check null for lastSnapshotId
-    blocks = dao.selectCandidates(10, null, "java");
+    blocks = dao.selectCandidates(dbSession, null, "java", singletonList("aa"));
-    assertThat(blocks.size(), is(2));
+    assertThat(blocks).hasSize(2);
   }
 
   @Test
-  public void shouldInsert() {
+  public void insert() {
-    db.prepareDbUnit(getClass(), "shouldInsert.xml");
+    db.prepareDbUnit(getClass(), "insert.xml");
+    dbSession.commit();
 
-    dao.insert(Arrays.asList(new DuplicationUnitDto(1, 2, "bb", 0, 1, 2)));
+    dao.insert(dbSession, singletonList(new DuplicationUnitDto()
+      .setProjectSnapshotId(1)
+      .setSnapshotId(2)
+      .setHash("bb")
+      .setIndexInFile(0)
+      .setStartLine(1)
+      .setEndLine(2)));
+    dbSession.commit();
 
-    db.assertDbUnit(getClass(), "shouldInsert-result.xml", "duplications_index");
+    db.assertDbUnit(getClass(), "insert-result.xml", "duplications_index");
   }
 
 }
diff --git a/...uplicationDaoTest/shouldInsert-result.xml → ...tion/DuplicationDaoTest/insert-result.xml b/...uplicationDaoTest/shouldInsert-result.xml → ...tion/DuplicationDaoTest/insert-result.xml
diff --git a/...ation/DuplicationDaoTest/shouldInsert.xml → ...duplication/DuplicationDaoTest/insert.xml b/...ation/DuplicationDaoTest/shouldInsert.xml → ...duplication/DuplicationDaoTest/insert.xml
diff --git a/...on/DuplicationDaoTest/shouldGetByHash.xml → .../DuplicationDaoTest/select_candidates.xml b/...on/DuplicationDaoTest/shouldGetByHash.xml → .../DuplicationDaoTest/select_candidates.xml
@@ -1,27 +1,27 @@
 <dataset>
 
-  <snapshots id="1" project_id="1" status="P" islast="0" purge_status="[null]"/>
+  <snapshots id="1" project_id="1" status="P" islast="[false]" purge_status="[null]"/>
-  <snapshots id="2" project_id="1" status="P" islast="0" purge_status="[null]"/>
+  <snapshots id="2" project_id="1" status="P" islast="[false]" purge_status="[null]"/>
-  <projects id="1" uuid="1" kee="bar-old" enabled="1" scope="FIL" qualifier="CLA" language="java"/>
+  <projects id="1" uuid="1" kee="bar-old" enabled="[true]" scope="FIL" qualifier="CLA" language="java"/>
 
-  <snapshots id="3" project_id="2" status="P" islast="1" purge_status="[null]"/>
+  <snapshots id="3" project_id="2" status="P" islast="[true]" purge_status="[null]"/>
-  <snapshots id="4" project_id="2" status="P" islast="1" purge_status="[null]"/>
+  <snapshots id="4" project_id="2" status="P" islast="[true]" purge_status="[null]"/>
-  <projects id="2" uuid="2" kee="bar-last" enabled="1" scope="FIL" qualifier="CLA" language="java"/>
+  <projects id="2" uuid="2" kee="bar-last" enabled="[true]" scope="FIL" qualifier="CLA" language="java"/>
 
-  <snapshots id="5" project_id="3" status="P" islast="0" purge_status="[null]"/>
+  <snapshots id="5" project_id="3" status="P" islast="[false]" purge_status="[null]"/>
-  <snapshots id="6" project_id="3" status="P" islast="0" purge_status="[null]"/>
+  <snapshots id="6" project_id="3" status="P" islast="[false]" purge_status="[null]"/>
-  <projects id="3" uuid="3" kee="foo-old" enabled="1" scope="FIL" qualifier="CLA" language="java"/>
+  <projects id="3" uuid="3" kee="foo-old" enabled="[true]" scope="FIL" qualifier="CLA" language="java"/>
 
-  <snapshots id="7" project_id="4" status="P" islast="1" purge_status="[null]"/>
+  <snapshots id="7" project_id="4" status="P" islast="[true]" purge_status="[null]"/>
-  <snapshots id="8" project_id="4" status="P" islast="1" purge_status="[null]"/>
+  <snapshots id="8" project_id="4" status="P" islast="[true]" purge_status="[null]"/>
-  <projects id="4" uuid="4" kee="foo-last" enabled="1" scope="FIL" qualifier="CLA" language="java"/>
+  <projects id="4" uuid="4" kee="foo-last" enabled="[true]" scope="FIL" qualifier="CLA" language="java"/>
 
-  <snapshots id="9" project_id="5" status="U" islast="0" purge_status="[null]"/>
+  <snapshots id="9" project_id="5" status="U" islast="[false]" purge_status="[null]"/>
-  <snapshots id="10" project_id="5" status="U" islast="0" purge_status="[null]"/>
+  <snapshots id="10" project_id="5" status="U" islast="[false]" purge_status="[null]"/>
-  <projects id="5" uuid="5" kee="foo" enabled="1" scope="FIL" qualifier="CLA" language="java"/>
+  <projects id="5" uuid="5" kee="foo" enabled="[true]" scope="FIL" qualifier="CLA" language="java"/>
 
   <snapshots id="11" project_id="6" purge_status="[null]" status="P" islast="1"/>
-  <projects id="6" uuid="6" kee="baz" enabled="1" scope="FIL" qualifier="CLA" language="grvy"/>
+  <projects id="6" uuid="6" kee="baz" enabled="[true]" scope="FIL" qualifier="CLA" language="grvy"/>
 
   <!-- Old snapshot of another project -->
   <!-- bar-old -->