SONAR-10430 do not compute distance for files with too far size

server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentDao.java

@@ -334,6 +334,18 @@ public List<ComponentDto> selectProjectsByNameQuery(DbSession dbSession, @Nullab
     return mapper(dbSession).selectProjectsByNameQuery(nameQueryForSql, includeModules);
   }
 
+  public List<KeyWithUuidDto> selectComponentKeysHavingIssuesToMerge(DbSession dbSession, String mergeBranchUuid) {
+    return mapper(dbSession).selectComponentKeysHavingIssuesToMerge(mergeBranchUuid);
+  }
+
+  /**
+   * Scroll all <strong>enabled</strong> files of the specified project (same project_uuid) in no specific order with
+   * 'SOURCE' source and a non null path.
+   */
+  public void scrollAllFilesForFileMove(DbSession session, String projectUuid, ResultHandler<FileMoveRowDto> handler) {
+    mapper(session).scrollAllFilesForFileMove(projectUuid, handler);
+  }
+
   public void insert(DbSession session, ComponentDto item) {
     mapper(session).insert(item);
   }
@@ -376,10 +388,6 @@ public void delete(DbSession session, long componentId) {
     mapper(session).delete(componentId);
   }
 
-  public List<KeyWithUuidDto> selectComponentKeysHavingIssuesToMerge(DbSession dbSession, String mergeBranchUuid) {
-    return mapper(dbSession).selectComponentKeysHavingIssuesToMerge(mergeBranchUuid);
-  }
-
   private static void checkThatNotTooManyComponents(ComponentQuery query) {
     checkThatNotTooManyConditions(query.getComponentIds(), "Too many component ids in query");
     checkThatNotTooManyConditions(query.getComponentKeys(), "Too many component keys in query");

server/sonar-db-dao/src/main/java/org/sonar/db/component/ComponentMapper.java

@@ -138,6 +138,8 @@ List<ComponentDto> selectComponentsFromProjectKeyAndScope(@Param("projectKey") S
 
   void scrollForIndexing(@Param("projectUuid") @Nullable String projectUuid, ResultHandler<ComponentDto> handler);
 
+  void scrollAllFilesForFileMove(@Param("projectUuid") String projectUuid, ResultHandler<FileMoveRowDto> handler);
+
   void insert(ComponentDto componentDto);
 
   void update(ComponentUpdateDto component);

server/sonar-db-dao/src/main/java/org/sonar/db/component/FileMoveRowDto.java

@@ -0,0 +1,59 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.component;
+
+public class FileMoveRowDto {
+  private long id;
+  private String kee;
+  private String uuid;
+  private String path;
+  private int lineCount;
+
+  public long getId() {
+    return id;
+  }
+
+  public String getKey() {
+    return kee;
+  }
+
+  public String getUuid() {
+    return uuid;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  public int getLineCount() {
+    return lineCount;
+  }
+
+  @Override
+  public String toString() {
+    return "FileMoveRowDto{" +
+      "id=" + id +
+      ", kee='" + kee + '\'' +
+      ", uuid='" + uuid + '\'' +
+      ", path='" + path + '\'' +
+      ", lineCount=" + lineCount +
+      '}';
+  }
+}

server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceDao.java

@@ -25,16 +25,20 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 import javax.annotation.CheckForNull;
 import org.apache.commons.dbutils.DbUtils;
 import org.apache.commons.io.IOUtils;
+import org.apache.ibatis.session.ResultHandler;
 import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
 import org.sonar.db.source.FileSourceDto.Type;
 
+import static org.sonar.db.DatabaseUtils.toUniqueAndSortedPartitions;
+
 public class FileSourceDao implements Dao {
 
   private static final Splitter END_OF_LINE_SPLITTER = Splitter.on('\n');
@@ -80,6 +84,16 @@ public List<String> selectLineHashes(DbSession dbSession, String fileUuid) {
     }
   }
 
+  /**
+   * Scroll line hashes of all <strong>enabled</strong> components (should be files, but not enforced) with specified
+   * keys in no specific order with 'SOURCE' source and a non null path.
+   */
+  public void scrollLineHashes(DbSession dbSession, Collection<String> fileKeys, ResultHandler<LineHashesWithKeyDto> rowHandler) {
+    for (List<String> partition : toUniqueAndSortedPartitions(fileKeys)) {
+      mapper(dbSession).scrollLineHashes(partition, rowHandler);
+    }
+  }
+
   public void readLineHashesStream(DbSession dbSession, String fileUuid, Consumer<Reader> consumer) {
     Connection connection = dbSession.getConnection();
     PreparedStatement pstmt = null;

server/sonar-db-dao/src/main/java/org/sonar/db/source/FileSourceMapper.java

@@ -19,9 +19,11 @@
  */
 package org.sonar.db.source;
 
+import java.util.Collection;
 import java.util.List;
 import javax.annotation.CheckForNull;
 import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.session.ResultHandler;
 
 public interface FileSourceMapper {
 
@@ -30,6 +32,8 @@ public interface FileSourceMapper {
   @CheckForNull
   FileSourceDto select(@Param("fileUuid") String fileUuid, @Param("dataType") String dataType);
 
+  void scrollLineHashes(@Param("fileKeys") Collection<String> fileKeys, ResultHandler<LineHashesWithKeyDto> rowHandler);
+
   @CheckForNull
   Integer selectLineHashesVersion(@Param("fileUuid") String fileUuid, @Param("dataType") String dataType);
 

server/sonar-db-dao/src/main/java/org/sonar/db/source/LineHashesWithKeyDto.java

@@ -0,0 +1,57 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.db.source;
+
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import static org.sonar.db.source.FileSourceDto.LINES_HASHES_SPLITTER;
+
+public class LineHashesWithKeyDto {
+  private String kee;
+  private String path;
+  private String lineHashes;
+
+  public String getKey() {
+    return kee;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  /** Used by MyBatis */
+  public String getRawLineHashes() {
+    return lineHashes;
+  }
+
+  /** Used by MyBatis */
+  public void setRawLineHashes(@Nullable String lineHashes) {
+    this.lineHashes = lineHashes;
+  }
+
+  public List<String> getLineHashes() {
+    if (lineHashes == null) {
+      return Collections.emptyList();
+    }
+    return LINES_HASHES_SPLITTER.splitToList(lineHashes);
+  }
+}

server/sonar-db-dao/src/main/resources/org/sonar/db/component/ComponentMapper.xml

@@ -496,6 +496,25 @@
       </if>
   </select>
 
+  <select id="scrollAllFilesForFileMove" parameterType="map" resultType="org.sonar.db.component.FileMoveRowDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    select
+      p.id,
+      p.uuid as uuid,
+      p.kee as kee,
+      p.path as path,
+      fs.line_count as lineCount
+    from projects p
+    inner join file_sources fs on
+      fs.file_uuid = p.uuid
+      and fs.data_type = 'SOURCE'
+    where
+      p.project_uuid = #{projectUuid,jdbcType=VARCHAR}
+      and p.enabled = ${_true}
+      and p.scope = 'FIL'
+      and p.qualifier in ('FIL', 'UTS')
+      and p.path is not null
+  </select>
+
   <select id="selectProjectsByNameQuery" resultType="Component">
     select
     <include refid="componentColumns"/>

server/sonar-db-dao/src/main/resources/org/sonar/db/source/FileSourceMapper.xml

@@ -28,9 +28,35 @@
   </select>
 
   <select id="selectHashesForProject" parameterType="map" resultType="org.sonar.db.source.FileSourceDto">
-    SELECT id, file_uuid as fileUuid, data_hash as dataHash, src_hash as srcHash, revision, updated_at as updatedAt
-    FROM file_sources
-    WHERE project_uuid = #{projectUuid} and data_type=#{dataType}
+    select
+      id,
+      file_uuid as fileUuid,
+      data_hash as dataHash,
+      src_hash as srcHash,
+      revision,
+      updated_at as updatedAt
+    from
+      file_sources
+    where
+      project_uuid = #{projectUuid}
+      and data_type=#{dataType}
+  </select>
+
+  <select id="scrollLineHashes" parameterType="map" resultType="org.sonar.db.source.LineHashesWithKeyDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    select
+      p.kee as kee,
+      p.path as path,
+      fs.line_hashes as rawLineHashes
+    from projects p
+    inner join file_sources fs on
+      fs.file_uuid = p.uuid
+      and fs.data_type = 'SOURCE'
+    where
+      p.kee in
+      <foreach collection="fileKeys" item="fileKey" open="(" close=")" separator=",">
+        #{fileKey,jdbcType=VARCHAR}
+      </foreach>
+      and p.path is not null
   </select>
 
   <select id="selectLineHashesVersion" parameterType="map" resultType="Integer">