SONAR-6229 Indexing of activity logs consumes too much memory during …

…server startup

sonar-core/src/main/java/org/sonar/core/activity/Activity.java → server/sonar-server/src/main/java/org/sonar/server/activity/Activity.java

@@ -17,30 +17,54 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.core.activity;
+package org.sonar.server.activity;
 
-import java.util.Date;
+import javax.annotation.Nullable;
+
+import java.util.LinkedHashMap;
 import java.util.Map;
 
-/**
- * @since 4.4
- */
-public interface Activity {
+public class Activity {
 
-  Type type();
+  public static enum Type {
+    QPROFILE, SERVER, ANALYSIS_REPORT
+  }
+
+  private Type type;
+  private String action;
+  private String message;
+  private final Map<String,Object> data = new LinkedHashMap<>();
+
+  public Type getType() {
+    return type;
+  }
 
-  String action();
+  public void setType(Type type) {
+    this.type = type;
+  }
 
-  Date time();
+  public String getAction() {
+    return action;
+  }
 
-  String login();
+  public void setAction(String action) {
+    this.action = action;
+  }
 
-  Map<String, String> details();
+  public String getMessage() {
+    return message;
+  }
 
-  String message();
+  public void setMessage(@Nullable String message) {
+    this.message = message;
+  }
 
-  public static enum Type {
-    NONE, QPROFILE, SERVER, ANALYSIS_REPORT
+  public Map<String, Object> getData() {
+    return data;
   }
 
+  public Activity setData(String key, Object val) {
+    this.data.put(key, val);
+    return this;
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/activity/ActivityService.java

@@ -19,60 +19,33 @@
  */
 package org.sonar.server.activity;
 
-import org.sonar.core.activity.Activity;
-import org.sonar.core.activity.ActivityLog;
+import org.sonar.api.ServerComponent;
+import org.sonar.api.utils.KeyValueFormat;
+import org.sonar.api.utils.internal.Uuids;
 import org.sonar.core.activity.db.ActivityDto;
-import org.sonar.core.persistence.DbSession;
-import org.sonar.server.activity.index.ActivityIndex;
-import org.sonar.server.activity.index.ActivityQuery;
+import org.sonar.server.activity.index.ActivityIndexer;
 import org.sonar.server.db.DbClient;
-import org.sonar.server.search.IndexClient;
-import org.sonar.server.search.QueryContext;
-import org.sonar.server.search.Result;
 import org.sonar.server.user.UserSession;
 
-import javax.annotation.Nullable;
-
-/**
- * Log service is used to log Activity classes which represents an event to DB and Index.
- *
- * @see org.sonar.core.activity.ActivityLog
- */
-public class ActivityService {
+public class ActivityService implements ServerComponent {
 
   private final DbClient dbClient;
-  private final IndexClient indexClient;
+  private final ActivityIndexer indexer;
 
-  public ActivityService(DbClient dbClient, IndexClient indexClient) {
+  public ActivityService(DbClient dbClient, ActivityIndexer indexer) {
     this.dbClient = dbClient;
-    this.indexClient = indexClient;
-  }
-
-  @Nullable
-  private String getAuthor() {
-    return (UserSession.get().login() != null) ? UserSession.get().login() : null;
-  }
-
-  private void save(DbSession session, ActivityDto log) {
-    dbClient.activityDao().insert(session,
-      log.setAuthor(getAuthor()));
-  }
-
-  public void write(DbSession session, Activity.Type type, String message) {
-    save(session, ActivityDto.createFor(message).setType(type));
-  }
-
-  public <L extends ActivityLog> void write(DbSession session, Activity.Type type, L log) {
-    this.save(session, ActivityDto.createFor(log)
-      .setType(type));
-  }
-
-  public ActivityQuery newActivityQuery() {
-    return new ActivityQuery();
+    this.indexer = indexer;
   }
 
-  public Result<Activity> search(ActivityQuery query, QueryContext options) {
-    ActivityIndex index = indexClient.get(ActivityIndex.class);
-    return new Result<Activity>(index, index.search(query, options));
+  public void save(Activity activity) {
+    ActivityDto dto = new ActivityDto()
+      .setKey(Uuids.create())
+      .setAuthor(UserSession.get().login())
+      .setAction(activity.getAction())
+      .setMessage(activity.getMessage())
+      .setData(KeyValueFormat.format(activity.getData()))
+      .setType(activity.getType().name());
+    dbClient.activityDao().insert(dto);
+    indexer.index();
   }
 }

server/sonar-server/src/main/java/org/sonar/server/activity/RubyQProfileActivityService.java

@@ -23,10 +23,10 @@
 import org.picocontainer.Startable;
 import org.sonar.api.ServerComponent;
 import org.sonar.api.utils.Paging;
+import org.sonar.server.es.SearchOptions;
 import org.sonar.server.qualityprofile.QProfileActivity;
 import org.sonar.server.qualityprofile.QProfileActivityQuery;
 import org.sonar.server.qualityprofile.QProfileService;
-import org.sonar.server.search.QueryContext;
 import org.sonar.server.search.Result;
 import org.sonar.server.util.RubyUtils;
 
@@ -51,11 +51,8 @@ public RubyQProfileActivityService(QProfileService service) {
    */
   public QProfileActivityResult search(Map<String, Object> params) {
     QProfileActivityQuery query = new QProfileActivityQuery();
-    QueryContext queryContext = new QueryContext().setMaxLimit();
-    List<String> profileKeys = RubyUtils.toStrings(params.get("profileKeys"));
-    if (profileKeys != null) {
-      query.setQprofileKeys(profileKeys);
-    }
+
+    query.setQprofileKey((String)params.get("profileKey"));
     Date since = RubyUtils.toDate(params.get("since"));
     if (since != null) {
       query.setSince(since);
@@ -64,12 +61,14 @@ public QProfileActivityResult search(Map<String, Object> params) {
     if (to != null) {
       query.setTo(to);
     }
+
+    SearchOptions options = new SearchOptions();
     Integer page = RubyUtils.toInteger(params.get("p"));
     int pageIndex = page != null ? page : 1;
-    queryContext.setPage(pageIndex, 50);
+    options.setPage(pageIndex, 50);
 
-    Result<QProfileActivity> result = service.searchActivities(query, queryContext);
-    return new QProfileActivityResult(result.getHits(), Paging.create(queryContext.getLimit(), pageIndex, (int) result.getTotal()));
+    Result<QProfileActivity> result = service.searchActivities(query, options);
+    return new QProfileActivityResult(result.getHits(), Paging.create(options.getLimit(), pageIndex, (int) result.getTotal()));
   }
 
   @Override

server/sonar-server/src/main/java/org/sonar/server/activity/db/ActivityDao.java

@@ -19,46 +19,39 @@
  */
 package org.sonar.server.activity.db;
 
+import org.sonar.api.ServerComponent;
 import org.sonar.api.utils.System2;
 import org.sonar.core.activity.db.ActivityDto;
 import org.sonar.core.activity.db.ActivityMapper;
+import org.sonar.core.persistence.DaoComponent;
 import org.sonar.core.persistence.DbSession;
-import org.sonar.server.db.BaseDao;
-import org.sonar.server.search.IndexDefinition;
+import org.sonar.core.persistence.MyBatis;
 
-import java.util.List;
+import java.util.Date;
 
-/**
- * @since 4.4
- */
-public class ActivityDao extends BaseDao<ActivityMapper, ActivityDto, String> {
+public class ActivityDao implements DaoComponent, ServerComponent {
 
-  public ActivityDao(System2 system) {
-    super(IndexDefinition.LOG, ActivityMapper.class, system);
-  }
+  private final MyBatis mybatis;
+  private final System2 system;
 
-  @Override
-  protected ActivityDto doGetNullableByKey(DbSession session, String key) {
-    throw new IllegalStateException("Cannot execute getByKey on Activities in DB");
+  public ActivityDao(MyBatis mybatis, System2 system) {
+    this.mybatis = mybatis;
+    this.system = system;
   }
 
-  @Override
-  protected ActivityDto doInsert(DbSession session, ActivityDto item) {
-    mapper(session).insert(item);
-    return item;
+  public void insert(ActivityDto dto) {
+    DbSession session = mybatis.openSession(false);
+    try {
+      insert(session, dto);
+      session.commit();
+    } finally {
+      MyBatis.closeQuietly(session);
+    }
   }
 
-  @Override
-  protected ActivityDto doUpdate(DbSession session, ActivityDto item) {
-    throw new IllegalStateException("Cannot update Log!");
+  public void insert(DbSession session, ActivityDto dto) {
+    dto.setCreatedAt(new Date(system.now()));
+    session.getMapper(ActivityMapper.class).insert(dto);
   }
 
-  @Override
-  protected void doDeleteByKey(DbSession session, String key) {
-    throw new IllegalStateException("Cannot delete Log!");
-  }
-
-  public List<ActivityDto> findAll(DbSession session) {
-    return mapper(session).selectAll();
-  }
 }

server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java

@@ -19,54 +19,88 @@
  */
 package org.sonar.server.activity.index;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.builder.ReflectionToStringBuilder;
-import org.sonar.core.activity.Activity;
 import org.sonar.server.search.BaseDoc;
-import org.sonar.server.search.IndexUtils;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
 
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Map;
 
-/**
- * @since 4.4
- */
-public class ActivityDoc extends BaseDoc implements Activity {
+public class ActivityDoc extends BaseDoc {
 
-  protected ActivityDoc(Map<String, Object> fields) {
+  public ActivityDoc(Map<String, Object> fields) {
     super(fields);
   }
 
-  @Override
-  public Date time() {
-    return IndexUtils.parseDateTime((String) getField(ActivityNormalizer.LogFields.CREATED_AT.field()));
+  @VisibleForTesting
+  ActivityDoc() {
+    super(new HashMap<String, Object>());
+  }
+
+  public void setCreatedAt(Date date) {
+    setField(ActivityIndexDefinition.FIELD_CREATED_AT, date);
+  }
+
+  public Date getCreatedAt() {
+    return getFieldAsDate(ActivityIndexDefinition.FIELD_CREATED_AT);
+  }
+
+  public String getKey() {
+    return this.getField(ActivityIndexDefinition.FIELD_KEY);
+  }
+
+  public void setKey(String s) {
+    setField(ActivityIndexDefinition.FIELD_KEY, s);
+  }
+
+  @CheckForNull
+  public String getLogin() {
+    return this.getNullableField(ActivityIndexDefinition.FIELD_LOGIN);
+  }
+
+  public void setLogin(@Nullable String s) {
+    setField(ActivityIndexDefinition.FIELD_LOGIN, s);
+  }
+
+  public String getType() {
+    return ((String) getField(ActivityIndexDefinition.FIELD_TYPE));
+  }
+
+  public void setType(String s) {
+    setField(ActivityIndexDefinition.FIELD_TYPE, s);
+  }
+
+  @CheckForNull
+  public String getAction() {
+    return this.getNullableField(ActivityIndexDefinition.FIELD_ACTION);
   }
 
-  @Override
-  public String login() {
-    return this.getNullableField(ActivityNormalizer.LogFields.LOGIN.field());
+  public void setAction(@Nullable String s) {
+    setField(ActivityIndexDefinition.FIELD_ACTION, s);
   }
 
-  @Override
-  public Type type() {
-    return Type.valueOf((String) getField(ActivityNormalizer.LogFields.TYPE.field()));
+  @CheckForNull
+  public Map<String, String> getDetails() {
+    return this.getNullableField(ActivityIndexDefinition.FIELD_DETAILS);
   }
 
-  @Override
-  public String action() {
-    return this.getNullableField(ActivityNormalizer.LogFields.ACTION.field());
+  public void setDetails(Map<String, String> details) {
+    setField(ActivityIndexDefinition.FIELD_DETAILS, details);
   }
 
-  @Override
-  public Map<String, String> details() {
-    return this.getNullableField(ActivityNormalizer.LogFields.DETAILS.field());
+  @CheckForNull
+  public String getMessage() {
+    return this.getNullableField(ActivityIndexDefinition.FIELD_MESSAGE);
   }
 
-  @Override
-  public String message() {
-    return this.getNullableField(ActivityNormalizer.LogFields.MESSAGE.field());
+  public void setMessage(@Nullable String s) {
+    setField(ActivityIndexDefinition.FIELD_MESSAGE, s);
   }
 
-  @Override
   public String toString() {
     return ReflectionToStringBuilder.toString(this);
   }