From 2f98a27637fad17711fc57b97149d5182e157e71 Mon Sep 17 00:00:00 2001 From: Patrick Huang Date: Fri, 30 Jan 2015 15:00:37 +1000 Subject: [PATCH] rhbz1183994 - create DAO query to query user matrix for date range --- .../zanata/model/UserTranslationMatrix.java | 105 ++++++++ .../java/org/zanata/dao/DatabaseSpecific.java | 18 ++ .../main/java/org/zanata/dao/NativeQuery.java | 4 + .../zanata/dao/TextFlowTargetHistoryDAO.java | 98 ++++++++ .../server/TranslationUpdateListener.java | 11 +- .../db/changelogs/db.changelog-3.7.xml | 57 +++++ .../WEB-INF/classes/META-INF/persistence.xml | 1 + .../dao/TextFlowTargetHistoryDAOTest.java | 237 ++++++++++++++++++ ...extFlowTargetReviewCommentsDAOJPATest.java | 1 + .../org/zanata/model/HTextFlowBuilder.java | 5 + .../zanata/rest/service/DateRangeTest.java | 48 ++++ .../test/resources/META-INF/persistence.xml | 1 + .../test/resources/arquillian/persistence.xml | 1 + 13 files changed, 582 insertions(+), 5 deletions(-) create mode 100644 zanata-model/src/main/java/org/zanata/model/UserTranslationMatrix.java create mode 100644 zanata-war/src/main/java/org/zanata/dao/DatabaseSpecific.java create mode 100644 zanata-war/src/test/java/org/zanata/dao/TextFlowTargetHistoryDAOTest.java create mode 100644 zanata-war/src/test/java/org/zanata/rest/service/DateRangeTest.java diff --git a/zanata-model/src/main/java/org/zanata/model/UserTranslationMatrix.java b/zanata-model/src/main/java/org/zanata/model/UserTranslationMatrix.java new file mode 100644 index 0000000000..69ecd2bba0 --- /dev/null +++ b/zanata-model/src/main/java/org/zanata/model/UserTranslationMatrix.java @@ -0,0 +1,105 @@ +/* + * Copyright 2015, Red Hat, Inc. and individual contributors as indicated by the + * @author tags. See the copyright.txt file in the distribution for a full + * listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 2.1 of the License, or (at your option) + * any later version. + * + * This software is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this software; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF + * site: http://www.fsf.org. + */ +package org.zanata.model; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Immutable; +import org.joda.time.DateTimeZone; +import org.zanata.common.ContentState; + +import javax.persistence.Access; +import javax.persistence.AccessType; +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.io.Serializable; +import java.util.Date; +import java.util.TimeZone; + +/** + * This is a hibernate entity that will store aggregate word counts for a + * particular user in a given day. It mainly serves as a cache. The day is + * stored with a timezone offset to UTC. This means if user change his timezone, + * the entry in database will no longer be valid and needs to be deleted. + * + * @auther pahuang + */ +@Entity +@Access(AccessType.FIELD) +@Immutable +@Getter +public class UserTranslationMatrix implements Serializable { + // we store timezone offset so that we can compare equivalent time zones. We + // could use any instance to base on. Just need to be consistent. + public static final int TIMEZONE_OFFSET_INSTANCE = 0; + private static final long serialVersionUID = 1L; + @Id + @GeneratedValue + private Long id; + + @JoinColumn(name = "person_id", nullable = false, updatable = false) + @ManyToOne(targetEntity = HPerson.class, cascade = CascadeType.DETACH, + optional = false) + private HPerson person; + + @JoinColumn(name = "project_iteration_id", nullable = false, + updatable = false) + @ManyToOne(targetEntity = HProjectIteration.class, + cascade = CascadeType.DETACH, optional = false) + private HProjectIteration projectIteration; + + @Column(nullable = false, updatable = false) + private ContentState savedState; + + @Column(nullable = false, updatable = false) + private Long wordCount; + + @Temporal(TemporalType.DATE) + @Column(nullable = false, updatable = false) + private Date savedDate; + + @Column(nullable = false, updatable = false) + @Setter + private long timeZoneOffset; + + @JoinColumn(name = "locale_id", nullable = false, updatable = false) + @ManyToOne(targetEntity = HPerson.class, cascade = CascadeType.DETACH, + optional = false) + private HLocale locale; + + public UserTranslationMatrix(HPerson person, + HProjectIteration projectIteration, HLocale locale, + ContentState savedState, Long wordCount, Date savedDate) { + this.person = person; + this.projectIteration = projectIteration; + this.locale = locale; + this.savedState = savedState; + this.wordCount = wordCount; + this.savedDate = savedDate; + } +} diff --git a/zanata-war/src/main/java/org/zanata/dao/DatabaseSpecific.java b/zanata-war/src/main/java/org/zanata/dao/DatabaseSpecific.java new file mode 100644 index 0000000000..8d0bb249da --- /dev/null +++ b/zanata-war/src/main/java/org/zanata/dao/DatabaseSpecific.java @@ -0,0 +1,18 @@ +package org.zanata.dao; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marker annotation to mark methods that will only work in certain database. + */ +@Target({ElementType.METHOD, ElementType.LOCAL_VARIABLE, ElementType.TYPE, ElementType.PARAMETER}) +@Retention(RetentionPolicy.SOURCE) +public @interface DatabaseSpecific { + /** + * Reason why this is database specific + */ + String value() default ""; +} diff --git a/zanata-war/src/main/java/org/zanata/dao/NativeQuery.java b/zanata-war/src/main/java/org/zanata/dao/NativeQuery.java index 2b4f692a64..d4041fa495 100644 --- a/zanata-war/src/main/java/org/zanata/dao/NativeQuery.java +++ b/zanata-war/src/main/java/org/zanata/dao/NativeQuery.java @@ -11,4 +11,8 @@ @Target({ElementType.METHOD, ElementType.LOCAL_VARIABLE, ElementType.TYPE, ElementType.PARAMETER}) @Retention(RetentionPolicy.SOURCE) public @interface NativeQuery { + /** + * Reason why it has to be native query. + */ + String value() default ""; } diff --git a/zanata-war/src/main/java/org/zanata/dao/TextFlowTargetHistoryDAO.java b/zanata-war/src/main/java/org/zanata/dao/TextFlowTargetHistoryDAO.java index 723b23563c..06050cb1c6 100644 --- a/zanata-war/src/main/java/org/zanata/dao/TextFlowTargetHistoryDAO.java +++ b/zanata-war/src/main/java/org/zanata/dao/TextFlowTargetHistoryDAO.java @@ -20,6 +20,8 @@ */ package org.zanata.dao; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Date; import java.util.List; @@ -29,8 +31,19 @@ import org.jboss.seam.annotations.AutoCreate; import org.jboss.seam.annotations.Name; import org.jboss.seam.annotations.Scope; +import org.joda.time.DateTime; +import org.zanata.common.ContentState; +import org.zanata.model.HLocale; +import org.zanata.model.HPerson; +import org.zanata.model.HProjectIteration; import org.zanata.model.HTextFlowTarget; import org.zanata.model.HTextFlowTargetHistory; +import org.zanata.model.UserTranslationMatrix; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; + +import static org.zanata.model.UserTranslationMatrix.TIMEZONE_OFFSET_INSTANCE; @Name("textFlowTargetHistoryDAO") @AutoCreate @@ -159,4 +172,89 @@ public boolean findConflictInHistory(HTextFlowTarget target, return count != 0; } + /** + * Query to get total wordCount of a person(translated_by_id or + * reviewed_by_id) from HTextFlowTarget union HTextFlowTargetHistory tables + * within given date range group by lastChangeDate (date portion only), + * project version, locale and state. + * + * HTextFlowTargetHistory: gets all records translated from user in any + * version, any locale and dateRange. + * + * HTextFlowTarget: gets all records translated from user in any version, + * any locale and dateRange. + * + * @param user + * a HPerson person + * @param fromDate + * date from + * @param toDate + * date to + * + * @return a list of UserTranslationMatrix object + */ + @NativeQuery("need to use union") + @DatabaseSpecific("uses mysql date() function. In test we can override stripTimeFromDateTimeFunction(String) below to workaround it.") + public List getUserTranslationMatrix( + HPerson user, DateTime fromDate, DateTime toDate) { + // @formatter:off + String queryHistory = "select history.id, iter.id as iteration, tft.locale as locale, tf.wordCount as wordCount, history.state as state, history.lastChanged as lastChanged " + + " from HTextFlowTargetHistory history " + + " join HTextFlowTarget tft on tft.id = history.target_id " + + " join HTextFlow tf on tf.id = tft.tf_id " + + " join HDocument doc on doc.id = tf.document_id " + + " join HProjectIteration iter on iter.id = doc.project_iteration_id " + + " where history.lastChanged >= :fromDate and history.lastChanged <= :toDate " + + " and history.last_modified_by_id = :user and (history.translated_by_id is not null or history.reviewed_by_id is not null)"; + + String queryTarget = "select tft.id, iter.id as iteration, tft.locale as locale, tf.wordCount as wordCount, tft.state as state, tft.lastChanged as lastChanged " + + " from HTextFlowTarget tft " + + " join HTextFlow tf on tf.id = tft.tf_id " + + " join HDocument doc on doc.id = tf.document_id " + + " join HProjectIteration iter on iter.id = doc.project_iteration_id " + + " where tft.lastChanged >= :fromDate and tft.lastChanged <= :toDate " + + " and tft.last_modified_by_id = :user and (tft.translated_by_id is not null or tft.reviewed_by_id is not null)"; + // @formatter:on + String dateOfLastChanged = stripTimeFromDateTimeFunction("lastChanged"); + String queryString = + "select " + dateOfLastChanged + ", iteration, locale, state, sum(wordCount)" + + " from (" + + " (" + queryHistory + ") union (" + queryTarget + ")" + + " ) as all_translation" + + " group by " + dateOfLastChanged + ", iteration, locale, state " + + " order by lastChanged, iteration, locale, state"; + Query query = getSession().createSQLQuery(queryString) + .setParameter("user", user.getId()) + .setTimestamp("fromDate", fromDate.toDate()) + .setTimestamp("toDate", toDate.toDate()); + @SuppressWarnings("unchecked") + List result = query.list(); + ImmutableList.Builder builder = + ImmutableList.builder(); + for (Object[] objects : result) { + Date savedDate = new DateTime(objects[0]).toDate(); + HProjectIteration iteration = + loadById(objects[1], HProjectIteration.class); + HLocale locale = loadById(objects[2], HLocale.class); + ContentState savedState = ContentState.values()[(int) objects[3]]; + long wordCount = + ((BigDecimal) objects[4]).toBigInteger().longValue(); + UserTranslationMatrix matrix = + new UserTranslationMatrix(user, iteration, locale, + savedState, wordCount, savedDate); + builder.add(matrix); + } + return builder.build(); + } + + // This is so we can override it in test and be able to test it against h2 + @VisibleForTesting + protected String stripTimeFromDateTimeFunction(String columnName) { + return "date(" + columnName + ")"; + } + + private T loadById(Object object, Class entityClass) { + return (T) getSession().byId(entityClass).load( + ((BigInteger) object).longValue()); + } } diff --git a/zanata-war/src/main/java/org/zanata/webtrans/server/TranslationUpdateListener.java b/zanata-war/src/main/java/org/zanata/webtrans/server/TranslationUpdateListener.java index 8461ae6180..11752668e4 100644 --- a/zanata-war/src/main/java/org/zanata/webtrans/server/TranslationUpdateListener.java +++ b/zanata-war/src/main/java/org/zanata/webtrans/server/TranslationUpdateListener.java @@ -92,7 +92,8 @@ public void onPostUpdate(final PostUpdateEvent event) { if (!(entity instanceof HTextFlowTarget)) { return; } - + final HTextFlowTarget target = + HTextFlowTarget.class.cast(event.getEntity()); try { new Work() { @Override @@ -102,8 +103,7 @@ protected Void work() throws Exception { Lists.newArrayList(event.getOldState()), Predicates.instanceOf(ContentState.class)); - HTextFlowTarget target = - HTextFlowTarget.class.cast(event.getEntity()); + prepareTransUnitUpdatedEvent(target.getVersionNum() - 1, oldContentState, target); return null; @@ -193,13 +193,14 @@ public void onPostInsert(final PostInsertEvent event) { if (!(entity instanceof HTextFlowTarget)) { return; } + final HTextFlowTarget target = + HTextFlowTarget.class.cast(event.getEntity()); try { new Work() { @Override protected Void work() throws Exception { - HTextFlowTarget target = - HTextFlowTarget.class.cast(event.getEntity()); + prepareTransUnitUpdatedEvent(0, ContentState.New, target); return null; } diff --git a/zanata-war/src/main/resources/db/changelogs/db.changelog-3.7.xml b/zanata-war/src/main/resources/db/changelogs/db.changelog-3.7.xml index 122e970918..bca04428c5 100644 --- a/zanata-war/src/main/resources/db/changelogs/db.changelog-3.7.xml +++ b/zanata-war/src/main/resources/db/changelogs/db.changelog-3.7.xml @@ -32,6 +32,63 @@ + + Create User translation matrix table (aggregated matrix for each day) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + add index for lastChanged column to HTextFlowTargetHistory table + + + + + Add revisionComment to HTextFlowTarget, HTextFlowTargetHistory diff --git a/zanata-war/src/main/webapp-jboss/WEB-INF/classes/META-INF/persistence.xml b/zanata-war/src/main/webapp-jboss/WEB-INF/classes/META-INF/persistence.xml index 8257039542..065c63b29d 100644 --- a/zanata-war/src/main/webapp-jboss/WEB-INF/classes/META-INF/persistence.xml +++ b/zanata-war/src/main/webapp-jboss/WEB-INF/classes/META-INF/persistence.xml @@ -49,6 +49,7 @@ org.zanata.model.tm.TransMemoryUnitVariant org.zanata.model.tm.TransMemory org.zanata.model.WebHook + org.zanata.model.UserTranslationMatrix