Skip to content

Commit

Permalink
RESTful service to List Diff Tasks : for large result lists avoid Out…
Browse files Browse the repository at this point in the history
…OfMemoryError #1333
  • Loading branch information
vrindanayak committed Apr 9, 2018
1 parent b771456 commit ef6007c
Show file tree
Hide file tree
Showing 7 changed files with 181 additions and 62 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import org.dcm4che3.json.JSONWriter;
import org.dcm4che3.ws.rs.MediaTypes;
import org.dcm4chee.arc.diff.DiffService;
import org.dcm4chee.arc.diff.DiffTaskOrder;
import org.dcm4chee.arc.diff.DiffTaskQuery;
import org.dcm4chee.arc.entity.AttributesBlob;
import org.dcm4chee.arc.entity.DiffTask;
import org.dcm4chee.arc.entity.QueueMessage;
Expand Down Expand Up @@ -144,12 +144,13 @@ public Response listDiffTasks(@QueryParam("accept") String accept) {
Variant.mediaTypes(MediaType.APPLICATION_JSON_TYPE, MediaTypes.TEXT_CSV_UTF8_TYPE).build())
.build();

List<DiffTask> diffTasks = diffService.listDiffTasks(
DiffTaskQuery diffTasks = diffService.listDiffTasks(
MatchTask.matchQueueMessage(
null, deviceName, status(), batchID, null, null, null, null),
MatchTask.matchDiffTask(localAET, primaryAET, secondaryAET, checkDifferent, checkMissing,
comparefields, createdTime, updatedTime),
order(orderby), parseInt(offset), parseInt(limit));
MatchTask.diffTaskOrder(orderby),
parseInt(offset), parseInt(limit));

return Response.ok(output.entity(diffTasks), output.type).build();
}
Expand Down Expand Up @@ -203,26 +204,30 @@ private Output selectMediaType(String accept) {
private enum Output {
JSON(MediaType.APPLICATION_JSON_TYPE) {
@Override
Object entity(final List<DiffTask> tasks) {
Object entity(final DiffTaskQuery tasks) {
return (StreamingOutput) out -> {
JsonGenerator gen = Json.createGenerator(out);
gen.writeStartArray();
for (DiffTask task : tasks)
task.writeAsJSONTo(gen);
gen.writeEnd();
gen.flush();
try (DiffTaskQuery t = tasks) {
JsonGenerator gen = Json.createGenerator(out);
gen.writeStartArray();
for (DiffTask task : t)
task.writeAsJSONTo(gen);
gen.writeEnd();
gen.flush();
}
};
}
},
CSV(MediaTypes.TEXT_CSV_UTF8_TYPE) {
@Override
Object entity(final List<DiffTask> tasks) {
Object entity(final DiffTaskQuery tasks) {
return (StreamingOutput) out -> {
Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
DiffTask.writeCSVHeader(writer, delimiter);
for (DiffTask task : tasks)
task.writeAsCSVTo(writer, delimiter);
writer.flush();
try (DiffTaskQuery t = tasks) {
Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
DiffTask.writeCSVHeader(writer, delimiter);
for (DiffTask task : t)
task.writeAsCSVTo(writer, delimiter);
writer.flush();
}
};
}
};
Expand All @@ -249,7 +254,7 @@ private static boolean isCSV(MediaType type) {
return csvCompatible;
}

abstract Object entity(final List<DiffTask> tasks);
abstract Object entity(final DiffTaskQuery tasks);
}

private StreamingOutput entity(List<byte[]> diffTaskAttributes) {
Expand Down Expand Up @@ -281,12 +286,6 @@ private static int parseInt(String s) {
return s != null ? Integer.parseInt(s) : 0;
}

private static DiffTaskOrder order(String orderby) {
return orderby != null
? DiffTaskOrder.valueOf(orderby.replace('-', '_'))
: DiffTaskOrder._updatedTime;
}

private QueueMessage.Status status() {
return status != null ? QueueMessage.Status.fromString(status) : null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,14 @@

package org.dcm4chee.arc.diff;

import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.Predicate;
import org.dcm4chee.arc.entity.DiffTask;
import org.dcm4chee.arc.qmgt.HttpServletRequestInfo;
import org.dcm4chee.arc.qmgt.Outcome;
import org.dcm4chee.arc.qmgt.QueueSizeLimitExceededException;

import java.util.Date;
import java.util.List;

/**
Expand All @@ -64,8 +66,8 @@ public interface DiffService {

Outcome executeDiffTask(DiffTask diffTask, HttpServletRequestInfo httpServletRequestInfo) throws Exception;

List<DiffTask> listDiffTasks(Predicate matchQueueMessage, Predicate matchDiffTask,
DiffTaskOrder order, int offset, int limit);
DiffTaskQuery listDiffTasks(Predicate matchQueueMessage, Predicate matchDiffTask,
OrderSpecifier<Date> order, int offset, int limit);

long countDiffTasks(Predicate matchQueueMessage, Predicate matchDiffTask);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,24 +41,13 @@

package org.dcm4chee.arc.diff;

import com.querydsl.core.types.OrderSpecifier;
import org.dcm4chee.arc.entity.QDiffTask;
import org.dcm4chee.arc.entity.DiffTask;

import java.util.Date;
import java.io.Closeable;

/**
* @author Gunter Zeilinger <gunterze@gmail.com>
* @since Mar 2018
* @author Vrinda Nayak <vrinda.nayak@j4care.com>
* @since Apr 2018
*/
public enum DiffTaskOrder {
createdTime(QDiffTask.diffTask.createdTime.asc()),
_createdTime(QDiffTask.diffTask.createdTime.desc()),
updatedTime(QDiffTask.diffTask.updatedTime.asc()),
_updatedTime(QDiffTask.diffTask.updatedTime.desc());

public final OrderSpecifier<Date> specifier;

DiffTaskOrder(OrderSpecifier<Date> specifier) {
this.specifier = specifier;
}
public interface DiffTaskQuery extends Closeable, Iterable<DiffTask> {
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,18 +45,20 @@
import com.querydsl.core.types.Predicate;
import com.querydsl.jpa.hibernate.HibernateQuery;
import org.dcm4che3.data.Attributes;
import org.dcm4chee.arc.diff.DiffContext;
import org.dcm4chee.arc.diff.DiffSCU;
import org.dcm4chee.arc.diff.DiffService;
import org.dcm4chee.arc.diff.DiffTaskOrder;
import org.dcm4che3.net.Device;
import org.dcm4chee.arc.conf.ArchiveDeviceExtension;
import org.dcm4chee.arc.diff.*;
import org.dcm4chee.arc.entity.*;
import org.dcm4chee.arc.qmgt.QueueManager;
import org.dcm4chee.arc.qmgt.QueueSizeLimitExceededException;
import org.hibernate.Session;
import org.hibernate.StatelessSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ejb.Stateless;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.inject.Inject;
import javax.jms.JMSException;
import javax.jms.Message;
Expand All @@ -82,6 +84,9 @@ public class DiffServiceEJB {
@Inject
private QueueManager queueManager;

@Inject
private Device device;

public void scheduleDiffTask(DiffContext ctx) throws QueueSizeLimitExceededException {
try {
ObjectMessage msg = queueManager.createObjectMessage(0);
Expand Down Expand Up @@ -133,15 +138,11 @@ public void updateDiffTask(DiffTask diffTask, DiffSCU diffSCU) {
diffTask.setDifferent(diffSCU.different());
}

public List<DiffTask> listDiffTasks(
Predicate matchQueueMessage, Predicate matchDiffTask, DiffTaskOrder order, int offset, int limit) {
HibernateQuery<DiffTask> diffTaskQuery = createQuery(matchQueueMessage, matchDiffTask);
if (limit > 0)
diffTaskQuery.limit(limit);
if (offset > 0)
diffTaskQuery.offset(offset);
diffTaskQuery.orderBy(order.specifier);
return diffTaskQuery.fetch();
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public DiffTaskQuery listDiffTasks(
Predicate matchQueueMessage, Predicate matchDiffTask, OrderSpecifier<Date> order, int offset, int limit) {
return new DiffTaskQueryImpl(
openStatelessSession(), queryFetchSize(), matchQueueMessage, matchDiffTask, order, offset, limit);
}

public long countDiffTasks(Predicate matchQueueMessage, Predicate matchDiffTask) {
Expand Down Expand Up @@ -169,4 +170,12 @@ public List<byte[]> getDiffTaskAttributes(DiffTask diffTask, int offset, int lim
.setMaxResults(limit)
.getResultList();
}

private StatelessSession openStatelessSession() {
return em.unwrap(Session.class).getSessionFactory().openStatelessSession();
}

private int queryFetchSize() {
return device.getDeviceExtension(ArchiveDeviceExtension.class).getQueryFetchSize();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,13 @@

package org.dcm4chee.arc.diff.impl;

import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.Predicate;
import org.dcm4che3.conf.api.ConfigurationException;
import org.dcm4che3.conf.api.IApplicationEntityCache;
import org.dcm4che3.data.Attributes;
import org.dcm4che3.net.Device;
import org.dcm4chee.arc.diff.DiffContext;
import org.dcm4chee.arc.diff.DiffService;
import org.dcm4chee.arc.diff.DiffSCU;
import org.dcm4chee.arc.diff.DiffTaskOrder;
import org.dcm4chee.arc.diff.*;
import org.dcm4chee.arc.entity.DiffTask;
import org.dcm4chee.arc.entity.QueueMessage;
import org.dcm4chee.arc.qmgt.HttpServletRequestInfo;
Expand All @@ -59,6 +57,7 @@

import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import java.util.Date;
import java.util.List;

/**
Expand Down Expand Up @@ -105,8 +104,8 @@ public Outcome executeDiffTask(DiffTask diffTask, HttpServletRequestInfo httpSer
}

@Override
public List<DiffTask> listDiffTasks(Predicate matchQueueMessage, Predicate matchDiffTask,
DiffTaskOrder order, int offset, int limit) {
public DiffTaskQuery listDiffTasks(Predicate matchQueueMessage, Predicate matchDiffTask,
OrderSpecifier<Date> order, int offset, int limit) {
return ejb.listDiffTasks(matchQueueMessage, matchDiffTask, order, offset, limit);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
* **** BEGIN LICENSE BLOCK *****
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is part of dcm4che, an implementation of DICOM(TM) in
* Java(TM), hosted at https://github.com/dcm4che.
*
* The Initial Developer of the Original Code is
* J4Care.
* Portions created by the Initial Developer are Copyright (C) 2015-2018
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
* See @authors listed below
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 2 or later (the "GPL"), or
* the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
* in which case the provisions of the GPL or the LGPL are applicable instead
* of those above. If you wish to allow use of your version of this file only
* under the terms of either the GPL or the LGPL, and not to allow others to
* use your version of this file under the terms of the MPL, indicate your
* decision by deleting the provisions above and replace them with the notice
* and other provisions required by the GPL or the LGPL. If you do not delete
* the provisions above, a recipient may use your version of this file under
* the terms of any one of the MPL, the GPL or the LGPL.
*
* **** END LICENSE BLOCK *****
*
*/

package org.dcm4chee.arc.diff.impl;

import com.mysema.commons.lang.CloseableIterator;
import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.Predicate;
import com.querydsl.jpa.hibernate.HibernateQuery;
import org.dcm4che3.util.SafeClose;
import org.dcm4chee.arc.diff.DiffTaskQuery;
import org.dcm4chee.arc.entity.*;
import org.hibernate.StatelessSession;
import org.hibernate.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Date;
import java.util.Iterator;

/**
* @author Vrinda Nayak <vrinda.nayak@j4care.com>
* @since Apr 2018
*/
class DiffTaskQueryImpl implements DiffTaskQuery {
private static final Logger LOG = LoggerFactory.getLogger(DiffTaskQueryImpl.class);
private final StatelessSession session;
private final HibernateQuery<DiffTask> query;
private Transaction transaction;
private CloseableIterator<DiffTask> iterate;

public DiffTaskQueryImpl(StatelessSession session, int fetchSize,
Predicate matchQueueMessage,
Predicate matchDiffTask,
OrderSpecifier<Date> order,
int offset, int limit) {
this.session = session;
HibernateQuery<QueueMessage> queueMsgQuery = new HibernateQuery<QueueMessage>(session)
.from(QQueueMessage.queueMessage)
.where(matchQueueMessage);

query = new HibernateQuery<DiffTask>(session)
.from(QDiffTask.diffTask)
.leftJoin(QDiffTask.diffTask.queueMessage, QQueueMessage.queueMessage)
.where(matchDiffTask, QDiffTask.diffTask.queueMessage.in(queueMsgQuery));
if (limit > 0)
query.limit(limit);
if (offset > 0)
query.offset(offset);
if (order != null)
query.orderBy(order);
query.setFetchSize(fetchSize);
}


@Override
public void close() {
SafeClose.close(iterate);
if (transaction != null) {
try {
transaction.commit();
} catch (Exception e) {
LOG.warn("Failed to commit transaction:\n{}", e);
}
}
SafeClose.close(session);
}

@Override
public Iterator<DiffTask> iterator() {
transaction = session.beginTransaction();
iterate = query.iterate();
return iterate;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,14 @@ public static OrderSpecifier<Date> retrieveBatchOrder(String orderby) {
return batchOrder(orderby, QRetrieveTask.retrieveTask.createdTime, QRetrieveTask.retrieveTask.updatedTime);
}

public static OrderSpecifier<Date> diffTaskOrder(String orderby) {
return taskOrder(orderby, QDiffTask.diffTask.createdTime, QDiffTask.diffTask.updatedTime);
}

public static OrderSpecifier<Date> diffBatchOrder(String orderby) {
return batchOrder(orderby, QDiffTask.diffTask.createdTime, QDiffTask.diffTask.updatedTime);
}

private static OrderSpecifier<Date> taskOrder(
String orderby, DateTimePath<Date> createdTime, DateTimePath<Date> updatedTime) {
return order(orderby, createdTime, updatedTime, ComparableExpressionBase::asc, ComparableExpressionBase::desc);
Expand All @@ -187,8 +195,8 @@ private static OrderSpecifier<Date> order(
String orderby,
DateTimePath<Date> createdTime,
DateTimePath<Date> updatedTime,
Function<DateTimeExpression<Date>,OrderSpecifier<Date>> asc,
Function<DateTimeExpression<Date>,OrderSpecifier<Date>> desc) {
Function<DateTimeExpression<Date>, OrderSpecifier<Date>> asc,
Function<DateTimeExpression<Date>, OrderSpecifier<Date>> desc) {
switch (orderby) {
case "createdTime":
return asc.apply(createdTime);
Expand Down

0 comments on commit ef6007c

Please sign in to comment.