Skip to content

Commit

Permalink
Merge pull request #1204 from virtualcell/shorten-n5-uri-and-name-dat…
Browse files Browse the repository at this point in the history 
…asets

JobID for Dataset Name and Truncated URL
  • Loading branch information
@AvocadoMoon
AvocadoMoon committed Apr 1, 2024
2 parents c03e91e + fdaf85a commit 238a954
Show file tree
Hide file tree
Showing 8 changed files with 48 additions and 63 deletions.
6 changes: 3 additions & 3 deletions vcell-core/src/main/java/cbit/vcell/export/server/ASCIIExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1269,7 +1269,7 @@ public Collection<ExportOutput> makeASCIIData(OutputContext outputContext, JobRe
return switch(asciiSpecs.getDataType()){
case PDE_VARIABLE_DATA -> exportPDEData(
outputContext,
jobRequest.getJobID(),
jobRequest.getExportJobID(),
user,
dataServerImpl,
exportSpecs.getVCDataIdentifier(),
Expand All @@ -1282,7 +1282,7 @@ public Collection<ExportOutput> makeASCIIData(OutputContext outputContext, JobRe
);
case ODE_VARIABLE_DATA -> exportODEData(
outputContext,
jobRequest.getJobID(),
jobRequest.getExportJobID(),
user,
dataServerImpl,
exportSpecs.getVCDataIdentifier(),
Expand All @@ -1293,7 +1293,7 @@ public Collection<ExportOutput> makeASCIIData(OutputContext outputContext, JobRe
);
case PDE_PARTICLE_DATA -> exportParticleData(
outputContext,
jobRequest.getJobID(),
jobRequest.getExportJobID(),
user,
dataServerImpl,
exportSpecs,
Expand Down
33 changes: 15 additions & 18 deletions vcell-core/src/main/java/cbit/vcell/export/server/ExportServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Hashtable;
import java.util.zip.DataFormatException;
Expand Down Expand Up @@ -208,7 +205,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
throw new DataAccessException("ERROR: user is null");
}
JobRequest newExportJob = JobRequest.createExportJobRequest(user);
jobRequestIDs.put(new Long(newExportJob.getJobID()), user);
jobRequestIDs.put(new Long(newExportJob.getExportJobID()), user);
if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): " + newExportJob + ", " + exportSpecs);
String fileFormat = null;
switch (exportSpecs.getFormat()) {
Expand Down Expand Up @@ -236,7 +233,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
// fileFormat = "IMAGEJ";
// break;
}
fireExportStarted(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat);
fireExportStarted(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat);

try {

Expand Down Expand Up @@ -281,11 +278,11 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
exportOutputs[0].writeDataToOutputStream(baos, fileDataContainerManager);//Get location of temp HDF5 file
File tempHDF5File = new File(baos.toString());
// File downloadableHDF5File = new File(exportBaseDir +exportOutputs[0].getSimID() + exportOutputs[0].getDataID() + ".hdf5");
File downloadableHDF5File = new File(exportBaseDir + newExportJob.getJobID() + ".hdf5");
File downloadableHDF5File = new File(exportBaseDir + newExportJob.getExportJobID() + ".hdf5");
Files.copy(tempHDF5File, downloadableHDF5File);
tempHDF5File.delete();
URL url = new URL(exportBaseURL + downloadableHDF5File.getName());
return fireExportCompleted(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
return fireExportCompleted(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
}
return makeRemoteFile(fileFormat, exportBaseDir, exportBaseURL, exportOutputs, exportSpecs, newExportJob,fileDataContainerManager);
case QUICKTIME:
Expand Down Expand Up @@ -331,7 +328,7 @@ public ExportEvent makeRemoteFile(OutputContext outputContext,User user, DataSer
}
catch (Throwable exc) {
lg.error(exc.getMessage(), exc);
fireExportFailed(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, exc.getMessage());
fireExportFailed(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, exc.getMessage());
throw new DataAccessException(exc.getMessage());
}
}
Expand All @@ -345,7 +342,7 @@ private ExportEvent makeRemoteFile(String fileFormat, String exportBaseDir, Stri
boolean exportValid = true;

// check outputs and package into zip file
File zipFile = new File(exportBaseDir + newExportJob.getJobID() + ".zip");
File zipFile = new File(exportBaseDir + newExportJob.getExportJobID() + ".zip");
BufferedOutputStream bout = new BufferedOutputStream(new FileOutputStream(zipFile));
ZipOutputStream zipOut = new ZipOutputStream(bout);
try {
Expand Down Expand Up @@ -376,7 +373,7 @@ private ExportEvent makeRemoteFile(String fileFormat, String exportBaseDir, Stri
completedExportRequests.put(exportSpecs, newExportJob);
if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + zipFile.getName());
URL url = new URL(exportBaseURL + zipFile.getName());
return fireExportCompleted(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
return fireExportCompleted(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
}
else {
throw new DataFormatException("Export Server could not produce valid data !");
Expand All @@ -390,10 +387,10 @@ private ExportEvent makeRemoteFile(String fileFormat, String exportBaseDir, Stri
*/
private ExportEvent makeRemoteFile(String fileFormat, String exportBaseDir, String exportBaseURL, ExportOutput[] exportOutputs, ExportSpecs exportSpecs, JobRequest newExportJob,FileDataContainerManager fileDataContainerManager) throws DataFormatException, IOException, MalformedURLException {
boolean exportValid = true;
fireExportAssembling(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat);
fireExportAssembling(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat);

// check outputs and package into zip file
File zipFile = new File(exportBaseDir + newExportJob.getJobID() + ".zip");
File zipFile = new File(exportBaseDir + newExportJob.getExportJobID() + ".zip");
FileOutputStream fileOut = new FileOutputStream(zipFile);
BufferedOutputStream bos = new BufferedOutputStream(fileOut);
ZipOutputStream zipOut = new ZipOutputStream(bos);
Expand All @@ -419,7 +416,7 @@ private ExportEvent makeRemoteFile(String fileFormat, String exportBaseDir, Stri
completedExportRequests.put(exportSpecs, newExportJob);
if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + zipFile.getName());
URL url = new URL(exportBaseURL + zipFile.getName());
return fireExportCompleted(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
return fireExportCompleted(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
}
else {
throw new DataFormatException("Export Server could not produce valid data !");
Expand All @@ -437,7 +434,7 @@ private ExportEvent makeRemoteFile_Unzipped(String fileFormat, String exportBase
{
//do the first file of exportOutputs separately (for VFRAP, there is only one export output)
String extStr = "." + fileFormat;
File file = new File(exportBaseDir + newExportJob.getJobID() + extStr);
File file = new File(exportBaseDir + newExportJob.getExportJobID() + extStr);
FileOutputStream fileOut = new FileOutputStream(file);
BufferedOutputStream out= new BufferedOutputStream(fileOut);
exportOutputs[0].writeDataToOutputStream(out,fileDataContainerManager);
Expand All @@ -449,7 +446,7 @@ private ExportEvent makeRemoteFile_Unzipped(String fileFormat, String exportBase
{
if (exportOutputs[i].isValid())
{
File moreFile = new File(exportBaseDir + newExportJob.getJobID()+"_"+ i + extStr);
File moreFile = new File(exportBaseDir + newExportJob.getExportJobID()+"_"+ i + extStr);
FileOutputStream moreFileOut = new FileOutputStream(moreFile);
ObjectOutputStream moreOut= new ObjectOutputStream(moreFileOut);
exportOutputs[i].writeDataToOutputStream(moreOut,fileDataContainerManager);
Expand All @@ -470,7 +467,7 @@ private ExportEvent makeRemoteFile_Unzipped(String fileFormat, String exportBase
completedExportRequests.put(exportSpecs, newExportJob);
if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + fileNames);
URL url = new URL(exportBaseURL + fileNames);
return fireExportCompleted(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
return fireExportCompleted(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, url.toString(),exportSpecs);
}
else {
throw new DataFormatException("Export Server could not produce valid data !");
Expand All @@ -484,9 +481,9 @@ private ExportEvent makeRemoteN5File(String fileFormat, String fileName, ExportO
String url = PropertyLoader.getRequiredProperty(PropertyLoader.s3ExportBaseURLProperty);
String uri = url + ":" + PropertyLoader.getRequiredProperty(PropertyLoader.s3ProxyExternalPort) + "/" + n5Exporter.getN5FilePathSuffix();
N5Specs n5Specs = (N5Specs) exportSpecs.getFormatSpecificSpecs();
uri += "?dataSetName=" + URLEncoder.encode(n5Specs.dataSetName, StandardCharsets.UTF_8);
uri += "?dataSetName=" + newExportJob.getExportJobID();
if (lg.isTraceEnabled()) lg.trace("ExportServiceImpl.makeRemoteFile(): Successfully exported to file: " + fileName);
return fireExportCompleted(newExportJob.getJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, uri, exportSpecs);
return fireExportCompleted(newExportJob.getExportJobID(), exportSpecs.getVCDataIdentifier(), fileFormat, uri, exportSpecs);
}
else {
throw new DataFormatException("Export Server could not produce valid data !");
Expand Down
2 changes: 1 addition & 1 deletion vcell-core/src/main/java/cbit/vcell/export/server/IMGExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ public ExportOutput[] makeMediaData(
}
ExportOutput[] mediaArr = null;
if(particleMode == FormatSpecificSpecs.PARTICLE_NONE){
mediaArr = makeMedia(exportServiceImpl,outputContext,jobRequest.getJobID(),user,dataServerImpl,exportSpecs,clientTaskStatusSupport,null/*particleInfo*/,fileDataContainerManager);
mediaArr = makeMedia(exportServiceImpl,outputContext,jobRequest.getExportJobID(),user,dataServerImpl,exportSpecs,clientTaskStatusSupport,null/*particleInfo*/,fileDataContainerManager);
}else if(particleMode == FormatSpecificSpecs.PARTICLE_SELECT){
// ParticleInfo particleInfo = checkParticles_unused(exportSpecs,user,dataServerImpl,jobRequest.getJobID());
try{
Expand Down
2 changes: 1 addition & 1 deletion vcell-core/src/main/java/cbit/vcell/export/server/JobRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public static JobRequest createExportJobRequest(User user) {
* Creation date: (4/3/2001 4:21:34 PM)
* @return long
*/
public long getJobID() {
public long getExportJobID() {
return jobID;
}
/**
Expand Down
23 changes: 7 additions & 16 deletions vcell-core/src/main/java/cbit/vcell/export/server/N5Exporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.DoubleArrayDataBlock;
import org.janelia.saalfeldlab.n5.N5FSWriter;
Expand All @@ -35,7 +34,6 @@
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;


public class N5Exporter implements ExportConstants {
Expand Down Expand Up @@ -105,15 +103,14 @@ private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs
// rewrite so that it still results in a tmp file does not raise File already exists error
N5FSWriter n5FSWriter = new N5FSWriter(getN5FileAbsolutePath(), new GsonBuilder());
DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, org.janelia.saalfeldlab.n5.DataType.FLOAT64, n5Specs.getCompression());
String dataSetName = n5Specs.dataSetName;
n5FSWriter.createDataset(dataSetName, datasetAttributes);
N5Specs.writeImageJMetaData(dimensions, blockSize, n5Specs.getCompression(),n5FSWriter, dataSetName, numVariables, blockSize[3], allTimes.length, exportSpecs.getHumanReadableExportData().subVolume);
n5FSWriter.createDataset(String.valueOf(jobID), datasetAttributes);
N5Specs.writeImageJMetaData(jobID, dimensions, blockSize, n5Specs.getCompression(), n5FSWriter, n5Specs.dataSetName, numVariables, blockSize[3], allTimes.length, exportSpecs.getHumanReadableExportData().subVolume);

//Create mask
for(int timeIndex = timeSpecs.getBeginTimeIndex(); timeIndex <= timeSpecs.getEndTimeIndex(); timeIndex++){
int normalizedTimeIndex = timeIndex - timeSpecs.getBeginTimeIndex();
DoubleArrayDataBlock doubleArrayDataBlock = new DoubleArrayDataBlock(blockSize, new long[]{0, 0, 0, 0, normalizedTimeIndex}, mask);
n5FSWriter.writeBlock(dataSetName, datasetAttributes, doubleArrayDataBlock);
n5FSWriter.writeBlock(String.valueOf(jobID), datasetAttributes, doubleArrayDataBlock);
}

for (int variableIndex=1; variableIndex < numVariables; variableIndex++){
Expand All @@ -122,7 +119,7 @@ private ExportOutput exportToN5(OutputContext outputContext, long jobID, N5Specs
int normalizedTimeIndex = timeIndex - timeSpecs.getBeginTimeIndex();
double[] data = this.dataServer.getSimDataBlock(outputContext, user, this.vcDataID, variableNames[variableIndex - 1], allTimes[timeIndex]).getData();
DoubleArrayDataBlock doubleArrayDataBlock = new DoubleArrayDataBlock(blockSize, new long[]{0, 0, variableIndex, 0, (normalizedTimeIndex)}, data);
n5FSWriter.writeBlock(dataSetName, datasetAttributes, doubleArrayDataBlock);
n5FSWriter.writeBlock(String.valueOf(jobID), datasetAttributes, doubleArrayDataBlock);
if(timeIndex % 3 == 0){
double progress = (double) (variableIndex + normalizedTimeIndex) / (numVariables + (numTimes * numVariables));
exportServiceImpl.fireExportProgress(jobID, vcDataID, N5Specs.n5Suffix.toUpperCase(), progress);
Expand Down Expand Up @@ -166,17 +163,11 @@ public String getN5FilePathSuffix(){
public String getN5FileNameHash(){
return actualHash(vcDataID.getDataKey().toString(), String.valueOf(vcDataID.getJobIndex()));
}

public static String getN5FileNameHash(String simID, String jobID){
return actualHash(simID, jobID);
}

private static String actualHash(String simID, String jobID) {
MessageDigest sha256 = DigestUtils.getMd5Digest();
sha256.update(simID.getBytes(StandardCharsets.UTF_8));
// sha256.update(jobID.getBytes(StandardCharsets.UTF_8));

return Hex.encodeHexString(sha256.digest());
String hashString = Hex.encodeHexString(sha256.digest());
return hashString.substring(17);
}


Expand All @@ -190,7 +181,7 @@ public ExportOutput makeN5Data(OutputContext outputContext, JobRequest jobReques
if (formatSpecs instanceof N5Specs n5Specs){
return exportToN5(
outputContext,
jobRequest.getJobID(),
jobRequest.getExportJobID(),
n5Specs,
exportSpecs,
fileDataContainerManager
Expand Down
4 changes: 2 additions & 2 deletions vcell-core/src/main/java/cbit/vcell/export/server/N5Specs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,12 @@ public String toString() {
return "N5Specs: [compression: " + format + ", chunking: " + dataType + ", switchRowsColumns: " + "]";
}

public static void writeImageJMetaData(long[] dimensions, int[] blockSize, Compression compression, N5FSWriter n5FSWriter, String datasetName, int numChannels, int zSlices,
public static void writeImageJMetaData(long jobID,long[] dimensions, int[] blockSize, Compression compression, N5FSWriter n5FSWriter, String datasetName, int numChannels, int zSlices,
int timeLength, HashMap<Integer, String> maskMapping) throws MathException, DataAccessException {
try {
HashMap<String, String> compresssionMap = new HashMap<>(){{put("type", compression.getType().toLowerCase());}};
ImageJMetaData imageJMetaData = ImageJMetaData.generateDefaultRecord(dimensions, blockSize, compresssionMap, datasetName, numChannels, zSlices, timeLength, maskMapping);
Path path = Path.of(n5FSWriter.getURI().getPath(), datasetName, "attributes.json");
Path path = Path.of(n5FSWriter.getURI().getPath(), String.valueOf(jobID), "attributes.json");
Gson gson = n5FSWriter.getGson();
String jsonRepresentation = gson.toJson(imageJMetaData, ImageJMetaData.class);
FileWriter fileWriter = new FileWriter(path.toFile());
Expand Down
12 changes: 6 additions & 6 deletions vcell-core/src/main/java/cbit/vcell/export/server/RasterExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,7 @@ public NrrdInfo[] makeRasterData(OutputContext outputContext,JobRequest jobReque
throws RemoteException, DataAccessException, IOException {
return exportPDEData(
outputContext,
jobRequest.getJobID(),
jobRequest.getExportJobID(),
user,
dataServerImpl,
exportSpecs.getVCDataIdentifier(),
Expand Down Expand Up @@ -862,7 +862,7 @@ public ExportOutput[] makeUCDData(OutputContext outputContext,JobRequest jobRequ
// for (int i = 0; i < variableSpecs.getVariableNames().length; i++){
for (int j = timeSpecs.getBeginTimeIndex(); j <= timeSpecs.getEndTimeIndex(); j++){
exportServiceImpl.fireExportProgress(
jobRequest.getJobID(), vcdID, "UCD",
jobRequest.getExportJobID(), vcdID, "UCD",
(double)(j-timeSpecs.getBeginTimeIndex())/(double)(timeSpecs.getEndTimeIndex()-timeSpecs.getBeginTimeIndex()+1));

// String fileID = simID + "_Full_" + formatTime(timeSpecs.getAllTimes()[j]) + "time_" + variableSpecs.getVariableNames().length + "vars";
Expand Down Expand Up @@ -967,7 +967,7 @@ public ExportOutput[] makeVTKImageData(OutputContext outputContext,JobRequest jo
Vector<ExportOutput> exportOutV = new Vector<ExportOutput>();
for (int j = timeSpecs.getBeginTimeIndex(); j <= timeSpecs.getEndTimeIndex(); j++){
exportServiceImpl.fireExportProgress(
jobRequest.getJobID(), vcdID, "VTKIMG",
jobRequest.getExportJobID(), vcdID, "VTKIMG",
(double)(j-timeSpecs.getBeginTimeIndex())/(double)(timeSpecs.getEndTimeIndex()-timeSpecs.getBeginTimeIndex()+1));

StringBuffer sb = new StringBuffer();
Expand Down Expand Up @@ -1051,7 +1051,7 @@ public ExportOutput[] makeVTKUnstructuredData_Chombo(OutputContext outputContext
ChomboVtkFileWriter chomboVTKFileWriter = new ChomboVtkFileWriter();
File[] vtkFiles = chomboVTKFileWriter.writeVtuExportFiles(chomboFiles, tmpDir, new ChomboVtkFileWriter.ProgressListener() {
public void progress(double percentDone) {
exportServiceImpl.fireExportProgress(jobRequest.getJobID(), vcdID, "VTKUNSTR", percentDone);
exportServiceImpl.fireExportProgress(jobRequest.getExportJobID(), vcdID, "VTKUNSTR", percentDone);
}
});

Expand Down Expand Up @@ -1080,7 +1080,7 @@ public ExportOutput[] makeVTKUnstructuredData_VCell(OutputContext outputContext,
CartesianMeshVtkFileWriter cartesianMeshVtkFileWriter = new CartesianMeshVtkFileWriter();
File[] vtkFiles = cartesianMeshVtkFileWriter.writeVtuExportFiles(vcellFiles, tmpDir, new CartesianMeshVtkFileWriter.ProgressListener() {
public void progress(double percentDone) {
exportServiceImpl.fireExportProgress(jobRequest.getJobID(), vcdID, "VTKUNSTR", percentDone);
exportServiceImpl.fireExportProgress(jobRequest.getExportJobID(), vcdID, "VTKUNSTR", percentDone);
}
});

Expand Down Expand Up @@ -1113,7 +1113,7 @@ public ExportOutput[] makeVTKUnstructuredData(OutputContext outputContext,JobReq
//for (int i = 0; i < variableSpecs.getVariableNames().length; i++){
for (int j = timeSpecs.getBeginTimeIndex(); j <= timeSpecs.getEndTimeIndex(); j++){
exportServiceImpl.fireExportProgress(
jobRequest.getJobID(), vcdID, "VTKUNSTR",
jobRequest.getExportJobID(), vcdID, "VTKUNSTR",
(double)(j-timeSpecs.getBeginTimeIndex())/(double)(timeSpecs.getEndTimeIndex()-timeSpecs.getBeginTimeIndex()+1));

//String fileID = simID + "_Full_" + formatTime(timeSpecs.getAllTimes()[j]) + "time_" + variableSpecs.getVariableNames().length + "vars";
Expand Down

0 comments on commit 238a954

Please sign in to comment.