Skip to content

Commit

Permalink
Added integration of Emailchemy
Browse files Browse the repository at this point in the history
  • Loading branch information
@jfarwer
jfarwer committed Jan 9, 2023
1 parent 1f18f7f commit dac9273
Show file tree
Hide file tree
Showing 36 changed files with 2,300 additions and 681 deletions.
2 changes: 1 addition & 1 deletion .idea/compiler.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 0 additions & 13 deletions .idea/libraries/Maven__org_apache_commons_commons_lang3_3_3_2.xml
View file

This file was deleted.

2 changes: 1 addition & 1 deletion WebContent/WEB-INF/classes/messages.properties
View file
Expand Up @@ -626,7 +626,7 @@ browse.loading = Loading
browse.messages = Messages

browse.label-all-messages = Label all these Messages
browse.download-as-mbox = Download Messages as MBOX
browse.export-message = Download Messages
browse.previous-message = Previous Message
browse.next-message = Next Message

Expand Down
1 change: 1 addition & 0 deletions WebContent/add-accession.jsp
View file
Expand Up @@ -134,6 +134,7 @@
// in turn, call to epadd.import_accession
//epadd.import_accession(e, muse.collect_input_fields());
//If you change exportAccessionProcessing then also change EmailExporter.EXPORT_ACCESSION_PROCESSING
var post_params = '&exportableAssets=exportAccessionProcessing&exportableAssetsFiles=' + accpath;
var page = "ajax/async/setExportableAssets.jsp";
console.log("add-accession.jsp: goButton.click: post_params=" + post_params);
Expand Down
4 changes: 2 additions & 2 deletions WebContent/ajax/async/export-from-appraisal.jsp
View file
Expand Up @@ -81,14 +81,14 @@
String folder = dir + File.separator + "ePADD archive of " + bestName;
List<Document> docsToExport = new ArrayList<>();
docsToExport = archive.getDocsForExport(Archive.Export_Mode.EXPORT_APPRAISAL_TO_PROCESSING);
docsToExport = archive.getDocsForExport(Archive.ExportMode.EXPORT_APPRAISAL_TO_PROCESSING);
//From appraisal to processing we do not remove any label from labelManager so pass the current
//label info map as an argument to set labelmanager for the exported archive.
JSPHelper.log.info("Exporting #"+docsToExport.size()+" docs");
// to do: need a progress bar here
try {
archive.export(docsToExport, Archive.Export_Mode.EXPORT_APPRAISAL_TO_PROCESSING, folder, "default",setStatusProvider);
archive.export(docsToExport, Archive.ExportMode.EXPORT_APPRAISAL_TO_PROCESSING, folder, "default",setStatusProvider);
} catch (Exception e) {
Util.print_exception ("Error trying to export archive", e, JSPHelper.log);
error = "Sorry, error exporting archive: " + e + ". Please see the log file for more details.";
Expand Down
8 changes: 4 additions & 4 deletions WebContent/ajax/async/export-from-processing.jsp
View file
Expand Up @@ -94,7 +94,7 @@ Archive archive = JSPHelper.getArchive(params);
String folderPublic = dir + File.separator + "ePADD archive of " + bestName + "-Discovery";
//same set of docs are exported from processing to delivery or discovery only difference being the
//content of these messages. In case of processing to discovery mode the redaction takes place.
List<Document> docsToExport = archive.getDocsForExport(Archive.Export_Mode.EXPORT_PROCESSING_TO_DELIVERY);
List<Document> docsToExport = archive.getDocsForExport(Archive.ExportMode.EXPORT_PROCESSING_TO_DELIVERY);
/*
Before v5 we faced an issue where after exporting to delivery/discovery the document's content got
Expand Down Expand Up @@ -122,7 +122,7 @@ Archive archive = JSPHelper.getArchive(params);
// archive.collectionMetadata.entityCounts = null;
archive.collectionMetadata.numPotentiallySensitiveMessages = -1;
try {
archive.export(docsToExport, Archive.Export_Mode.EXPORT_PROCESSING_TO_DELIVERY, folder, "default",setStatusProvider);
archive.export(docsToExport, Archive.ExportMode.EXPORT_PROCESSING_TO_DELIVERY, folder, "default",setStatusProvider);
// //remove fordelivery archive from the global map
// SimpleSessions.removeFromGlobalArchiveMap(folder,fordelivery);
} catch (Exception e) {
Expand All @@ -139,7 +139,7 @@ Archive archive = JSPHelper.getArchive(params);
//Read archive from folderPublic directory and operate on that.
//Archive forDeliveryPublic = SimpleSessions.readArchiveIfPresent(folderPublic);
//With the introduction of 'Transfer only to Delivery' label, the set of docs exported to Delivery will not be same for Discovery.
docsToExport = archive.getDocsForExport(Archive.Export_Mode.EXPORT_PROCESSING_TO_DISCOVERY);
docsToExport = archive.getDocsForExport(Archive.ExportMode.EXPORT_PROCESSING_TO_DISCOVERY);
JSPHelper.log.info("Exporting for discovery");
/*v6- why were we exporting correspondent authority file separately? Removed now.
try {
Expand All @@ -156,7 +156,7 @@ Archive archive = JSPHelper.getArchive(params);
}
*/
try {
archive.export(docsToExport, Archive.Export_Mode.EXPORT_PROCESSING_TO_DISCOVERY/* public mode */, folderPublic, "default",setStatusProvider);
archive.export(docsToExport, Archive.ExportMode.EXPORT_PROCESSING_TO_DISCOVERY/* public mode */, folderPublic, "default",setStatusProvider);
}catch (Exception e) {
Util.print_exception ("Error trying to export archive", e, JSPHelper.log);
error = "Sorry, error exporting archive: " + e + ". Please see the log file for more details.";
Expand Down
51 changes: 33 additions & 18 deletions WebContent/ajax/async/setExportableAssets.jsp
View file
@@ -1,7 +1,7 @@
<%@page language="java" contentType="application/json;charset=UTF-8"%>
<%@ page import="edu.stanford.muse.webapp.JSPHelper" %>
<%@ page import="edu.stanford.muse.util.Util" %>
<%@ page import="org.json.JSONArray" %><%@ page import="org.json.JSONObject"%><%@ page import="org.json.CDL"%><%@ page import="org.apache.commons.io.FileUtils"%><%@ page import="au.com.bytecode.opencsv.CSVWriter"%><%@ page import="java.util.*"%><%@ page import="edu.stanford.muse.ner.model.NEType"%><%@ page import="edu.stanford.muse.index.*"%><%@ page import="java.io.*"%><%@ page import="java.util.zip.GZIPOutputStream"%><%@ page import="java.util.zip.ZipOutputStream"%><%@ page import="java.util.zip.ZipEntry"%><%@ page import="edu.stanford.muse.util.EmailUtils"%><%@ page import="edu.stanford.muse.webapp.ModeConfig"%><%@ page import="edu.stanford.muse.AddressBookManager.AddressBook"%><%@ page import="edu.stanford.muse.email.StatusProvider"%><%@ page import="com.google.common.collect.Multimap"%><%@ page import="java.util.function.Consumer"%><%@ page import="edu.stanford.epadd.util.OperationInfo"%><%@ page import="edu.stanford.muse.email.StaticStatusProvider"%>
<%@ page import="org.json.JSONArray" %><%@ page import="org.json.JSONObject"%><%@ page import="org.json.CDL"%><%@ page import="org.apache.commons.io.FileUtils"%><%@ page import="au.com.bytecode.opencsv.CSVWriter"%><%@ page import="java.util.*"%><%@ page import="edu.stanford.muse.ner.model.NEType"%><%@ page import="edu.stanford.muse.index.*"%><%@ page import="java.io.*"%><%@ page import="java.util.zip.GZIPOutputStream"%><%@ page import="java.util.zip.ZipOutputStream"%><%@ page import="java.util.zip.ZipEntry"%><%@ page import="edu.stanford.muse.util.EmailUtils"%><%@ page import="edu.stanford.muse.webapp.ModeConfig"%><%@ page import="edu.stanford.muse.AddressBookManager.AddressBook"%><%@ page import="edu.stanford.muse.email.StatusProvider"%><%@ page import="com.google.common.collect.Multimap"%><%@ page import="java.util.function.Consumer"%><%@ page import="edu.stanford.epadd.util.OperationInfo"%><%@ page import="edu.stanford.muse.email.StaticStatusProvider"%><%@ page import="edu.stanford.epadd.util.EmailConvert"%>
<%
//This api needs to be supported for both types of flows - long running with status bar and normal (without status bar). Theso two invocation types of this jsp will be identified
Expand Down Expand Up @@ -76,7 +76,9 @@ if(opID==null){
<%!
private void setExportableAssets(Multimap<String,String> params, Consumer<StatusProvider> setStatusProvider, HttpSession session, JSONObject resultJSON, Archive archive){
if(setStatusProvider!=null)
setStatusProvider.accept(new StaticStatusProvider("setting exportable assets..."));
{
setStatusProvider.accept(new StaticStatusProvider("Copying files for preservation ..."));
}
String exportableAssets = JSPHelper.getParam(params,"exportableAssets");
String exportableAssetsFiles = JSPHelper.getParam(params,"exportableAssetsFiles");
Expand All @@ -101,24 +103,37 @@ private void setExportableAssets(Multimap<String,String> params, Consumer<Status
assetsLocation = new ArrayList(Arrays.asList(exportableAssetsFiles.split("\\^-\\^")));
}
if ("exportAcquisitioned".equals(exportableAssets)){
System.out.println("exportableAssets = exportAcquisitioned");
System.out.println("exportableAssets = " + exportableAssets);
if (exportableAssets != null && !exportableAssets.isEmpty())
{
switch(exportableAssets)
{
case EmailExporter.EXPORT_ACQUISITIONED:
// Notes: assetsLocation here should be a list of full path filenames
exportResult = archive.setExportableAssets(Archive.Exportable_Assets.EXPORTABLE_APPRAISAL_CANONICAL_ACQUISITIONED, assetsLocation);
exportResult = archive.setExportableAssets(Archive.Exportable_Assets.EXPORTABLE_APPRAISAL_NORMALIZED_ACQUISITIONED, assetsLocation);
} else if ("exportAppraised".equals(exportableAssets)){
System.out.println("exportableAssets = exportAppraised");
exportResult = archive.setExportableAssets(Archive.Exportable_Assets.EXPORTABLE_APPRAISAL_NORMALIZED_APPRAISED);
} else if ("exportProcessing".equals(exportableAssets)){
System.out.println("exportableAssets = exportProcessing");
exportResult = archive.setExportableAssets(Archive.Exportable_Assets.EXPORTABLE_PROCESSING_NORMALIZED);
} else if ("exportAccessionProcessing".equals(exportableAssets)){
System.out.println("exportableAssets = exportAccessionProcessing");
new EmailExporter(exportableAssets, archive, setStatusProvider).exportExportableAssets(Archive.AssetType.APPRAISAL_CANONICAL_ACQUISITIONED, assetsLocation);
new EmailExporter(exportableAssets, archive, setStatusProvider).exportExportableAssets(Archive.AssetType.APPRAISAL_NORMALIZED_ACQUISITIONED, assetsLocation);
EmailConvert.deleteTmpDir();
break;
case EmailExporter.EXPORT_APPRAISED_EML:
case EmailExporter.EXPORT_APPRAISED_MBOX:
new EmailExporter(exportableAssets, archive, setStatusProvider).exportExportableAssets(Archive.AssetType.APPRAISAL_NORMALIZED_APPRAISED);
break;
case EmailExporter.EXPORT_PROCESSING:
new EmailExporter(exportableAssets, archive, setStatusProvider).exportExportableAssets(Archive.AssetType.PROCESSING_NORMALIZED);
break;
case EmailExporter.EXPORT_ACCESSION_PROCESSING:
// Notes: assetsLocation here should be a list of folder paths
exportResult = archive.setExportableAssets(Archive.Exportable_Assets.EXPORTABLE_PROCESSING_NORMALIZED, assetsLocation);
} else if ("exportProcessed".equals(exportableAssets)){
System.out.println("exportableAssets = exportProcessed");
exportResult = archive.setExportableAssets(Archive.Exportable_Assets.EXPORTABLE_PROCESSING_NORMALIZED_PROCESSED);
new EmailExporter(exportableAssets, archive, setStatusProvider).exportExportableAssets(Archive.AssetType.PROCESSING_NORMALIZED, assetsLocation);
break;
case EmailExporter.EXPORT_PROCESSED_EML:
case EmailExporter.EXPORT_PROCESSED_MBOX:
new EmailExporter(exportableAssets, archive, setStatusProvider).exportExportableAssets(Archive.AssetType.PROCESSING_NORMALIZED_PROCESSED);
break;
}
}
if (!Util.nullOrEmpty(error)){
Expand Down
46 changes: 27 additions & 19 deletions WebContent/ajax/downloadData.jsp
View file
@@ -1,7 +1,7 @@
<%@page language="java" contentType="application/json;charset=UTF-8"%>
<%@ page import="edu.stanford.muse.webapp.JSPHelper" %>
<%@ page import="edu.stanford.muse.util.Util" %>
<%@ page import="org.json.JSONArray" %><%@ page import="org.json.JSONObject"%><%@ page import="org.json.CDL"%><%@ page import="org.apache.commons.io.FileUtils"%><%@ page import="au.com.bytecode.opencsv.CSVWriter"%><%@ page import="java.util.*"%><%@ page import="edu.stanford.muse.ner.model.NEType"%><%@ page import="edu.stanford.muse.index.*"%><%@ page import="java.io.*"%><%@ page import="java.util.zip.GZIPOutputStream"%><%@ page import="java.util.zip.ZipOutputStream"%><%@ page import="java.util.zip.ZipEntry"%><%@ page import="edu.stanford.muse.util.EmailUtils"%><%@ page import="edu.stanford.muse.webapp.ModeConfig"%><%@ page import="edu.stanford.muse.AddressBookManager.AddressBook"%><%@ page import="edu.stanford.muse.email.StatusProvider"%><%@ page import="com.google.common.collect.Multimap"%><%@ page import="java.util.function.Consumer"%><%@ page import="edu.stanford.epadd.util.OperationInfo"%><%@ page import="edu.stanford.muse.email.StaticStatusProvider"%>
<%@ page import="org.json.JSONArray" %><%@ page import="org.json.JSONObject"%><%@ page import="org.json.CDL"%><%@ page import="org.apache.commons.io.FileUtils"%><%@ page import="au.com.bytecode.opencsv.CSVWriter"%><%@ page import="java.util.*"%><%@ page import="edu.stanford.muse.ner.model.NEType"%><%@ page import="edu.stanford.muse.index.*"%><%@ page import="java.io.*"%><%@ page import="java.util.zip.GZIPOutputStream"%><%@ page import="java.util.zip.ZipOutputStream"%><%@ page import="java.util.zip.ZipEntry"%><%@ page import="edu.stanford.muse.util.EmailUtils"%><%@ page import="edu.stanford.muse.webapp.ModeConfig"%><%@ page import="edu.stanford.muse.AddressBookManager.AddressBook"%><%@ page import="edu.stanford.muse.email.StatusProvider"%><%@ page import="com.google.common.collect.Multimap"%><%@ page import="java.util.function.Consumer"%><%@ page import="edu.stanford.epadd.util.OperationInfo"%><%@ page import="edu.stanford.muse.email.StaticStatusProvider"%><%@ page import="edu.stanford.epadd.util.EmailConvert"%><%@ page import="com.weirdkid.emailchemy.api.Format"%><%@ page import="com.weirdkid.emailchemy.api.OutputFormat"%><%@ page import="com.weirdkid.emailchemy.api.Converter"%><%@ page import="edu.stanford.epadd.util.EmlCreator"%>
<%
//This api needs to be supported for both types of flows - long running with status bar and normal (without status bar). Theso two invocation types of this jsp will be identified
Expand Down Expand Up @@ -174,7 +174,7 @@ try{
new File(pathToDirectory).mkdir();//create the direcotry.
PrintWriter pw = null;
try {
List<Document> docs = archive.getDocsForExport(Archive.Export_Mode.EXPORT_PROCESSING_TO_DELIVERY);
List<Document> docs = archive.getDocsForExport(Archive.ExportMode.EXPORT_PROCESSING_TO_DELIVERY);
Set<Document> docset = new LinkedHashSet<>(docs);//convert to set to remove possible duplicates.
int i=1;
for(Document doc: docset){
Expand All @@ -197,49 +197,57 @@ try{
error="Error exporting confirmed correspondents";///
}
}else if(query.equals("to-mbox")){
}else if(query.equals("to-mbox") || query.equals("to-eml")){
String type=JSPHelper.getParam(params,"type");
Set<Document> docset = null;
String fnameprefix=null;
Archive.Export_Mode mode=null;
if(ModeConfig.isAppraisalMode())
mode = Archive.Export_Mode.EXPORT_APPRAISAL_TO_PROCESSING;
else if(ModeConfig.isProcessingMode())
mode =Archive.Export_Mode.EXPORT_PROCESSING_TO_DELIVERY;//to discovery is also same so no difference.
if(type.equals("all")){
docset=archive.getAllDocsAsSet();
fnameprefix="all-messages";
}else if(type.equals("non-restricted")){
docset = new LinkedHashSet<>(archive.getDocsForExport(Archive.Export_Mode.EXPORT_PROCESSING_TO_DELIVERY));
docset = new LinkedHashSet<>(archive.getDocsForExport(Archive.ExportMode.EXPORT_PROCESSING_TO_DELIVERY));
fnameprefix="non-restricted-messages";
}else if(type.equals("restricted")){
docset = new LinkedHashSet<>(archive.getDocsForExport(Archive.Export_Mode.EXPORT_PROCESSING_TO_DELIVERY));
docset = new LinkedHashSet<>(archive.getDocsForExport(Archive.ExportMode.EXPORT_PROCESSING_TO_DELIVERY));
Set<Document> alldocs = new LinkedHashSet<>(archive.getAllDocsAsSet());
alldocs.removeAll(docset);//now alldocs contain those messages which are not exported,i.e. are restricted.
docset=alldocs;
fnameprefix="restricted-messages";
}
String pathToFile = Archive.TEMP_SUBDIR + File.separator + fnameprefix+".mbox";
String pathToTmpDir = Archive.TEMP_SUBDIR;
String pathToMboxFile = Archive.TEMP_SUBDIR + File.separator + fnameprefix+".mbox";
Util.deleteAllFilesWithSuffix(Archive.TEMP_SUBDIR,"mbox",JSPHelper.log);
PrintWriter pw = null;
PrintWriter pw;
try {
pw = new PrintWriter(pathToFile, "UTF-8");
pw = new PrintWriter(pathToMboxFile, "UTF-8");
boolean stripQuoted = false;
for (Document ed: docset)
{
EmailUtils.printToMbox(archive, (EmailDocument) ed, pw,archive.getBlobStore(), stripQuoted);
}
pw.close();
//return it's URL to download
String contentURL = "serveTemp.jsp?archiveID="+ArchiveReaderWriter.getArchiveIDForArchive(archive)+"&file="+fnameprefix+".mbox" ;
String contentURL = "serveTemp.jsp?archiveID=" + ArchiveReaderWriter.getArchiveIDForArchive(archive) + "&file=";
if (query.equals("to-mbox"))
{
contentURL = contentURL + fnameprefix + ".mbox" ;
}
else
{
EmlCreator emlCreator = new EmlCreator(setStatusProvider);
String emlDir = emlCreator.convertToEml(pathToMboxFile, pathToTmpDir, docset.size());
String zipFileName = pathToTmpDir + File.separatorChar + fnameprefix + "_as_eml.zip";
Util.zipDirectory(emlDir, zipFileName);
Util.deleteDir(emlDir, JSPHelper.log);
contentURL = contentURL + fnameprefix + "_as_eml.zip";
}
downloadURL = /*appURL + "/" + */ contentURL;
} catch(Exception e){
//Util.print_exception ("Error exporting authorities", e, JSPHelper.log);
e.printStackTrace();
error="Error exporting confirmed correspondents";///
JSPHelper.log.error("Error downloading emails. query: " + query + ". Exception: " + e);
error = "Error exporting emails";
}
}//////////////////////////////////////Download unconfirmed correspondent file//////////////////////////////////////////////////////////////////////
Expand Down
2 changes: 1 addition & 1 deletion WebContent/ajax/importAccession.jsp
View file
Expand Up @@ -6,7 +6,7 @@
<%@page language="java" import="edu.stanford.muse.webapp.*"%>
<%@ page language="java" import="edu.stanford.muse.index.Archive"%>
<%@ page language="java" import="org.apache.commons.io.FileUtils"%>
<%@ page import="org.joda.time.DateTime"%><%@ page import="edu.stanford.muse.index.ArchiveReaderWriter"%><%@ page import="edu.stanford.muse.email.FetchStats"%><%@ page import="java.util.*"%><%@ page import="edu.stanford.muse.email.FolderInfo"%><%@ page import="edu.stanford.muse.util.Pair"%><%@ page import="org.apache.commons.lang.StringUtils"%>
<%@ page import="org.joda.time.DateTime"%><%@ page import="edu.stanford.muse.index.ArchiveReaderWriter"%><%@ page import="edu.stanford.muse.email.FetchStats"%><%@ page import="java.util.*"%><%@ page import="edu.stanford.muse.email.FolderInfo"%><%@ page import="edu.stanford.muse.util.Pair"%><%@ page import="org.apache.commons.lang.StringUtils"%><%@ page import="edu.stanford.epadd.util.EmailConvert"%>
<%
/* copies new accession into REPO_DIR and then loads it from there */
JSONObject result = new JSONObject();
Expand Down

0 comments on commit dac9273

Please sign in to comment.