Skip to content

Memory Improvements in Export/Ingest #11417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ofahimIQSS merged 4 commits into from
Apr 22, 2025
Merged

Memory Improvements in Export/Ingest #11417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9114462
potential memory leaks
qqmyers Apr 9, 2025
74b8813
incremental reads
qqmyers Apr 9, 2025
248be67
release note
qqmyers Apr 17, 2025
5f9abe8
Merge remote-tracking branch 'IQSS/develop' into Perf1
qqmyers Apr 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions doc/release-notes/11417-export-improvements.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Export Improvements

Memory usage has been reduced and potential memory leaks closed in the metadata exporters
314 changes: 150 additions & 164 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/TabularSubsetGenerator.java
View file
Open in desktop

Large diffs are not rendered by default.

16 changes: 15 additions & 1 deletion src/main/java/edu/harvard/iq/dataverse/export/DDIExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
import jakarta.json.JsonObject;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;

import org.apache.solr.common.util.IOUtils;

import javax.xml.stream.XMLOutputFactory;

/**
Expand Down Expand Up @@ -44,14 +47,25 @@ public String getDisplayName(Locale locale) {

@Override
public void exportDataset(ExportDataProvider dataProvider, OutputStream outputStream) throws ExportException {
XMLStreamWriter xmlw = null;
//XMLStreamWriter is not auto-closable - can't use try-with-resources here
try {
XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
xmlw.writeStartDocument();
xmlw.flush();
DdiExportUtil.datasetJson2ddi(dataProvider.getDatasetJson(), dataProvider.getDatasetFileDetails(),
outputStream);
} catch (XMLStreamException xse) {
throw new ExportException("Caught XMLStreamException performing DDI export", xse);
} finally {
if (xmlw != null) {
try {
xmlw.close();
} catch (XMLStreamException e) {
// Log this exception, but don't rethrow as it's not the primary issue
e.printStackTrace();
}
}
}
}

Expand Down
81 changes: 53 additions & 28 deletions src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,19 +117,31 @@ private static String dto2ddi(DatasetDTO datasetDto) throws XMLStreamException {
}

private static void dtoddi(DatasetDTO datasetDto, OutputStream outputStream) throws XMLStreamException {
XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
xmlw.writeStartElement("codeBook");
xmlw.writeDefaultNamespace("ddi:codebook:2_5");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION);
xmlw.writeAttribute("version", DDIExporter.DEFAULT_XML_VERSION);
if(DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) {
xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage());
}
createStdyDscr(xmlw, datasetDto);
createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles());
xmlw.writeEndElement(); // codeBook
xmlw.flush();
XMLStreamWriter xmlw = null;
try {
xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
xmlw.writeStartElement("codeBook");
xmlw.writeDefaultNamespace("ddi:codebook:2_5");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION);
xmlw.writeAttribute("version", DDIExporter.DEFAULT_XML_VERSION);
if (DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) {
xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage());
}
createStdyDscr(xmlw, datasetDto);
createOtherMats(xmlw, datasetDto.getDatasetVersion().getFiles());
xmlw.writeEndElement(); // codeBook
xmlw.flush();
} finally {
if (xmlw != null) {
try {
xmlw.close();
} catch (XMLStreamException e) {
// Log this exception, but don't rethrow as it's in finally block
logger.log(Level.WARNING, "Error closing XMLStreamWriter", e);
}
}
}
}


Expand All @@ -139,21 +151,34 @@ public static void datasetJson2ddi(JsonObject datasetDtoAsJson, JsonArray fileDe
Gson gson = new Gson();
DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson.toString(), DatasetDTO.class);

XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
xmlw.writeStartElement("codeBook");
xmlw.writeDefaultNamespace("ddi:codebook:2_5");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION);
xmlw.writeAttribute("version", DDIExporter.DEFAULT_XML_VERSION);
if(DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) {
xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage());
}
createStdyDscr(xmlw, datasetDto);
createFileDscr(xmlw, fileDetails);
createDataDscr(xmlw, fileDetails);
createOtherMatsFromFileMetadatas(xmlw, fileDetails);
xmlw.writeEndElement(); // codeBook
xmlw.flush();
XMLStreamWriter xmlw = null;
try {
xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);

xmlw.writeStartElement("codeBook");
xmlw.writeDefaultNamespace("ddi:codebook:2_5");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xsi:schemaLocation", DDIExporter.DEFAULT_XML_NAMESPACE + " " + DDIExporter.DEFAULT_XML_SCHEMALOCATION);
xmlw.writeAttribute("version", DDIExporter.DEFAULT_XML_VERSION);
if (DvObjectContainer.isMetadataLanguageSet(datasetDto.getMetadataLanguage())) {
xmlw.writeAttribute("xml:lang", datasetDto.getMetadataLanguage());
}
createStdyDscr(xmlw, datasetDto);
createFileDscr(xmlw, fileDetails);
createDataDscr(xmlw, fileDetails);
createOtherMatsFromFileMetadatas(xmlw, fileDetails);
xmlw.writeEndElement(); // codeBook
xmlw.flush();
} finally {
if (xmlw != null) {
try {
xmlw.close();
} catch (XMLStreamException e) {
// Log this exception, but don't rethrow as it's in finally block
logger.log(Level.WARNING, "Error closing XMLStreamWriter", e);
}
}
}
}

/**
Expand Down
56 changes: 34 additions & 22 deletions src/main/java/edu/harvard/iq/dataverse/export/dublincore/DublinCoreExportUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import jakarta.json.JsonObject;
import javax.xml.stream.XMLOutputFactory;
Expand Down Expand Up @@ -68,29 +69,40 @@ public static void datasetJson2dublincore(JsonObject datasetDtoAsJson, OutputStr
}

private static void dto2dublincore(DatasetDTO datasetDto, OutputStream outputStream, String dcFlavor) throws XMLStreamException {
XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
if (DC_FLAVOR_DCTERMS.equals(dcFlavor)) {
xmlw.writeStartDocument();
xmlw.writeStartElement("metadata");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xmlns:dc", DC_XML_NAMESPACE);
xmlw.writeAttribute("xmlns:dcterms", DCTERMS_XML_NAMESPACE);
xmlw.writeDefaultNamespace(DCTERMS_DEFAULT_NAMESPACE);
//xmlw.writeAttribute("xsi:schemaLocation", DCTERMS_DEFAULT_NAMESPACE+" "+DCTERMS_XML_SCHEMALOCATION);
createDC(xmlw, datasetDto, dcFlavor);
} else if (DC_FLAVOR_OAI.equals(dcFlavor)) {
xmlw.writeStartElement("oai_dc:dc");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xmlns:oai_dc", OAI_DC_XML_NAMESPACE);
xmlw.writeAttribute("xmlns:dc", DC_XML_NAMESPACE);
xmlw.writeAttribute("xsi:schemaLocation", OAI_DC_XML_NAMESPACE+" "+OAI_DC_XML_SCHEMALOCATION);
//writeAttribute(xmlw, "version", DEFAULT_XML_VERSION);
createOAIDC(xmlw, datasetDto, dcFlavor);
XMLStreamWriter xmlw = null;
try {
xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
if (DC_FLAVOR_DCTERMS.equals(dcFlavor)) {
xmlw.writeStartDocument();
xmlw.writeStartElement("metadata");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xmlns:dc", DC_XML_NAMESPACE);
xmlw.writeAttribute("xmlns:dcterms", DCTERMS_XML_NAMESPACE);
xmlw.writeDefaultNamespace(DCTERMS_DEFAULT_NAMESPACE);
// xmlw.writeAttribute("xsi:schemaLocation", DCTERMS_DEFAULT_NAMESPACE+" "+DCTERMS_XML_SCHEMALOCATION);
createDC(xmlw, datasetDto, dcFlavor);
} else if (DC_FLAVOR_OAI.equals(dcFlavor)) {
xmlw.writeStartElement("oai_dc:dc");
xmlw.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
xmlw.writeAttribute("xmlns:oai_dc", OAI_DC_XML_NAMESPACE);
xmlw.writeAttribute("xmlns:dc", DC_XML_NAMESPACE);
xmlw.writeAttribute("xsi:schemaLocation", OAI_DC_XML_NAMESPACE + " " + OAI_DC_XML_SCHEMALOCATION);
// writeAttribute(xmlw, "version", DEFAULT_XML_VERSION);
createOAIDC(xmlw, datasetDto, dcFlavor);
}

xmlw.writeEndElement(); // <metadata> or <oai_dc:dc>
xmlw.flush();
} finally {
if (xmlw != null) {
try {
xmlw.close();
} catch (XMLStreamException e) {
// Log this exception, but don't rethrow as it's in finally block
logger.log(Level.WARNING, "Error closing XMLStreamWriter", e);
}
}
}


xmlw.writeEndElement(); // <metadata> or <oai_dc:dc>
xmlw.flush();
}

//UPDATED by rmo-cdsp:
Expand Down
29 changes: 21 additions & 8 deletions src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.OutputStream;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;

import jakarta.json.JsonObject;
Expand Down Expand Up @@ -49,19 +50,31 @@ public static void datasetJson2openaire(JsonObject datasetDtoAsJson, OutputStrea
}

private static void dto2openaire(DatasetDTO datasetDto, OutputStream outputStream) throws XMLStreamException {
XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);
XMLStreamWriter xmlw = null;
try {
xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream);

xmlw.writeStartElement("resource"); // <resource>
xmlw.writeStartElement("resource"); // <resource>

xmlw.writeAttribute("xmlns:xsi", XSI_NAMESPACE);
xmlw.writeAttribute("xmlns", RESOURCE_NAMESPACE);
xmlw.writeAttribute("xsi:schemaLocation", RESOURCE_NAMESPACE + " " + RESOURCE_SCHEMA_LOCATION);
xmlw.writeAttribute("xmlns:xsi", XSI_NAMESPACE);
xmlw.writeAttribute("xmlns", RESOURCE_NAMESPACE);
xmlw.writeAttribute("xsi:schemaLocation", RESOURCE_NAMESPACE + " " + RESOURCE_SCHEMA_LOCATION);

createOpenAire(xmlw, datasetDto);
createOpenAire(xmlw, datasetDto);

xmlw.writeEndElement(); // </resource>
xmlw.writeEndElement(); // </resource>

xmlw.flush();
xmlw.flush();
} finally {
if (xmlw != null) {
try {
xmlw.close();
} catch (XMLStreamException e) {
// Log this exception, but don't rethrow as it's in finally block
logger.log(Level.WARNING, "Error closing XMLStreamWriter", e);
}
}
}
}

private static void createOpenAire(XMLStreamWriter xmlw, DatasetDTO datasetDto) throws XMLStreamException {
Expand Down
Loading