Skip to content
This repository has been archived by the owner on Nov 9, 2017. It is now read-only.

Commit

Permalink
remove all file-system-based long-term raw document persistence, use …
Browse files Browse the repository at this point in the history 
…database persistence instead
  • Loading branch information
@davidmason
davidmason committed Sep 24, 2012
1 parent 4b58a00 commit 8dfc839
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 121 deletions.
63 changes: 52 additions & 11 deletions zanata-war/src/main/java/org/zanata/action/ProjectIterationFilesAction.java
View file
Expand Up @@ -26,6 +26,10 @@
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.Blob;
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
Expand All @@ -34,17 +38,22 @@

import javax.faces.context.FacesContext;

import org.hibernate.Hibernate;
import org.hibernate.validator.InvalidStateException;
import org.jboss.seam.ScopeType;
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Logger;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.jboss.seam.annotations.security.Restrict;
import org.jboss.seam.faces.FacesMessages;
import org.jboss.seam.international.StatusMessage.Severity;
import org.jboss.seam.log.Log;
import org.jboss.seam.security.management.JpaIdentityStore;
import org.jboss.seam.util.Hex;
import org.zanata.annotation.CachedMethodResult;
import org.zanata.annotation.CachedMethods;
import org.zanata.common.DocumentType;
import org.zanata.common.EntityStatus;
import org.zanata.common.LocaleId;
import org.zanata.common.MergeType;
Expand All @@ -59,6 +68,7 @@
import org.zanata.model.HIterationProject;
import org.zanata.model.HLocale;
import org.zanata.model.HProjectIteration;
import org.zanata.model.HRawDocument;
import org.zanata.rest.StringSet;
import org.zanata.rest.dto.extensions.ExtensionType;
import org.zanata.rest.dto.resource.Resource;
Expand Down Expand Up @@ -92,6 +102,9 @@ public class ProjectIterationFilesAction

private String localeId;

@Logger
private Log log;

@In
private ZanataIdentity identity;

Expand Down Expand Up @@ -292,15 +305,27 @@ else if (docId.contains("/"))
}

File tempFile = null;
byte[] md5hash;
try
{
tempFile = translationFileServiceImpl.persistToTempFile(documentFileUpload.getFileContents());
MessageDigest md = MessageDigest.getInstance("MD5");
InputStream fileContents = new DigestInputStream(documentFileUpload.getFileContents(), md);
tempFile = translationFileServiceImpl.persistToTempFile(fileContents);
md5hash = md.digest();
}
catch (ZanataServiceException e) {
log.error("Failed writing temp file for document {0}", e, documentFileUpload.getDocId());
FacesMessages.instance().add(Severity.ERROR, "Error saving uploaded document {0} to server.", fileName);
return;
}
catch (NoSuchAlgorithmException e)
{
log.error("MD5 hash algorithm not available", e);
FacesMessages.instance().add(Severity.ERROR, "Error generating hash for uploaded document {0}.", fileName);
return;
}

HDocument document = null;
try
{
Resource doc;
Expand All @@ -315,7 +340,7 @@ else if (docId.contains("/"))
doc.setLang( new LocaleId(documentFileUpload.getSourceLang()) );
Set<String> extensions = Collections.<String>emptySet();
// TODO Copy Trans values
documentServiceImpl.saveDocument(projectSlug, iterationSlug, doc, extensions, false);
document = documentServiceImpl.saveDocument(projectSlug, iterationSlug, doc, extensions, false);
showUploadSuccessMessage();
}
catch (SecurityException e)
Expand All @@ -326,18 +351,34 @@ else if (docId.contains("/"))
FacesMessages.instance().add(Severity.ERROR, "Invalid document format for {0}.", fileName);
}

try
{
translationFileServiceImpl.persistDocument(new FileInputStream(tempFile), projectSlug, iterationSlug, documentPath, fileName);
}
catch (FileNotFoundException e)
{
FacesMessages.instance().add(Severity.ERROR, "Error saving uploaded document {0} on server, download in original format may fail.", documentFileUpload.getFileName());

if (document == null) {
// error message for failed parse already added.
}
catch (ZanataServiceException e)
else
{
FacesMessages.instance().add(Severity.ERROR, "Error saving uploaded document {0} on server, download in original format may fail.", documentFileUpload.getFileName());
HRawDocument rawDocument = new HRawDocument();
rawDocument.setDocument(document);
rawDocument.setContentHash(new String(Hex.encodeHex(md5hash)));
rawDocument.setType(DocumentType.typeFor(translationFileServiceImpl.extractExtension(fileName)));
rawDocument.setUploadedBy(identity.getCredentials().getUsername());

FileInputStream tempFileStream = null;
try
{
tempFileStream = new FileInputStream(tempFile);
Blob fileContents = Hibernate.createBlob(tempFileStream, (int)tempFile.length());
rawDocument.setContent(fileContents);
documentDAO.addRawDocument(document, rawDocument);
documentDAO.flush();
}
catch (FileNotFoundException e)
{
log.error("Failed to open stream from temp source file", e);
FacesMessages.instance().add(Severity.ERROR, "Error saving uploaded document {0} on server, download in original format may fail.", documentFileUpload.getFileName());
}
}

translationFileServiceImpl.removeTempFile(tempFile);
}

Expand Down
71 changes: 60 additions & 11 deletions zanata-war/src/main/java/org/zanata/rest/service/FileService.java
View file
Expand Up @@ -27,6 +27,9 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.Blob;
import java.sql.SQLException;
import java.util.Collections;
Expand Down Expand Up @@ -57,6 +60,7 @@
import org.jboss.seam.annotations.Logger;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.log.Log;
import org.jboss.seam.util.Hex;
import org.zanata.adapter.FileFormatAdapter;
import org.zanata.adapter.po.PoWriter2;
import org.zanata.common.ContentState;
Expand Down Expand Up @@ -176,24 +180,53 @@ public Response uploadSourceFile( @PathParam("projectSlug") String projectSlug,
}

// persist bytes to file
// TODO generate hash during persist.
File tempFile = null;
if (uploadForm.getFirst())
{
// TODO if !getLast() add new document upload and first raw doc part
try
if (uploadForm.getLast())
{
tempFile = translationFileServiceImpl.persistToTempFile(uploadForm.getFileStream());
// single part, can go straight to temp file.
try
{
MessageDigest md = MessageDigest.getInstance("MD5");
InputStream fileContents = new DigestInputStream(uploadForm.getFileStream(), md);
tempFile = translationFileServiceImpl.persistToTempFile(fileContents);
String md5hash = new String(Hex.encodeHex(md.digest()));
tempFile = translationFileServiceImpl.persistToTempFile(fileContents);
if (!md5hash.equals(uploadForm.getHash()))
{
return Response.status(Status.CONFLICT)
.entity("MD5 hash " + uploadForm.getHash()
+ " sent with request does not match server-generated hash "
+ md5hash + ". Aborted upload operation.\n")
.build();
}
}
catch (ZanataServiceException e) {
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(e)
.build();
}
catch (NoSuchAlgorithmException e)
{
log.error("MD5 hash algorithm not available", e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(e)
.build();
}
}
catch (ZanataServiceException e) {
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(e)
else
{
// TODO new document upload and first raw doc part
return Response.status(Status.fromStatusCode(501))
.entity("Multiple part file upload is not yet implemented. Send entire file with first=true and last=true.\n")
.build();
}
}
else
{
// TODO append to existing temp file
// TODO add new DocumentUploadPart to DocumentUpload
// Return document upload id in response
return Response.status(Status.fromStatusCode(501))
.entity("Multiple part file upload is not yet implemented. Send entire file with first=true and last=true.\n")
.build();
Expand All @@ -206,9 +239,11 @@ public Response uploadSourceFile( @PathParam("projectSlug") String projectSlug,
.build();
}

// have entire file, proceed with parsing

// TODO reconstitute file from parts if required.
// TODO generate and check hash for reconstituted document.

// have entire file, proceed with parsing

if (isPotFile)
{
Expand Down Expand Up @@ -430,6 +465,8 @@ public Response uploadTranslationFile( @PathParam("projectSlug") String projectS
.build();
}

// TODO check and handle multiple part uploads

// TODO useful error messages for failed parsing
TranslationsResource transRes = translationFileServiceImpl.parseTranslationFile(uploadForm.getFileStream(),
uploadForm.getFileType(), locale);
Expand Down Expand Up @@ -603,7 +640,7 @@ else if ( (SUBSTITUTE_APPROVED.equals(fileType) || SUBSTITUTE_APPROVED_AND_FUZZY
TranslationsResource transRes =
(TranslationsResource) this.translatedDocResourceService.getTranslations(docId, new LocaleId(locale), extensions, true).getEntity();
// Filter to only provide translated targets. "Preview" downloads include fuzzy.
// Using new list is used as transRes list appears not to be a modifiable implementation.
// New list is used as transRes list appears not to be a modifiable implementation.
Map<String, TextFlowTarget> translations = new HashMap<String, TextFlowTarget>();
boolean useFuzzy = SUBSTITUTE_APPROVED_AND_FUZZY.equals(fileType);
for (TextFlowTarget target : transRes.getTextFlowTargets())
Expand All @@ -614,7 +651,19 @@ else if ( (SUBSTITUTE_APPROVED.equals(fileType) || SUBSTITUTE_APPROVED_AND_FUZZY
}
}

URI uri = translationFileServiceImpl.getDocumentURI(projectSlug, iterationSlug, document.getPath(), document.getName());
HDocument hDocument = documentDAO.getByProjectIterationAndDocId(projectSlug, iterationSlug, docId);
URI uri;
try
{
File tempFile = translationFileServiceImpl.persistToTempFile(hDocument.getRawDocument().getContent().getBinaryStream());
uri = tempFile.toURI();
}
catch (SQLException e)
{
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(e)
.build();
}
StreamingOutput output = new FormatAdapterStreamingOutput(uri, translations, locale, translationFileServiceImpl.getAdapterFor(docId));
response = Response.ok()
.header("Content-Disposition", "attachment; filename=\"" + document.getName() + "\"")
Expand Down
27 changes: 0 additions & 27 deletions zanata-war/src/main/java/org/zanata/service/TranslationFileService.java
View file
Expand Up @@ -102,8 +102,6 @@ public interface TranslationFileService

Set<String> getSupportedExtensions();

public URI getDocumentURI(String projectSlug, String iterationSlug, String docPath, String docName);

FileFormatAdapter getAdapterFor(String fileNameOrExtension);

/**
Expand All @@ -125,35 +123,10 @@ public interface TranslationFileService
*/
void removeTempFile(File tempFile);

/**
* Add a document to persistent storage, overwriting any equivalent existing document.
*
* A document is equivalent if it has the same project and version slug, docPath and docName.
*
* @param docContents contents of the document, will be in a closed state when this method completes.
* @param projectSlug
* @param iterationSlug
* @param docPath
* @param docNameAndExt
* @throws ZanataServiceException if the document cannot be persisted
*/
void persistDocument(InputStream docContents, String projectSlug, String iterationSlug, String docPath, String docName) throws ZanataServiceException;

boolean hasPersistedDocument(String projectSlug, String iterationSlug, String docPath, String docName);

String getFileExtension(String projectSlug, String iterationSlug, String docPath, String docName);

/**
* Stream the contents of a document from persistence.
*
* @param projectSlug
* @param iterationSlug
* @param docPath
* @param docNameAndExt
* @return the document as an InputStream, or null if no document is in persistence with the given credentials.
*/
InputStream streamDocument(String projectSlug, String iterationSlug, String docPath, String docName);

/**
*
* @param fileNameOrExtension
Expand Down

0 comments on commit 8dfc839

Please sign in to comment.