From ca80ef8150fc95be2b9b8855b901570eb44d5aa6 Mon Sep 17 00:00:00 2001 From: David Mason Date: Wed, 24 Jul 2013 17:47:30 +1000 Subject: [PATCH] rhbz980658 migrate raw documents to file system --- .../MigrateRawDocumentsToFileSystem.java | 275 ++++++++++++++++++ .../db/changelogs/db.changelog-3.1.xml | 4 + 2 files changed, 279 insertions(+) create mode 100644 zanata-war/src/main/java/org/zanata/liquibase/custom/MigrateRawDocumentsToFileSystem.java diff --git a/zanata-war/src/main/java/org/zanata/liquibase/custom/MigrateRawDocumentsToFileSystem.java b/zanata-war/src/main/java/org/zanata/liquibase/custom/MigrateRawDocumentsToFileSystem.java new file mode 100644 index 0000000000..9539057366 --- /dev/null +++ b/zanata-war/src/main/java/org/zanata/liquibase/custom/MigrateRawDocumentsToFileSystem.java @@ -0,0 +1,275 @@ +/* + * Copyright 2013, Red Hat, Inc. and individual contributors + * as indicated by the @author tags. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.zanata.liquibase.custom; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.sql.Blob; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import javax.naming.InitialContext; +import javax.naming.NameNotFoundException; +import javax.naming.NamingException; + +import liquibase.change.custom.CustomTaskChange; +import liquibase.database.Database; +import liquibase.database.jvm.JdbcConnection; +import liquibase.exception.CustomChangeException; +import liquibase.exception.DatabaseException; +import liquibase.exception.SetupException; +import liquibase.exception.ValidationErrors; +import liquibase.resource.ResourceAccessor; + +import org.zanata.common.DocumentType; + +import com.google.common.io.Files; +import com.google.common.io.InputSupplier; + +public class MigrateRawDocumentsToFileSystem implements CustomTaskChange +{ + + private static final String BASE_PATH_JNDI_NAME = "java:global/zanata/files/document-storage"; + private static final String RAW_DOCUMENTS_SUBDIRECTORY = "documents"; + + private static final String CONTENTS_SQL = "select contentLocation, content from HRawDocumentContent"; + //@formatter:off + private static final String ID_TYPE_SQL = "select d.documentId, rd.type" + + " from HRawDocument rd, HDocument_RawDocument d" + + " where d.rawDocumentId = rd.id and rd.contentLocation = ?"; + //@formatter:on + private static final String UPDATE_LOCATION_SQL = "update HRawDocument set contentLocation = ? where contentLocation = ?"; + private static final String DELETE_OLD_CONTENT_SQL = "delete from HRawDocumentContent where contentLocation = ?"; + + private File docsDirectory; + + private int docsCount; + private int successCount; + private int problemsCount; + + private PreparedStatement contentsStatement; + private PreparedStatement idAndTypeStatement; + private PreparedStatement updateLocationStatement; + private PreparedStatement deleteOldContentStatement; + + @Override + public String getConfirmationMessage() + { + return "Raw documents migrated from database to file system under path " + + docsDirectory.getAbsolutePath() + ". Total documents: " + docsCount + + ", success: " + successCount + ", problems: " + problemsCount; + } + + @Override + public void setUp() throws SetupException + { + String basePath = getPathFromJndi(); + docsDirectory = new File(basePath, RAW_DOCUMENTS_SUBDIRECTORY); + try + { + docsDirectory.mkdirs(); + } + catch (SecurityException e) + { + throw new SetupException(e); + } + docsCount = 0; + successCount = 0; + problemsCount = 0; + } + + private String getPathFromJndi() throws SetupException + { + String failureMessage = "Unable to migrate documents to file system. Document storage must " + + "be properly configured in jndi under name \"" + BASE_PATH_JNDI_NAME + "\""; + InitialContext initContext = null; + String basePath; + try + { + initContext = new InitialContext(); + basePath = (String) initContext.lookup(BASE_PATH_JNDI_NAME); + + if (basePath == null) + { + throw new SetupException(failureMessage); + } + } + catch (NameNotFoundException e) + { + throw new SetupException(failureMessage, e); + } + catch (NamingException e) + { + throw new SetupException(failureMessage, e); + } + finally + { + if (initContext != null) + { + try + { + initContext.close(); + } + catch (NamingException e) + { + e.printStackTrace(); + } + } + } + return basePath; + } + + @Override + public void setFileOpener(ResourceAccessor resourceAccessor) + { + } + + @Override + public ValidationErrors validate(Database database) + { + return new ValidationErrors(); + } + + @Override + public void execute(Database database) throws CustomChangeException + { + JdbcConnection conn = (JdbcConnection) database.getConnection(); + try + { + prepareStatements(conn); + migrateAllRawContents(); + } + catch (Exception e) + { + throw new CustomChangeException(e); + } + } + + private void prepareStatements(JdbcConnection conn) throws DatabaseException + { + contentsStatement = conn.prepareStatement(CONTENTS_SQL); + idAndTypeStatement = conn.prepareStatement(ID_TYPE_SQL); + updateLocationStatement = conn.prepareStatement(UPDATE_LOCATION_SQL); + deleteOldContentStatement = conn.prepareStatement(DELETE_OLD_CONTENT_SQL); + } + + private void migrateAllRawContents() throws Exception + { + ResultSet contentsResult = contentsStatement.executeQuery(); + while (contentsResult.next()) + { + try + { + docsCount++; + String oldLocation = contentsResult.getString("contentLocation"); + Blob content = contentsResult.getBlob("content"); + migrateRawContentFromLocation(content, oldLocation); + successCount++; + } + catch (Exception e) + { + problemsCount++; + throw e; + } + } + } + + private void migrateRawContentFromLocation(Blob content, String oldLocation) throws SQLException, IOException + { + idAndTypeStatement.setString(1, oldLocation); + ResultSet idAndTypeResult = idAndTypeStatement.executeQuery(); + if (idAndTypeResult.next()) + { + String fileName = fileNameFromResults(idAndTypeResult); + writeBlobToFile(content, fileName); + changeLocationFromOldToNew(oldLocation, fileName); + deleteOldContent(oldLocation); + } + else + { + throw new RuntimeException("Raw document content with no matching raw document, " + + "HRawDocumentContent.contentLocation = " + oldLocation); + } + } + + private static String fileNameFromResults(ResultSet idAndTypeResult) throws SQLException + { + long docId = idAndTypeResult.getLong("documentId"); + String type = idAndTypeResult.getString("type"); + return fileNameFromIdAndType(docId, type); + } + + private static String fileNameFromIdAndType(Long docId, String type) + { + String extension = DocumentType.valueOf(type).getExtension(); + return docId.toString() + "." + extension; + } + + private void writeBlobToFile(Blob content, String fileName) throws SQLException, IOException + { + File outputFile = createFileInConfiguredDirectory(fileName); + writeStreamToFile(content.getBinaryStream(), outputFile); + // releasing blob resources may not be necessary, but makes me feel better and shouldn't cause problems + content.free(); + content = null; + } + + private File createFileInConfiguredDirectory(String fileName) + { + return new File(docsDirectory, fileName); + } + + private void writeStreamToFile(final InputStream stream, File file) throws IOException + { + InputSupplier input = new InputSupplier() + { + public InputStream getInput() throws IOException + { + return stream; + } + }; + Files.copy(input, file); + } + + private void changeLocationFromOldToNew(String oldLocation, String newLocation) throws SQLException + { + updateLocationStatement.setString(1, newLocation); + updateLocationStatement.setString(2, oldLocation); + int updatedRows = updateLocationStatement.executeUpdate(); + if (updatedRows != 1) + { + throw new RuntimeException("Tried to update contentLocation for 1 HRawDocument, but updated " + updatedRows); + } + } + + private void deleteOldContent(String oldLocation) throws SQLException + { + deleteOldContentStatement.setString(1, oldLocation); + int deletedRows = deleteOldContentStatement.executeUpdate(); + if (deletedRows != 1) + { + throw new RuntimeException("Tried to delete 1 row from HRawDocumentContent, but deleted " + deletedRows); + } + } + +} diff --git a/zanata-war/src/main/resources/db/changelogs/db.changelog-3.1.xml b/zanata-war/src/main/resources/db/changelogs/db.changelog-3.1.xml index 0557e885a4..26b35f0733 100644 --- a/zanata-war/src/main/resources/db/changelogs/db.changelog-3.1.xml +++ b/zanata-war/src/main/resources/db/changelogs/db.changelog-3.1.xml @@ -107,5 +107,9 @@ + + Migrate existing raw document contents from database to file system + + \ No newline at end of file