-
Notifications
You must be signed in to change notification settings - Fork 464
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
6 changed files
with
214 additions
and
93 deletions.
There are no files selected for viewing
67 changes: 67 additions & 0 deletions
67
dotCMS/src/main/java/com/dotcms/storage/BinaryBlockFileJoinerImpl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,67 @@ | ||
package com.dotcms.storage; | ||
|
||
import com.dotcms.util.FileJoiner; | ||
import com.dotcms.util.FileJoinerImpl; | ||
|
||
import java.io.File; | ||
import java.io.IOException; | ||
import java.io.InputStream; | ||
import java.io.OutputStream; | ||
import java.util.function.Function; | ||
|
||
/** | ||
* Instead of joining into a file will join into a bytes stream | ||
* The joining in memory will be limited by a given size | ||
* @author jsanca | ||
*/ | ||
public class BinaryBlockFileJoinerImpl extends FileJoinerImpl implements FileJoiner { | ||
|
||
private byte [] buffer; | ||
private int bufferIndex = 0; | ||
|
||
public BinaryBlockFileJoinerImpl(final File file, final int maxSize) throws IOException { | ||
super(file); | ||
this.buffer = new byte[maxSize]; | ||
} | ||
|
||
public BinaryBlockFileJoinerImpl(OutputStream outputStream, final int maxSize) { | ||
super(outputStream); | ||
this.buffer = new byte[maxSize]; | ||
} | ||
|
||
public BinaryBlockFileJoinerImpl(OutputStream outputStream, int bufferSize, final int maxSize) { | ||
super(outputStream, bufferSize); | ||
this.buffer = new byte[maxSize]; | ||
} | ||
|
||
public BinaryBlockFileJoinerImpl(OutputStream outputStream, Function<File, InputStream> inputStreamCreatorFunction, int bufferSize, final int maxSize) { | ||
super(outputStream, inputStreamCreatorFunction, bufferSize); | ||
this.buffer = new byte[maxSize]; | ||
} | ||
|
||
@Override | ||
public void join(final byte[] bytes, final int offset, final int length) { | ||
|
||
int nextSize = this.bufferIndex + length; | ||
|
||
if (nextSize >= bytes.length) { | ||
if (null != this.buffer) { | ||
|
||
super.join(this.buffer, 0, this.buffer.length); | ||
this.buffer = null; | ||
} | ||
|
||
super.join(bytes, offset, length); | ||
} else { | ||
|
||
for (int i = offset; i < length; ++i) { | ||
|
||
this.buffer[this.bufferIndex++] = bytes[i]; | ||
} | ||
} | ||
} | ||
|
||
public byte [] getBytes() { | ||
return this.buffer; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
13 changes: 13 additions & 0 deletions
13
dotCMS/src/main/java/com/dotcms/storage/repository/FileRepositoryManager.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
package com.dotcms.storage.repository; | ||
|
||
import java.io.File; | ||
|
||
/** | ||
* Abstract a file repository, it could follow diff implementation strategies | ||
*/ | ||
public interface FileRepositoryManager { | ||
|
||
boolean exists (String fileName); | ||
|
||
File getOrCreateFile (String fileName); | ||
} |
15 changes: 15 additions & 0 deletions
15
dotCMS/src/main/java/com/dotcms/storage/repository/TempFileRepositoryManager.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
package com.dotcms.storage.repository; | ||
|
||
import java.io.File; | ||
|
||
public class TempFileRepositoryManager implements FileRepositoryManager { | ||
@Override | ||
public boolean exists(String fileName) { | ||
return false; | ||
} | ||
|
||
@Override | ||
public File getOrCreateFile(String fileName) { | ||
return null; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,100 +1,26 @@ | ||
package com.dotcms.util; | ||
|
||
import com.dotmarketing.exception.DotRuntimeException; | ||
import com.dotmarketing.util.Config; | ||
import com.liferay.util.FileUtil; | ||
import io.vavr.control.Try; | ||
|
||
import java.io.Closeable; | ||
import java.io.File; | ||
import java.io.Flushable; | ||
import java.io.IOException; | ||
import java.io.InputStream; | ||
import java.io.OutputStream; | ||
import java.util.function.Function; | ||
|
||
/** | ||
* A File Joiner is a class take chucks of a file (they could be an actual file [previously splitted]) or a collection of byte chucks. | ||
* or both. | ||
* In case you are gonna use the file instead of the raw bytes, you should need to define a buffer previous by passing the chuck file of the | ||
* diff pieces (by default is 2mb) witch is the same default of the {@link FileSplitter} and {@link FileByteSplitter} | ||
* Abstraction of the FileJoiner, allows to join files or binary block into one single file (or similar) | ||
* @author jsanca | ||
*/ | ||
public class FileJoiner implements AutoCloseable, Flushable, Closeable { | ||
|
||
private final OutputStream outputStream; | ||
private final byte [] buffer; | ||
private final Function<File, InputStream> inputStreamCreatorFunction; | ||
public interface FileJoiner extends AutoCloseable, Flushable, Closeable { | ||
|
||
/** | ||
* Defaut size for the chucks, 2mb | ||
* Join the whole file | ||
* @param file | ||
*/ | ||
private static final int CHUNK_SIZE = Config.getIntProperty("FILE_SPLITTER_CHUNK_SIZE", 2097152); // 2mb | ||
|
||
|
||
private static InputStream createCompressorOutputStream (final File file) { | ||
|
||
final String compressor = Config.getStringProperty("FILE_SPLITTER_COMPRESSOR", "gzip"); | ||
return Try.of(()->FileUtil.createInputStream(file.toPath(), compressor)).getOrElseThrow(DotRuntimeException::new); | ||
} | ||
|
||
public FileJoiner(final File file) throws IOException { | ||
this (FileUtil.createOutputStream(file), FileJoiner::createCompressorOutputStream, CHUNK_SIZE); | ||
} | ||
|
||
public FileJoiner(final OutputStream outputStream) { | ||
this (outputStream, FileJoiner::createCompressorOutputStream, CHUNK_SIZE); | ||
} | ||
|
||
public FileJoiner(final OutputStream outputStream, final int bufferSize) { | ||
this (outputStream, FileJoiner::createCompressorOutputStream, bufferSize); | ||
} | ||
|
||
public FileJoiner(final OutputStream outputStream, | ||
final Function<File, InputStream> inputStreamCreatorFunction, | ||
final int bufferSize) { | ||
|
||
this.outputStream = outputStream; | ||
this.inputStreamCreatorFunction = inputStreamCreatorFunction; | ||
this.buffer = bufferSize>0?new byte[bufferSize]: | ||
new byte[1]; // probably won't need any buffer to join. | ||
} | ||
|
||
public void join (final File file) { | ||
|
||
try (InputStream inputStream = this.inputStreamCreatorFunction.apply(file)) { | ||
void join(File file); | ||
|
||
int bytesRead = 0; | ||
while ((bytesRead = inputStream.read(this.buffer)) > 0) { | ||
|
||
this.join(this.buffer, 0, bytesRead); | ||
} | ||
} catch (IOException e) { | ||
|
||
throw new DotRuntimeException(e); | ||
} | ||
} | ||
|
||
public void join (final byte[] bytes, final int offset, final int length) { | ||
|
||
try { | ||
|
||
this.outputStream.write(bytes, offset, length); | ||
} catch (IOException e) { | ||
|
||
throw new DotRuntimeException(e); | ||
} | ||
} | ||
|
||
@Override | ||
public void close() throws IOException { | ||
|
||
CloseUtils.closeQuietly(this.outputStream); | ||
} | ||
|
||
@Override | ||
public void flush() throws IOException { | ||
|
||
this.outputStream.flush(); | ||
} | ||
/** | ||
* Join a binary block | ||
* @param bytes | ||
* @param offset | ||
* @param length | ||
*/ | ||
void join(byte[] bytes, int offset, int length); | ||
} |
102 changes: 102 additions & 0 deletions
102
dotCMS/src/main/java/com/dotcms/util/FileJoinerImpl.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
package com.dotcms.util; | ||
|
||
import com.dotmarketing.exception.DotRuntimeException; | ||
import com.dotmarketing.util.Config; | ||
import com.liferay.util.FileUtil; | ||
import io.vavr.control.Try; | ||
|
||
import java.io.Closeable; | ||
import java.io.File; | ||
import java.io.Flushable; | ||
import java.io.IOException; | ||
import java.io.InputStream; | ||
import java.io.OutputStream; | ||
import java.util.function.Function; | ||
|
||
/** | ||
* A File Joiner is a class take chucks of a file (they could be an actual file [previously splitted]) or a collection of byte chucks. | ||
* or both. | ||
* In case you are gonna use the file instead of the raw bytes, you should need to define a buffer previous by passing the chuck file of the | ||
* diff pieces (by default is 2mb) witch is the same default of the {@link FileSplitter} and {@link FileByteSplitter} | ||
* @author jsanca | ||
*/ | ||
public class FileJoinerImpl implements FileJoiner { | ||
|
||
private final OutputStream outputStream; | ||
private final byte [] buffer; | ||
private final Function<File, InputStream> inputStreamCreatorFunction; | ||
|
||
/** | ||
* Defaut size for the chucks, 2mb | ||
*/ | ||
private static final int CHUNK_SIZE = Config.getIntProperty("FILE_SPLITTER_CHUNK_SIZE", 2097152); // 2mb | ||
|
||
|
||
private static InputStream createCompressorOutputStream (final File file) { | ||
|
||
final String compressor = Config.getStringProperty("FILE_SPLITTER_COMPRESSOR", "gzip"); | ||
return Try.of(()->FileUtil.createInputStream(file.toPath(), compressor)).getOrElseThrow(DotRuntimeException::new); | ||
} | ||
|
||
public FileJoinerImpl(final File file) throws IOException { | ||
this (FileUtil.createOutputStream(file), FileJoinerImpl::createCompressorOutputStream, CHUNK_SIZE); | ||
} | ||
|
||
public FileJoinerImpl(final OutputStream outputStream) { | ||
this (outputStream, FileJoinerImpl::createCompressorOutputStream, CHUNK_SIZE); | ||
} | ||
|
||
public FileJoinerImpl(final OutputStream outputStream, final int bufferSize) { | ||
this (outputStream, FileJoinerImpl::createCompressorOutputStream, bufferSize); | ||
} | ||
|
||
public FileJoinerImpl(final OutputStream outputStream, | ||
final Function<File, InputStream> inputStreamCreatorFunction, | ||
final int bufferSize) { | ||
|
||
this.outputStream = outputStream; | ||
this.inputStreamCreatorFunction = inputStreamCreatorFunction; | ||
this.buffer = bufferSize>0?new byte[bufferSize]: | ||
new byte[1]; // probably won't need any buffer to join. | ||
} | ||
|
||
@Override | ||
public void join(final File file) { | ||
|
||
try (InputStream inputStream = this.inputStreamCreatorFunction.apply(file)) { | ||
|
||
int bytesRead = 0; | ||
while ((bytesRead = inputStream.read(this.buffer)) > 0) { | ||
|
||
this.join(this.buffer, 0, bytesRead); | ||
} | ||
} catch (IOException e) { | ||
|
||
throw new DotRuntimeException(e); | ||
} | ||
} | ||
|
||
@Override | ||
public void join(final byte[] bytes, final int offset, final int length) { | ||
|
||
try { | ||
|
||
this.outputStream.write(bytes, offset, length); | ||
} catch (IOException e) { | ||
|
||
throw new DotRuntimeException(e); | ||
} | ||
} | ||
|
||
@Override | ||
public void close() throws IOException { | ||
|
||
CloseUtils.closeQuietly(this.outputStream); | ||
} | ||
|
||
@Override | ||
public void flush() throws IOException { | ||
|
||
this.outputStream.flush(); | ||
} | ||
} |