Skip to content

Commit

Permalink
Merge branch 'rutebanken_google_cloud_storage' into rutebanken_develop
Browse files Browse the repository at this point in the history
# Conflicts:
#	mobi.chouette.common/src/main/java/mobi/chouette/common/PropertyNames.java
#	mobi.chouette.dao.iev/src/main/java/mobi/chouette/dao/iev/JobDAO.java
#	mobi.chouette.exchange/src/main/java/mobi/chouette/exchange/AbstractInputValidator.java
#	mobi.chouette.exchange/src/main/java/mobi/chouette/exchange/exporter/AbstractExporterCommand.java
#	mobi.chouette.model.iev/src/main/java/mobi/chouette/model/iev/Job.java
#	mobi.chouette.service/src/main/java/mobi/chouette/service/JobService.java
#	mobi.chouette.service/src/main/java/mobi/chouette/service/JobServiceManager.java
#	mobi.chouette.ws/src/main/java/mobi/chouette/ws/RestService.java
  • Loading branch information
erlendnils1 committed Oct 12, 2017
2 parents f0450e8 + be04fd9 commit 6b45b62
Show file tree
Hide file tree
Showing 19 changed files with 580 additions and 99 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,5 @@ public interface PropertyNames {

String REFERENTIAL_LOCK_MANAGER_IMPLEMENTATION = ".referential.lock.manager.impl";
String KUBERNETES_ENABLED = ".kubernetes.enabled";
String FILE_STORE_IMPLEMENTATION = ".file.store.impl";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package mobi.chouette.common.file;

/**
* Exception thrown when file access fails.
*/
public class FileServiceException extends RuntimeException {

public FileServiceException() {
}

public FileServiceException(String message) {
super(message);
}

public FileServiceException(String message, Throwable cause) {
super(message, cause);
}

public FileServiceException(Throwable cause) {
super(cause);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package mobi.chouette.common.file;

import java.io.InputStream;
import java.nio.file.Path;

/**
* Access of permanent files.
*/
public interface FileStore {

InputStream getFileContent(Path filePath);

void writeFile(Path filePath, InputStream content);

boolean delete(Path filePath);

void deleteFolder(Path folder);

void createFolder(Path folder);

boolean exists(Path filePath);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package mobi.chouette.common.file;

import java.util.Set;

import javax.enterprise.context.spi.CreationalContext;
import javax.enterprise.inject.spi.Bean;
import javax.enterprise.inject.spi.CDI;

import lombok.extern.log4j.Log4j;
import mobi.chouette.common.ContenerChecker;
import mobi.chouette.common.PropertyNames;

@Log4j
public class FileStoreFactory {

private static FileStore FILE_STORE;

private static final Object LOCK = new Object();

public static final FileStore getFileStore() {
if (FILE_STORE == null) {

synchronized (LOCK) {
if (FILE_STORE == null) {
ContenerChecker contenerChecker = null;
try {
contenerChecker = CDI.current().select(ContenerChecker.class).get();
} catch (Exception e) {
log.warn("Failed to access CDI, using default fileStore impl");
}

if (contenerChecker != null) {
String implBeanName = System.getProperty(contenerChecker.getContext() + PropertyNames.FILE_STORE_IMPLEMENTATION);
if (implBeanName != null) {
Set<Bean<?>> beans = CDI.current().getBeanManager().getBeans(implBeanName);

if (beans.size() > 0) {
Bean<FileStore> bean = (Bean<FileStore>) beans.iterator().next();
CreationalContext<FileStore> ctx = CDI.current().getBeanManager().createCreationalContext(bean);
FILE_STORE = (FileStore)
CDI.current().getBeanManager().getReference(bean, FileStore.class, ctx);
} else {
throw new IllegalArgumentException("FileStore implementation with bean name: " + implBeanName + " not found");
}
} else {
log.warn("No FileStore implementation defined, using LocalFileStore as default");
}

}

if (FILE_STORE == null) {
FILE_STORE = new LocalFileStore();
}
}
}

}
return FILE_STORE;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package mobi.chouette.common.file;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;

import javax.ejb.Stateless;

import lombok.extern.log4j.Log4j;

import org.apache.commons.io.FileUtils;

import static mobi.chouette.common.file.LocalFileStore.BEAN_NAME;

/**
* Store permanent files in local file system.
*/
@Stateless(name = BEAN_NAME)
@Log4j
public class LocalFileStore implements FileStore {

public static final String BEAN_NAME = "LocalFileStore";

@Override
public InputStream getFileContent(Path filePath) {
try {
return new FileInputStream(filePath.toFile());
} catch (IOException ioE) {
throw new FileServiceException("Failed to read from file: " + ioE.getMessage(), ioE);
}
}


@Override
public void writeFile(Path filePath, InputStream content) {
try {
FileUtils.copyInputStreamToFile(content, filePath.toFile());
} catch (IOException ioE) {
throw new FileServiceException("Failed to write to file: " + ioE.getMessage(), ioE);
}
}

@Override
public boolean delete(Path filePath) {
File file = filePath.toFile();
if (file.exists()) {
return filePath.toFile().delete();
}
return false;
}

@Override
public void deleteFolder(Path folder) {
if (Files.exists(folder)) {
try {
FileUtils.deleteDirectory(folder.toFile());
} catch (Exception e) {
throw new FileServiceException("Failed to delete folder: " + folder, e);
}
}
}

@Override
public boolean exists(Path filePath) {
return Files.exists(filePath);
}

@Override
public void createFolder(Path folder) {
try {
if (!Files.exists(folder)) {
Files.createDirectories(folder);
}
} catch (IOException ioE) {
throw new FileServiceException("Failed to create folder: " + ioE.getMessage(), ioE);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package mobi.chouette.dao.iev;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

Expand Down Expand Up @@ -116,6 +115,21 @@ public List<Job> findByStatus(Job.STATUS status) {
return result;
}

public List<Job> findByStatusesAndUpdatedSince(List<Job.STATUS> statuses, Date since) {
List<Job> result;
CriteriaBuilder builder = em.getCriteriaBuilder();
CriteriaQuery<Job> criteria = builder.createQuery(type);
Root<Job> root = criteria.from(type);
Predicate statusPredicate = root.get(Job_.status).in(statuses);
// Created jobs are only in initialization phase, should not be sent
Predicate updatedSincePredicate = builder.greaterThanOrEqualTo(root.get(Job_.updated), since);
criteria.where(builder.and(statusPredicate, updatedSincePredicate));
criteria.orderBy(builder.asc(root.get(Job_.created)));
TypedQuery<Job> query = em.createQuery(criteria);
result = query.getResultList();
return result;
}

public List<Job> getNextJobs(){
Query query = em
.createQuery("from Job j where j.status in ( ?1 ) and j.referential not in (SELECT a.referential from Job a where a.status=?2) order by id");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,12 @@
import mobi.chouette.common.Context;
import mobi.chouette.common.JSONUtil;
import mobi.chouette.common.JobData;
import mobi.chouette.common.file.FileStoreFactory;
import mobi.chouette.common.file.FileStoreFactory;
import mobi.chouette.exchange.report.ActionReport;
import mobi.chouette.exchange.validation.parameters.ValidationParameters;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;

@Log4j
Expand Down Expand Up @@ -50,13 +53,15 @@ protected boolean checkFileExistenceInZip(String fileName, Path filePath, String
ZipFile zipFile = null;
File file = null;
try {
file = new File(filePath.toString());
file = File.createTempFile("archive", ".zip");
FileUtils.copyInputStreamToFile(FileStoreFactory.getFileStore().getFileContent(filePath), file);

zipFile = new ZipFile(file);
if (isEmpty(zipFile)){
isZipFileValid = true;
} else {
for (Enumeration<? extends ZipEntry> e = zipFile.entries();
e.hasMoreElements();) {
e.hasMoreElements(); ) {
ZipEntry ze = e.nextElement();
String name = ze.getName();
if (name.endsWith("." + format) && !name.contains("metadata")) {
Expand All @@ -67,7 +72,11 @@ protected boolean checkFileExistenceInZip(String fileName, Path filePath, String
}
} catch (IOException e) {
log.error("Erreur ouverture fichier zip " + fileName);
}
} finally {
if (file != null) {
file.delete();
}
}
}

return isZipFileValid;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package mobi.chouette.exchange;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Path;
Expand All @@ -15,6 +17,7 @@
import mobi.chouette.common.JobData;
import mobi.chouette.common.chain.Command;
import mobi.chouette.common.chain.CommandFactory;
import mobi.chouette.common.file.FileStoreFactory;
import mobi.chouette.exchange.parameters.AbstractParameter;
import mobi.chouette.exchange.report.ProgressionReport;
import mobi.chouette.exchange.report.Report;
Expand Down Expand Up @@ -81,9 +84,11 @@ public void saveReport(Context context, boolean force) {
Path path = Paths.get(jobData.getPathName(), REPORT_FILE);
// pseudo pretty print
try {
PrintStream stream = new PrintStream(path.toFile(), "UTF-8");
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
PrintStream stream = new PrintStream(outputStream, false, "UTF-8");
report.print(stream);
stream.close();
FileStoreFactory.getFileStore().writeFile(path, new ByteArrayInputStream(outputStream.toByteArray()));
} catch (Exception e) {
log.error("failed to save report", e);
}
Expand All @@ -92,6 +97,8 @@ public void saveReport(Context context, boolean force) {

}



/**
* @param context
*/
Expand All @@ -110,10 +117,13 @@ public void saveMainValidationReport(Context context, boolean force) {
JobData jobData = (JobData) context.get(JOB_DATA);
Path path = Paths.get(jobData.getPathName(), VALIDATION_FILE);

ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try {
PrintStream stream = new PrintStream(path.toFile(), "UTF-8");
PrintStream stream = new PrintStream(outputStream, false, "UTF-8");
report.print(stream);
stream.close();
FileStoreFactory.getFileStore().writeFile(path, new ByteArrayInputStream(outputStream.toByteArray()));

} catch (Exception e) {
log.error("failed to save validation report", e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,24 @@ public boolean process(Context context, ProcessingCommands commands, Progression
AbstractExportParameter parameters = (AbstractExportParameter) context.get(CONFIGURATION);
ActionReporter reporter = ActionReporter.Factory.getInstance();

try {
// initialisation
List<? extends Command> preProcessingCommands = commands.getPreProcessingCommands(context, true);
progression.initialize(context, preProcessingCommands.size() + (mode.equals(Mode.line) ? 1 : 0));
for (Command exportCommand : preProcessingCommands) {
result = exportCommand.execute(context);
if (!result) {
reporter.setActionError(context, ActionReporter.ERROR_CODE.NO_DATA_FOUND, "no data selected");
progression.execute(context);
return ERROR;
}
progression.execute(context);
}
// initialisation
JobData jobData = (JobData) context.get(JOB_DATA);
String path = jobData.getPathName();
File output = new File(path, OUTPUT);
if (!output.exists())
Files.createDirectories(output.toPath());

List<? extends Command> preProcessingCommands = commands.getPreProcessingCommands(context, true);
progression.initialize(context, preProcessingCommands.size() + (mode.equals(Mode.line) ? 1 : 0));
for (Command exportCommand : preProcessingCommands) {
result = exportCommand.execute(context);
if (!result) {
reporter.setActionError(context, ActionReporter.ERROR_CODE.NO_DATA_FOUND, "no data selected");
progression.execute(context);
return ERROR;
}
progression.execute(context);
}

if (mode.equals(Mode.line)) {
String type = parameters.getReferencesType();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
import mobi.chouette.common.JobData;
import mobi.chouette.common.chain.Command;
import mobi.chouette.common.chain.CommandFactory;

import org.apache.commons.io.FileUtils;
import mobi.chouette.common.file.FileStoreFactory;

import com.jamonapi.Monitor;
import com.jamonapi.MonitorFactory;
import org.apache.commons.io.FileUtils;

@Log4j
public class CompressCommand implements Command, Constant {
Expand All @@ -42,6 +42,9 @@ public boolean execute(Context context) throws Exception {
File outputFile = filename.toFile();
if (outputFile.exists()) outputFile.delete();
FileUtil.compress(target.toString(), filename.toString());
// Store file in permanent storage
FileStoreFactory.getFileStore().writeFile(filename, FileUtils.openInputStream(filename.toFile()));

result = SUCCESS;
try {
FileUtils.deleteDirectory(target.toFile());
Expand Down
Loading

0 comments on commit 6b45b62

Please sign in to comment.