Skip to content

Commit

Permalink
Merge 6aeae7b into cc2a1e9
Browse files Browse the repository at this point in the history
  • Loading branch information
@quinarygio
quinarygio committed Aug 3, 2018
2 parents cc2a1e9 + 6aeae7b commit 10424d9
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 66 deletions.
6 changes: 6 additions & 0 deletions histodb-server/src/main/java/eu/itesla_project/histodb/domain/Attribute.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,10 @@ public Attribute(String name) {
public String getName() {
return name;
}

@Override
public String toString() {
return "Attribute [name=" + name + "]";
}

}
115 changes: 74 additions & 41 deletions histodb-server/src/main/java/eu/itesla_project/histodb/web/HistoDataResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@
*/
package eu.itesla_project.histodb.web;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Locale;
import java.util.zip.GZIPOutputStream;
import javax.inject.Inject;

import org.mapdb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
Expand All @@ -27,6 +29,8 @@
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.WebRequest;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;

import eu.itesla_project.histodb.QueryParams;
import eu.itesla_project.histodb.config.HistoDbConfiguration;
import eu.itesla_project.histodb.domain.DataSet;
Expand All @@ -48,14 +52,17 @@ public class HistoDataResource {
private HistoDbConfiguration config;

@Inject
private HistoDataService service;
private HistoDataService histoDataService;

@GetMapping(value = "/{db}/{prefix}/{postfix}/itesla/referenceCIM")
public ResponseEntity<String> getReferenceCIM(@PathVariable String db, @PathVariable String prefix,
@PathVariable String postfix) {
try (HistoDataSource hds = HistoDataSourceFactory.getInstance(config, db, prefix, postfix)) {
String ref = hds.getReferenceNetwork() != null ? hds.getReferenceNetwork().getId() : "";
return new ResponseEntity<String>(ref, null, HttpStatus.OK);
} catch (DBException.FileLocked e) {
log.error("DBFile is locked ", e.getMessage());
return new ResponseEntity<String>(e.getMessage(), HttpStatus.SERVICE_UNAVAILABLE);
} catch (Exception e) {
log.error("Get reference CIM error", e);
return new ResponseEntity<String>(e.getMessage(), HttpStatus.NOT_FOUND);
Expand All @@ -74,12 +81,15 @@ public ResponseEntity<String> setReferenceCIM(@PathVariable String db, @PathVari
}
log.info("Set reference CIM from path " + dir);
Path filePath = Paths.get(dir);
service.importReferenceNetwork(hds, filePath);
histoDataService.importReferenceNetwork(hds, filePath);
return new ResponseEntity<String>(
"Set ReferenceCIM: " + hds.getReferenceNetwork() != null ? hds.getReferenceNetwork().getId()
: "" + " for data source " + prefix + "/" + postfix, HttpStatus.OK);
} catch (DBException.FileLocked e) {
log.error("DBFile is locked ", e.getMessage());
return new ResponseEntity<String>(e.getMessage(), HttpStatus.SERVICE_UNAVAILABLE);
} catch (Exception e) {
log.error("Set reference CIM error", e.getMessage());
log.error("Set reference CIM error", e);
return new ResponseEntity<String>(e.getMessage(), HttpStatus.NOT_FOUND);
}
}
Expand All @@ -98,7 +108,10 @@ public ResponseEntity<String> importNetworks(@PathVariable String db, @PathVaria
boolean parallel = parallelStr != null ? Boolean.parseBoolean(parallelStr) : true;

try (HistoDataSource hds = HistoDataSourceFactory.getInstance(config, db, prefix, postfix)) {
service.importData(hds, importDir, parallel);
histoDataService.importData(hds, importDir, parallel);
} catch (DBException.FileLocked e) {
log.error("DBFile is locked ", e.getMessage());
return new ResponseEntity<String>(e.getMessage(), HttpStatus.SERVICE_UNAVAILABLE);
} catch (Exception e) {
log.error("Import data error", e);
return new ResponseEntity<String>(e.getMessage(), HttpStatus.NOT_FOUND);
Expand All @@ -109,91 +122,111 @@ public ResponseEntity<String> importNetworks(@PathVariable String db, @PathVaria
}

@RequestMapping("/{db}/{prefix}/{postfix}/itesla/{service}")
public ResponseEntity<byte[]> getData(@PathVariable String db, @PathVariable String prefix,
public ResponseEntity<StreamingResponseBody> getData(@PathVariable String db, @PathVariable String prefix,
@PathVariable String postfix, @PathVariable String service, WebRequest request) {
log.info("getData " + service);
String format = (String) request.getAttribute("format", 0);

String format = (String) request.getAttribute("format", 0);
log.info("get " + service + " format " + format);
try (HistoDataSource hds = HistoDataSourceFactory.getInstance(config, db, prefix, postfix)) {

QueryParams queryParams = new QueryParams(request);

DataSet data = null;
if (service.equals("data")) {
data = this.service.getData(hds, queryParams);
data = histoDataService.getData(hds, queryParams);
} else if (service.equals("stats")) {
data = this.service.getStats(hds, queryParams);
data = histoDataService.getStats(hds, queryParams);
} else {
return new ResponseEntity<byte[]>(service.getBytes(), HttpStatus.NOT_FOUND);
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse(service), HttpStatus.NOT_FOUND);
}
StringWriter wr = new StringWriter();
data.writeCsv(wr, new Locale(config.getFormatter().getLocale()), config.getFormatter().getSeparator(),
queryParams.isHeaders());
byte[] bytes = wr.toString().getBytes();
HttpHeaders header = new HttpHeaders();

if (format != null && format.equalsIgnoreCase("zip")) {
bytes = gzip(bytes);
boolean zipped = format != null && format.equalsIgnoreCase("zip");

StreamingResponseBody responseBody = getStreamingResponse(data, queryParams.isHeaders(), zipped);

HttpHeaders header = new HttpHeaders();
if (zipped) {
header.add("Content-Encoding", "gzip");
}

header.setContentType(new MediaType("text", "csv"));
return new ResponseEntity<byte[]>(bytes, header, HttpStatus.OK);
return new ResponseEntity<StreamingResponseBody>(responseBody, header, HttpStatus.OK);
} catch (DBException.FileLocked e) {
log.error("DBFile is locked ", e.getMessage());
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse(e.getMessage()), HttpStatus.SERVICE_UNAVAILABLE);
} catch (Exception e) {
log.error("getData error ", e);
return new ResponseEntity<byte[]>(e.getMessage() != null ? e.getMessage().getBytes() : null, HttpStatus.BAD_REQUEST);
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse(e.getMessage()), HttpStatus.BAD_REQUEST);
}
}

@RequestMapping("/{db}/{prefix}/{postfix}/itesla/data/{service}")
public ResponseEntity<byte[]> forecastDiff(@PathVariable String db, @PathVariable String prefix,
public ResponseEntity<StreamingResponseBody> forecastDiff(@PathVariable String db, @PathVariable String prefix,
@PathVariable String postfix, @PathVariable String service, WebRequest request) {
String format = (String) request.getAttribute("format", 0);

log.info("get " + service + " format " + format);
if (!service.equals("forecastsDiff")) {
return new ResponseEntity<byte[]>(service.getBytes(), HttpStatus.NOT_FOUND);
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse(service), HttpStatus.NOT_FOUND);
}

try (HistoDataSource hds = HistoDataSourceFactory.getInstance(config, db, prefix, postfix)) {

QueryParams queryParams = new QueryParams(request);

if (queryParams.getForecastTime() < 0 && (queryParams.getHorizon() == null || queryParams.getHorizon().equals("SN"))) {
return new ResponseEntity<byte[]>("ForecastsDiff operation must be used with either a positive 'forecast' value or a non-snapshot 'horizon'".getBytes(), HttpStatus.BAD_REQUEST);
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse("ForecastsDiff operation must be used with either a positive 'forecast' value or a non-snapshot 'horizon'"), HttpStatus.BAD_REQUEST);
}

DataSet data = this.service.getForecastDiff(hds, queryParams);
StringWriter wr = new StringWriter();
data.writeCsv(wr, new Locale(config.getFormatter().getLocale()), config.getFormatter().getSeparator(),
queryParams.isHeaders());
byte[] bytes = wr.toString().getBytes();
DataSet data = this.histoDataService.getForecastDiff(hds, queryParams);

boolean zipped = format != null && format.equalsIgnoreCase("zip");

StreamingResponseBody responseBody = getStreamingResponse(data, queryParams.isHeaders(), zipped);

HttpHeaders header = new HttpHeaders();

if (format != null && format.equalsIgnoreCase("zip")) {
bytes = gzip(bytes);
if (zipped) {
header.add("Content-Encoding", "gzip");
}

header.setContentType(new MediaType("text", "csv"));
return new ResponseEntity<byte[]>(bytes, header, HttpStatus.OK);
return new ResponseEntity<StreamingResponseBody>(responseBody, header, HttpStatus.OK);
} catch (DBException.FileLocked e) {
log.error("DBFile is locked ", e.getMessage());
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse(e.getMessage()), HttpStatus.SERVICE_UNAVAILABLE);
} catch (Exception e) {
log.error("Forecastdiff error ", e);
return new ResponseEntity<byte[]>(e.getMessage() != null ? e.getMessage().getBytes() : null, HttpStatus.BAD_REQUEST);
return new ResponseEntity<StreamingResponseBody>(getErrorStreamingResponse(e.getMessage()), HttpStatus.BAD_REQUEST);
}
}

private byte[] gzip(byte[] in) throws IOException {
ByteArrayOutputStream bao = new ByteArrayOutputStream();
GZIPOutputStream out = new GZIPOutputStream(bao);
out.write(in);
out.close();
return bao.toByteArray();
private StreamingResponseBody getStreamingResponse(DataSet data, boolean headers, boolean zipped) {
return new StreamingResponseBody() {

@Override
public void writeTo(OutputStream out) throws IOException {
OutputStreamWriter ow = zipped ? new OutputStreamWriter(new GZIPOutputStream(out, true)) : new OutputStreamWriter(out);
data.writeCsv(ow, new Locale(config.getFormatter().getLocale()), config.getFormatter().getSeparator(),
headers);
}
};
}

private StreamingResponseBody getErrorStreamingResponse(String message) {
return new StreamingResponseBody() {

@Override
public void writeTo(OutputStream out) throws IOException {
out.write(message != null ? message.getBytes() : "".getBytes());
}
};
}

public void setConfig(HistoDbConfiguration config) {
this.config = config;
}

public void setService(HistoDataService service) {
this.service = service;
this.histoDataService = service;
}
}
1 change: 1 addition & 0 deletions histodb-server/src/main/resources/application.properties
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
server.max-http-header-size=65536
server.jetty.max-http-post-size=0
server.tomcat.max-http-post-size=0
spring.mvc.async.request-timeout=3600000
67 changes: 51 additions & 16 deletions histodb-server/src/test/java/eu/techrain/histodbserver/HistodbServerApplicationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import org.junit.Test;
import org.springframework.http.ResponseEntity;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;

import eu.itesla_project.histodb.config.HistoDbConfiguration;
import eu.itesla_project.histodb.service.mapdb.HistoDataServiceImpl;
Expand All @@ -35,14 +36,19 @@ public void histodbServerTest() throws IOException {
config.getFormatter().setSeparator(';');
resource.setConfig(config);
resource.setService(new HistoDataServiceImpl());
importNetworksMissingDir();
importNetworks();
setWrongReferenceCIM();
setReferenceCIMMissing();
setReferenceCIM();
getReferenceCIM();
getData();
getDataZip();
getFilteredData();
getDataByIds();
getForecastsDiff();
getForecastsDiffZip();
getForecastsDiffError();
getStats();
}

Expand All @@ -52,14 +58,20 @@ public void setReferenceCIM() throws IOException {
ResponseEntity<String> resp = resource.setReferenceCIM("iteslasim", "test", "2017", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void setWrongReferenceCIM() {
WebRequestMock wr = new WebRequestMock();
wr.setParameter("dir", (String[]) Collections.singletonList("src/test/resources/error").toArray(new String[1]));
ResponseEntity<String> resp = resource.setReferenceCIM("iteslasim", "test", "2017", wr);
assertEquals(404, resp.getStatusCodeValue());
}


public void setReferenceCIMMissing() {
WebRequestMock wr = new WebRequestMock();
ResponseEntity<String> resp = resource.setReferenceCIM("iteslasim", "test", "2017", wr);
assertEquals(400, resp.getStatusCodeValue());
}

public void getReferenceCIM() {
ResponseEntity<String> resp = resource.getReferenceCIM("iteslasim", "test", "2017");
assertEquals(200, resp.getStatusCodeValue());
Expand All @@ -72,17 +84,31 @@ public void importNetworks() throws IOException {
assertEquals(200, resp.getStatusCodeValue());
}

public void importNetworksMissingDir() throws IOException {
WebRequestMock wr = new WebRequestMock();
ResponseEntity <String> resp = (ResponseEntity<String>) resource.importNetworks("iteslasim", "test", "2017", wr);
assertEquals(400, resp.getStatusCodeValue());
}

public void getData() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "csv", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]) );
ResponseEntity <byte[]> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
ResponseEntity <StreamingResponseBody> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getDataZip() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "zip", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
ResponseEntity <StreamingResponseBody> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getFilteredData() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "zip", 0);
wr.setAttribute("format", "zip", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
wr.setParameter("forecast", (String[]) Collections.singletonList("0").toArray(new String[1]));
wr.setParameter("time", (String[]) Collections.singletonList("[2017-01-10T01:00:00Z,2017-01-10T12:00:00Z]").toArray(new String[1]));
Expand All @@ -92,42 +118,51 @@ public void getFilteredData() throws IOException {
String[] equip = {"gen,loads,shunts,stations,2wt,3wt,lines,dangling"};
wr.setParameter("equip", equip);
wr.setParameter("colRange", (String[]) Collections.singletonList("2-5").toArray(new String[1]));
ResponseEntity <byte[]> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
ResponseEntity <StreamingResponseBody> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getDataByIds() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "csv", 0);
wr.setAttribute("format", "csv", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
String[] ids = {"load1,load2,substation1,generator1,LINE1"};
wr.setParameter("ids", ids);
ResponseEntity <byte[]> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
wr.setParameter("ids", ids);
ResponseEntity <StreamingResponseBody> resp = resource.getData("iteslasim", "test", "2017", "data", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getStats() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "csv", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]) );
ResponseEntity <byte[]> resp = resource.getData("iteslasim", "test", "2017", "stats", wr);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
ResponseEntity <StreamingResponseBody> resp = resource.getData("iteslasim", "test", "2017", "stats", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getForecastsDiff() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "csv", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]) );
wr.setParameter("forecast", (String[]) Collections.singletonList("540").toArray(new String[1]) );
ResponseEntity <byte[]> resp = resource.forecastDiff("iteslasim", "test", "2017", "forecastsDiff", wr);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
wr.setParameter("forecast", (String[]) Collections.singletonList("540").toArray(new String[1]));
ResponseEntity <StreamingResponseBody> resp = resource.forecastDiff("iteslasim", "test", "2017", "forecastsDiff", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getForecastsDiffZip() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "csv", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
wr.setParameter("forecast", (String[]) Collections.singletonList("540").toArray(new String[1]));
ResponseEntity <StreamingResponseBody> resp = resource.forecastDiff("iteslasim", "test", "2017", "forecastsDiff", wr);
assertEquals(200, resp.getStatusCodeValue());
}

public void getForecastsDiffError() throws IOException {
WebRequestMock wr = new WebRequestMock();
wr.setAttribute("format", "csv", 0);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]) );
ResponseEntity <byte[]> resp = resource.forecastDiff("iteslasim", "test", "2017", "forecastsDiff", wr);
wr.setParameter("headers", (String[]) Collections.singletonList("true").toArray(new String[1]));
ResponseEntity <StreamingResponseBody> resp = resource.forecastDiff("iteslasim", "test", "2017", "forecastsDiff", wr);
assertEquals(400, resp.getStatusCodeValue());
}

Expand Down

0 comments on commit 10424d9

Please sign in to comment.