Skip to content

Commit

Permalink
Merge pull request #135 from EGA-archive/bugfix/byte_length_missing
Browse files Browse the repository at this point in the history
Fixing bytes length for localEGA profile
  • Loading branch information
anandmohan777 committed May 3, 2019
2 parents 223f7f9 + 536544f commit 97833fd
Show file tree
Hide file tree
Showing 10 changed files with 148 additions and 131 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/*
* Copyright 2016 ELIXIR EGA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.elixir.ega.ebi.dataedge.service;

import eu.elixir.ega.ebi.shared.dto.File;

/**
* @author amohan
*/
public interface FileLengthService {

long getContentLength(File reqFile, String destinationFormat, long startCoordinate, long endCoordinate);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Copyright 2016 ELIXIR EGA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.elixir.ega.ebi.dataedge.service.internal;

import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;

import eu.elixir.ega.ebi.dataedge.service.FileLengthService;
import eu.elixir.ega.ebi.shared.dto.File;

/**
* @author amohan
*/
@Service
@Primary
@Profile("!LocalEGA")
public class FileLengthServiceImpl implements FileLengthService {

@Override
public long getContentLength(File reqFile, String destinationFormat, long startCoordinate, long endCoordinate) {
long length = 0;

// EncryptionFormat
// If destinationFormat is encrypted then we add extra 16 byte to store the IV.
int prefix = 16;
if (destinationFormat.equalsIgnoreCase("plain"))
prefix = 0;

// Range specified?
// The AES encrypted data has a 16 byte random IV at the start, so to
// get the correct (raw) file length we need to remove this 16 byte
// prefix.
if (startCoordinate > 0 || endCoordinate > 0) {
length = endCoordinate - startCoordinate;
} else {
length = reqFile.getFileSize() - 16;
}

return (length + prefix);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright 2016 ELIXIR EGA
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.elixir.ega.ebi.dataedge.service.internal;

import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;

import eu.elixir.ega.ebi.dataedge.service.FileLengthService;
import eu.elixir.ega.ebi.shared.dto.File;

/**
* @author amohan
*/
@Service
@Profile("LocalEGA")
public class LocalEGAFileLengthServiceImpl implements FileLengthService {

@Override
public long getContentLength(File reqFile, String destinationFormat, long startCoordinate, long endCoordinate) {
long length = 0;

if (startCoordinate > 0 || endCoordinate > 0) {
length = endCoordinate - startCoordinate;
} else {
// Removing 32 bytes from the actual file size because when the file is stored in S3,
// the sha256sum is added at the beginning so the length would be -32
length = reqFile.getFileSize() - 32;
}

return length;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import eu.elixir.ega.ebi.shared.dto.EventEntry;
import eu.elixir.ega.ebi.shared.service.DownloaderLogService;
import eu.elixir.ega.ebi.shared.service.FileInfoService;
import eu.elixir.ega.ebi.dataedge.service.FileLengthService;
import eu.elixir.ega.ebi.dataedge.service.FileService;
import eu.elixir.ega.ebi.shared.dto.File;
import eu.elixir.ega.ebi.shared.dto.FileIndexFile;
Expand Down Expand Up @@ -102,6 +103,9 @@ public class RemoteFileServiceImpl implements FileService {

@Autowired
private FileInfoService fileInfoService;

@Autowired
private FileLengthService fileLengthService;

@Override
//@HystrixCommand
Expand Down Expand Up @@ -147,12 +151,12 @@ public void getFile(String fileId,
String headerValue = dlIdentifier.toString();
response = setHeaders(response, headerValue);

long fileLength = fileLengthService.getContentLength(reqFile, destinationFormat, startCoordinate, endCoordinate);

// Content Length of response (if available)
response.setContentLengthLong(getContentLength(reqFile, destinationFormat, startCoordinate, endCoordinate));
response.setContentLengthLong(fileLength);

// If byte range, set response 206
long fileLength = reqFile.getFileSize();
if (destinationFormat.equalsIgnoreCase("plain")) fileLength -= 16;
if (startCoordinate > 0 || endCoordinate > 0) {
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.addHeader("Content-Range", "bytes " + startCoordinate +
Expand Down Expand Up @@ -275,8 +279,8 @@ public void getFileHead(String fileId,
response = setHeaders(response, headerValue);

// Content Length of response (if available)
response.setContentLengthLong(getContentLength(reqFile, destinationFormat, 0, 0));
response.addHeader("X-Content-Length", String.valueOf(getContentLength(reqFile, destinationFormat, 0, 0)));
response.setContentLengthLong(fileLengthService.getContentLength(reqFile, destinationFormat, 0, 0));
response.addHeader("X-Content-Length", String.valueOf(fileLengthService.getContentLength(reqFile, destinationFormat, 0, 0)));
}
}

Expand Down Expand Up @@ -661,6 +665,7 @@ private URI getResUri(String fileStableIdPath,
builder = UriComponentsBuilder.fromHttpUrl(url)
.queryParam("destinationFormat", destFormat)
.queryParam("destinationKey", destKey)
.queryParam("destinationIV", destIV)
.queryParam("filePath", fileStableIdPath); // TEST!!
} else if (destFormat.equalsIgnoreCase("plain")) {
builder = UriComponentsBuilder.fromHttpUrl(url)
Expand Down Expand Up @@ -773,22 +778,4 @@ private VariantContext filterMe(VariantContext context, List<String> fields, Lis
return context;
}

private long getContentLength(File reqFile, String destinationFormat, long startCoordinate, long endCoordinate) {
long length = 0;

// EncryptionFormat
int prefix = 16;
if (destinationFormat.equalsIgnoreCase("plain"))
prefix = 0;

// Range specified?
if (startCoordinate > 0 || endCoordinate > 0) {
length = endCoordinate - startCoordinate;
} else {
length = reqFile.getFileSize() - 16;
}

return (length + prefix);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import eu.elixir.ega.ebi.dataedge.config.InternalErrorException;
import eu.elixir.ega.ebi.dataedge.dto.*;
import eu.elixir.ega.ebi.dataedge.service.FileLengthService;
import eu.elixir.ega.ebi.shared.service.DownloaderLogService;
import eu.elixir.ega.ebi.shared.service.FileInfoService;
import eu.elixir.ega.ebi.shared.dto.File;
Expand Down Expand Up @@ -102,6 +103,9 @@ public class RemoteFileServiceImplTest {

@Mock
private FileInfoService fileInfoService;

@Mock
private FileLengthService fileLengthService;

/**
* Test class for
Expand Down
6 changes: 3 additions & 3 deletions ega-data-api-it/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@
<file.port>9051</file.port>
<res.url>http://localhost</res.url>
<res.port>9092</res.port>
<res.file.checksum>8705adaec1c7cbb27b27456178c15aae</res.file.checksum>
<res.sourceKey>097c7804e08ab2dacdd960e970e88339d9217264b984af8596d3bfd951e378a6</res.sourceKey>
<res.sourceIV>e47bbd537686fe01a427175f81906ddd</res.sourceIV>
<res.file.checksum>c34af3b83f322f7d10e28ad29ba1884c</res.file.checksum>
<res.sourceKey>25f8658f55e9621a7a6c94b7007b5d96eb5807aa50f785a47121430e2781e293</res.sourceKey>
<res.sourceIV>4b6254537d641c7bbc63a405c0a37039</res.sourceIV>
<res.filePath>14.enc</res.filePath>
<dataedge.url>http://localhost</dataedge.url>
<dataedge.port>9058</dataedge.port>
Expand Down
103 changes: 0 additions & 103 deletions ega-data-api-it/src/main/resources/init.sql

This file was deleted.

Binary file modified extras/14.enc
Binary file not shown.
2 changes: 1 addition & 1 deletion extras/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ services:
- "9058:9058"
- "5059:5059"
environment:
- SPRING_PROFILES_ACTIVE=no-oss
- SPRING_PROFILES_ACTIVE=no-oss,LocalEGA
- server.port=9058
- JWTKEY=${JWTKEY}
- FILEDATABASE.listOfServers=filedatabase:9051
Expand Down
2 changes: 1 addition & 1 deletion extras/postgresdb/init.sql
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ CREATE TABLE dev_ega_file.download_log (

INSERT INTO dev_ega_file.file_dataset(file_id, dataset_id) values('EGAF00000000014', 'EGAD00010000919');

INSERT INTO dev_ega_file.file(file_id, file_name, file_path, file_type, file_size, unencrypted_checksum, unencrypted_checksum_type, header) VALUES ('EGAF00000000014', '14.enc', '14.enc' , 'EGAF00000000014.enc', 1220,'8705adaec1c7cbb27b27456178c15aae', 'plain', '6372797074346768010000009b020000c1c14c033efbea862334cc7e011000a2f5606712b87f448a310443584ac85e2dff614955680ee2b30a1d6c6c83de71d2734786d30b4b45d8caaf1fd36de138aaf1acfdddc4a74764df1d1e602fd7ad932d5f1063501593658ec2d9e48d5755815a65673eece627ac6b1df442f4d41175ba5c56c86de02fcf297ede3847e48641b322a0081394a299f2f2e32e944ac86f9a43a841006fa2433b149dcfdd2727bbe0256509283675915f230731d29850f088212fc8d90b1f8fb04dcde93ac0bb96b16b9f16c28972d495d2efa2e1a215e1be2b3cb5b3012aec557be1554e0dd5fe183e8242e62e57a1a4cba1c14220864a48f66bfc9344dc53147453b46b5d75a8e853f146cd68ffc1e5f03c223529b7230004fec43d2e559cb041305cda3fa3599496411e68302c742289f1ae888a2e7353b44ff42ebee17dc98f25da4b78f08cdae787df3bb3251080a1ca3f5f43fdeed0057635a9deec5c95f4c9362f8ed3c2a729ec0eb7cf9ac617dce21909c96cc9da4223caeab9bfec4c006eb61456fd14d09890bc4faa94a3e6fee4be655960cad831f4e75d460b55776d66753c41479f100f17ece8f1f703c09d4a614e7c997c75d6716943228ca23ffcd4eed2f06568d40d503f8f87495b80fb38b8da83bf201578729f5140335e51c83d993d5488419a2afc01a5defd912e10c9a76a9cdd6680765b5dcbc589efcc79c2244e783518495c2612103faf166469e7af200d0bd27a01a5069575acf45b24e5bb37f8cff34f8c8b5fcac827d84f4b8fe222f68de6ab5efc5f25b60b832fec6ed3321ff57648240f6f8d9e0f51b1ad5a4f64b037bdcb64b7a74db00fd77f5e2de764d38e5c7a0641894df5474bced50d255e09f29fb36487dc10423973f63f0af0b61a2ec9bf47890ec2');
INSERT INTO dev_ega_file.file(file_id, file_name, file_path, file_type, file_size, unencrypted_checksum, unencrypted_checksum_type, header) VALUES ('EGAF00000000014', '14.enc', '14.enc' , 'EGAF00000000014.enc', 1287,'c34af3b83f322f7d10e28ad29ba1884c', 'plain', '6372797074346768010000009c020000c1c14c033efbea862334cc7e010ffc0854fbbeb86868ba363562b751ff1a61e2d3a522d672e3d45b9b58ca072c2c76f76198cf90ed654dfd35a6e4e0fb008ba7be397a06c961b5cc775705c28b88443e6812d13276006bd8615f234255fbeee57f6601a50fd89300e8fd1bea7f4d771b69d98c93dc97ae4869652a911c47339297e8b3077740e8418d19bfdaaa92c1bc744810f617aac72591678f6ece1707b64015e3139967117408831e9c9a5b8c72ace8dbd8b1a0f7419b7e253c2b20a83de746886b615268a6bdf3ccb84221c45be9b493a45f15d3d6fb01b28d142d42f748cf4c153fa8b834069899f75bb78882af41e36d2b19c82bb6a1d194cabf6ff64fe05db02460bbda206628466a8ed9ee91c7c9b3094e7936be8917669a1e5644e0cbd9dfd9fc4a5a50ffe3e1d1a95c5834416a187577076d2a29380b65da0be3c837f2296b1692694af8daf46d3bd584bf0d7415d4f1aff62774083166b8435e113d18e49bcf5746a675d5ffa741b1c1e37e7a1ccf97154280135a670240e22467166d05493142b3254c5dcc4f8570152ec0c592eb64770104e7e916bf8f812076be7d08c4ab400025f9f2265029bb1e9ef75e7204b26dd81a4c5b855cdc2e83ba46fddc4ee623af0bbdb08decfbaf6c4dbd5c23798ec9929274666205e236a4df52890c3d903ab0eee690b9e9a34f850f456b1d8110615115845ad8e9bc608a2cb8d0b4d1b671a236dc126a17c8d2d27b01957917699d24f9d80f0faab3f28cdf95cfcc3c767587891c11e0f69295a1e730e6639db6b3ef3e40e01d8881308b42fd01a193f52358ec6893af787cf8095767a1282123308b12ffb70197250de9935cb305f40837c7b54d63b1d5a33e5bb7b1bf9d8e24cb30e80f5b131b08246cd5f3b47b7dbe8d6f39ed0938');

INSERT INTO dev_ega_file.file_key(file_id, encryption_key_id, encryption_algorithm) values ('EGAF00000000014', 1 , 'aes256' );

Expand Down

0 comments on commit 97833fd

Please sign in to comment.