feat(files): add a tab on the person page to create, list, download a…

…nd delete files (documents) for that person

The files are stored in Google Cloud Storage. See the README for details.

.gitignore

@@ -26,4 +26,5 @@ nbdist/
 .nb-gradle/
 
 classes
-.java-version
+.java-version
+secrets

README.md

@@ -41,7 +41,32 @@ Then run `yarn start` to start the app, using the proxy conf to reroute calls to
 
 The application will be available on http://localhost:4200
 
+### External services
 
+The application stores uploaded documents in Google Cloud Storage. This means that the application
+needs credentials (which are json files generated by Google) in order to work fine.
+
+Two separate accounts, and thus credentials files, are used:
+
+ - one for development
+ - one for production
+
+We expect to stay inside the "Always Free" tier, whatever the account is.
+
+Here are various interesting links regarding Google Cloud Storage (GCS):
+
+ - [The landing page of GCS](https://cloud.google.com/storage/)
+ - [The pricing page of GCS](https://cloud.google.com/storage/pricing). It describes the conditions 
+   of the "Always Free" tier. The most importtant part being that (as of now), the only regions 
+   eligible to this free tier are `us-west1`, `us-central1`, and `us-east1`. This means that the buckets
+   created for the application should be created in one of these 3 regions. We currently use only one bucket 
+   (for each account - dev and prod)
+ - [The Google Cloud console](https://console.cloud.google.com/storage): Make sure to select the appropriate 
+   Google account when visiting it, and to select the `globe42` project. It allows creating the bucket,
+   browsing and deleting the files, creating credentials (service accounts), etc.
+ - [The documentation](https://cloud.google.com/storage/docs/), and 
+   [the javadoc of the Java client library](https://googlecloudplatform.github.io/google-cloud-java/0.23.1/apidocs/index.html)
+
 ## Build
 
 To build the app, just run:
@@ -52,13 +77,21 @@ This will build a standalone jar at `backend/build/libs/globe42.jar`, that you c
 
     java -jar backend/build/libs/globe42.jar --globe42.secretKey=<some secret key>
 
-
 To start the application with the demo profile, add this command-line option:  
 
     --spring.profiles.active=demo
 
 And the full app runs on http://localhost:9000
 
+By default, the default GCS credentials are used when launching the app this way. That means
+that the GCS APIs won't be accessible unless you set the `GOOGLE_APPLICATION_CREDENTIALS` as
+described in [the documentation about default credentials](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork)
+
+To avoid setting a global environment variable, you can instead use this command-line option:
+
+    --globe42.googleCloudStorageCredentialsPath=secrets/google-cloud-storage-dev.json
+
+This credentials file is located in [the Ninja Squad Drive](https://drive.google.com/drive/u/1/folders/0B0FLWwufPzrTN1NVTDZJMWZTVXc)
 
 ## Deployment on CleverCloud
 
@@ -82,4 +115,9 @@ That's it!
 
 see the [logs console](https://console.clever-cloud.com/organisations/orga_dd753560-9dfe-4c93-a891-c639d138354b/applications/app_5e422400-281d-499b-b34c-7555c2f7fadd/logs) and cross your fingers ;-)
 
+### Google Cloud Storage credentials on CleverCloud
+
+Applications on CleverCloud don't have access to the file system. So, instead of defining an environment variable
+containing the path of the GCS credentials, we use an environment variable, `globe42.googleCloudStorageCredentials`,
+containing the *content* of the production credentials file. 
 

backend/build.gradle

@@ -46,16 +46,20 @@ bootJar {
 bootRun {
     args '--spring.profiles.active=demo'
     args '--globe42.secretKey=QMwbcwa19VV02Oy5T7LSWyV+/wZrOsRRfhCR6TkapsY='
+    args '--globe42.googleCloudStorageCredentialsPath=' + rootProject.file('secrets/google-cloud-storage-dev.json')
 }
 
 dependencies {
     compile 'org.springframework.boot:spring-boot-starter-data-jpa'
     compile 'org.springframework.boot:spring-boot-starter-web'
+    compile 'io.jsonwebtoken:jjwt:0.7.0'
+    compile 'com.google.cloud:google-cloud-storage:1.5.1'
+
     testCompile 'org.springframework.boot:spring-boot-starter-test'
     testCompile 'com.ninja-squad:DbSetup:2.1.0'
+
     runtime 'org.postgresql:postgresql:9.4.1212'
     runtime 'org.flywaydb:flyway-core'
-    compile 'io.jsonwebtoken:jjwt:0.7.0'
 }
 
 // remove default tasks added by flyway plugin

backend/src/main/java/org/globe42/storage/FileDTO.java

@@ -0,0 +1,62 @@
+package org.globe42.storage;
+
+import java.time.Instant;
+
+import com.google.cloud.storage.BlobInfo;
+
+/**
+ * Information about a file (stored in a Google Cloud Storage)
+ * @author JB Nizet
+ */
+public final class FileDTO {
+
+    /**
+     * The name of the file
+     */
+    private final String name;
+
+    /**
+     * The size of the file, in bytes
+     */
+    private final Long size;
+
+    /**
+     * The instant when the file was created in the storage
+     */
+    private final Instant creationInstant;
+
+    /**
+     * The content type of the file
+     */
+    private final String contentType;
+
+    public FileDTO(BlobInfo blob, String prefix) {
+        this.name = blob.getName().substring(prefix.length());
+        this.size = blob.getSize();
+        this.creationInstant = blob.getCreateTime() == null ? Instant.now() : Instant.ofEpochMilli(blob.getCreateTime());
+        this.contentType = blob.getContentType();
+    }
+
+    public FileDTO(String name, Long size, Instant creationInstant, String contentType) {
+        this.name = name;
+        this.size = size;
+        this.creationInstant = creationInstant;
+        this.contentType = contentType;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Long getSize() {
+        return size;
+    }
+
+    public Instant getCreationInstant() {
+        return creationInstant;
+    }
+
+    public String getContentType() {
+        return contentType;
+    }
+}

backend/src/main/java/org/globe42/storage/ReadableFile.java

@@ -0,0 +1,28 @@
+package org.globe42.storage;
+
+import java.io.InputStream;
+import java.nio.channels.Channels;
+
+import com.google.cloud.storage.Blob;
+
+/**
+ * A file that can be read
+ * @author JB Nizet
+ */
+public class ReadableFile {
+    private final FileDTO file;
+    private final Blob blob;
+
+    public ReadableFile(Blob blob, String prefix) {
+        this.blob = blob;
+        this.file = new FileDTO(blob, prefix);
+    }
+
+    public InputStream getInputStream() {
+        return Channels.newInputStream(this.blob.reader());
+    }
+
+    public FileDTO getFile() {
+        return file;
+    }
+}

backend/src/main/java/org/globe42/storage/StorageConfig.java

@@ -0,0 +1,78 @@
+package org.globe42.storage;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Config class for Google Cloud Storage beans. See the README for useful links regarding Google Cloud Storage.
+ * @author JB Nizet
+ */
+@Configuration
+public class StorageConfig {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(StorageConfig.class);
+
+    /**
+     * The JSON string containing the credentials, loaded from the property
+     * <code>globe42.googleCloudStorageCredentials</code>. If not
+     * null (typically in production, on clever cloud, where the whole JSON credentials are stored in an environment
+     * variable) then this is used as the source of the google credentials.
+     */
+    private final String credentials;
+
+    /**
+     * The path to the json file containing the credentials, loaded from the property
+     * <code>globe42.googleCloudStorageCredentialsPath</code>. Only used if {@link #credentials} is null.
+     * Typically used in dev mode, where specifying a file path in a command-line property is easier.
+     */
+    private final File credentialsPath;
+
+    public StorageConfig(@Value("${globe42.googleCloudStorageCredentials:#{null}}") String credentials,
+                         @Value("${globe42.googleCloudStorageCredentialsPath:#{null}}") File credentialsPath) {
+        this.credentials = credentials;
+        this.credentialsPath = credentialsPath;
+    }
+
+    @Bean
+    public Storage storage() throws IOException {
+        if (this.credentials != null) {
+            LOGGER.info("Property globe42.googleCloudStorageCredentials is set." +
+                            " Using its value as Google Cloud Storage JSON credentials");
+            InputStream in = new ByteArrayInputStream(this.credentials.getBytes(StandardCharsets.UTF_8));
+            return StorageOptions
+                .newBuilder()
+                .setCredentials(GoogleCredentials.fromStream(in))
+                .build()
+                .getService();
+        }
+        else if (this.credentialsPath != null) {
+            LOGGER.info("Property globe42.googleCloudStorageCredentialsPath is set." +
+                            " Using its value as a JSON file path to the Google Cloud Storage credentials");
+            try (InputStream in = new FileInputStream(this.credentialsPath)) {
+                return StorageOptions
+                    .newBuilder()
+                    .setCredentials(GoogleCredentials.fromStream(in))
+                    .build()
+                    .getService();
+            }
+        }
+        else {
+            LOGGER.warn("Neither property globe42.googleCloudStorageCredentials nor globe42.googleCloudStorageCredentials is set." +
+                            " Using default instance credentials.");
+            return StorageOptions.getDefaultInstance().getService();
+        }
+    }
+}

backend/src/main/java/org/globe42/storage/StorageService.java

@@ -0,0 +1,93 @@
+package org.globe42.storage;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.channels.Channels;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import com.google.api.gax.paging.Page;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.common.io.ByteStreams;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Service;
+
+/**
+ * Service used to wrap the Google cloud storage API
+ * @author JB Nizet
+ */
+@Service
+public class StorageService {
+
+    static final String PERSON_FILES_BUCKET = "personfiles";
+
+    private final Storage storage;
+
+    public StorageService(Storage storage) {
+        this.storage = storage;
+    }
+
+    /**
+     * Lists the files in the given directory
+     * @param directory the directory (serving as a prefix) to list the files
+     * @return the list of files in the given directory. The directory prefix is stripped of from the names of the
+     * returned files
+     */
+    public List<FileDTO> list(String directory) {
+        final String prefix = toPrefix(directory);
+        Page<Blob> page = storage.list(PERSON_FILES_BUCKET,
+                                       Storage.BlobListOption.pageSize(10_000),
+                                       Storage.BlobListOption.currentDirectory(),
+                                       Storage.BlobListOption.prefix(prefix));
+        return StreamSupport.stream(page.getValues().spliterator(), false)
+            .map(blob -> new FileDTO(blob, prefix))
+            .collect(Collectors.toList());
+    }
+
+    /**
+     * Gets the given file in the given directory
+     * @param directory the directory (serving as a prefix) of the file
+     * @return the file in the given directory. The directory prefix is stripped of from the name of the
+     * returned file
+     */
+    public ReadableFile get(String directory, String name) {
+        String prefix = toPrefix(directory);
+        Blob blob = storage.get(PERSON_FILES_BUCKET, prefix + name);
+        return new ReadableFile(blob, prefix);
+    }
+
+    public FileDTO create(String directory,
+                          String name,
+                          String contentType,
+                          InputStream data) {
+        String prefix = toPrefix(directory);
+        BlobId blobId = BlobId.of(PERSON_FILES_BUCKET, prefix + name);
+        if (contentType == null) {
+            contentType = MediaType.APPLICATION_OCTET_STREAM.toString();
+        }
+        BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(contentType).build();
+        try (OutputStream out = Channels.newOutputStream(storage.writer(blobInfo))) {
+            ByteStreams.copy(data, out);
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        return new FileDTO(blobInfo, prefix);
+    }
+
+    public void delete(String directory, String name) {
+        String prefix = toPrefix(directory);
+        storage.delete(PERSON_FILES_BUCKET, prefix + name);
+    }
+
+    private String toPrefix(String directory) {
+        return directory.endsWith("/") ? directory : directory + "/";
+    }
+}

backend/src/main/java/org/globe42/web/AsyncConfig.java

@@ -0,0 +1,23 @@
+package org.globe42.web;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+/**
+ * Configuration class used to configure a thread pool used for asynchronous request processing (file downloads)
+ * @author JB Nizet
+ */
+@Configuration
+public class AsyncConfig implements WebMvcConfigurer {
+    @Override
+    public void configureAsyncSupport(AsyncSupportConfigurer configurer) {
+        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
+        taskExecutor.setCorePoolSize(1);
+        taskExecutor.setMaxPoolSize(30);
+        taskExecutor.setThreadNamePrefix("GlobeWebAsync");
+        taskExecutor.initialize();
+        configurer.setTaskExecutor(taskExecutor);
+    }
+}