Mgraff memory objectstore (#30)

Implement an in-memory object store.

kayenta-objectstore-memory/kayenta-objectstore-memory.gradle

@@ -0,0 +1,6 @@
+dependencies {
+  compile project(":kayenta-core")
+  compile spinnaker.dependency('bootWeb')
+  compile spinnaker.dependency("korkSwagger")
+  compile spinnaker.dependency('lombok')
+}

kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryConfiguration.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.memory.config;
+
+import com.netflix.kayenta.memory.security.MemoryAccountCredentials;
+import com.netflix.kayenta.memory.security.MemoryNamedAccountCredentials;
+import com.netflix.kayenta.memory.storage.MemoryStorageService;
+import com.netflix.kayenta.security.AccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.kayenta.storage.StorageService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.CollectionUtils;
+
+import java.util.List;
+
+@Configuration
+@EnableConfigurationProperties
+@ConditionalOnProperty("kayenta.memory.enabled")
+@ComponentScan({"com.netflix.kayenta.memory"})
+@Slf4j
+public class MemoryConfiguration {
+
+    @Bean
+    @ConfigurationProperties("kayenta.memory")
+    MemoryConfigurationProperties memoryConfigurationProperties() {
+        return new MemoryConfigurationProperties();
+    }
+
+    @Bean
+    StorageService storageService(MemoryConfigurationProperties memoryConfigurationProperties,
+                                  AccountCredentialsRepository accountCredentialsRepository) {
+        MemoryStorageService.MemoryStorageServiceBuilder memoryStorageServiceBuilder = MemoryStorageService.builder();
+
+        for (MemoryManagedAccount memoryManagedAccount : memoryConfigurationProperties.getAccounts()) {
+            String name = memoryManagedAccount.getName();
+            String namespace = memoryManagedAccount.getNamespace();
+            List<AccountCredentials.Type> supportedTypes = memoryManagedAccount.getSupportedTypes();
+
+            log.info("Registering Memory account {} with supported types {}.", name, supportedTypes);
+
+            MemoryAccountCredentials memoryAccountCredentials =
+                    MemoryAccountCredentials.builder().build();
+            MemoryNamedAccountCredentials.MemoryNamedAccountCredentialsBuilder memoryNamedAccountCredentialsBuilder =
+                    MemoryNamedAccountCredentials.builder()
+                    .name(name)
+                    .namespace(namespace)
+                    .credentials(memoryAccountCredentials);
+
+            if (!CollectionUtils.isEmpty(supportedTypes)) {
+                memoryNamedAccountCredentialsBuilder.supportedTypes(supportedTypes);
+            }
+
+            MemoryNamedAccountCredentials memoryNamedAccountCredentials = memoryNamedAccountCredentialsBuilder.build();
+            accountCredentialsRepository.save(name, memoryNamedAccountCredentials);
+            memoryStorageServiceBuilder.accountName(name);
+        }
+
+        MemoryStorageService memoryStorageService = memoryStorageServiceBuilder.build();
+
+        log.info("Populated MemoryStorageService with {} in-memory accounts.", memoryStorageService.getAccountNames().size());
+
+        return memoryStorageService;
+    }
+}

kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryConfigurationProperties.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.memory.config;
+
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MemoryConfigurationProperties {
+    @Getter
+    private List<MemoryManagedAccount> accounts = new ArrayList<>();
+}

kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/config/MemoryManagedAccount.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.memory.config;
+
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Data
+public class MemoryManagedAccount {
+    @NotNull
+    private String name;
+
+    private String namespace;
+
+    private List<AccountCredentials.Type> supportedTypes;
+}

kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryAccountCredentials.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.memory.security;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Optional;
+
+@Builder
+@Data
+@Slf4j
+// TODO: Not sure what kind of credentials or configuration is really required here yet.
+public class MemoryAccountCredentials {
+
+    private static String applicationVersion =
+            Optional.ofNullable(MemoryAccountCredentials.class.getPackage().getImplementationVersion()).orElse("Unknown");
+}

kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/security/MemoryNamedAccountCredentials.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.memory.security;
+
+import com.netflix.kayenta.security.AccountCredentials;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Singular;
+
+import javax.validation.constraints.NotNull;
+import java.util.List;
+
+@Builder
+@Data
+public class MemoryNamedAccountCredentials implements AccountCredentials<MemoryAccountCredentials> {
+    @NotNull
+    private String name;
+
+    @NotNull
+    @Singular
+    private List<Type> supportedTypes;
+
+    @NotNull
+    private MemoryAccountCredentials credentials;
+
+    @NotNull
+    private String namespace;
+
+    @Override
+    public String getType() {
+        return "memory";
+    }
+}

kayenta-objectstore-memory/src/main/java/com/netflix/kayenta/memory/storage/MemoryStorageService.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright 2017 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.kayenta.memory.storage;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.kayenta.memory.security.MemoryNamedAccountCredentials;
+import com.netflix.kayenta.security.AccountCredentialsRepository;
+import com.netflix.kayenta.storage.ObjectType;
+import com.netflix.kayenta.storage.StorageService;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Singular;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.validation.constraints.NotNull;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+@Builder
+class ObjectMetadata {
+    @NotNull
+    public final String name;
+}
+
+@Builder
+@Slf4j
+public class MemoryStorageService implements StorageService {
+
+    public static class MemoryStorageServiceBuilder {
+        private Map<String, String> entries = new ConcurrentHashMap<String, String>();
+    }
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    @NotNull
+    @Singular
+    @Getter
+    private List<String> accountNames;
+
+    @Autowired
+    AccountCredentialsRepository accountCredentialsRepository;
+
+    private Map<String, String> entries;
+
+    @Override
+    public boolean servicesAccount(String accountName) {
+        return accountNames.contains(accountName);
+    }
+
+    @Override
+    public <T> T loadObject(String accountName, ObjectType objectType, String objectKey) throws IllegalArgumentException {
+        String key = makekey(accountName, objectType, objectKey);
+        log.info("Getting key {}", key);
+        String json = entries.get(key);
+        if (json == null)
+            throw new IllegalArgumentException("No such object named " + key);
+
+        try {
+            return objectMapper.readValue(json, objectType.getTypeReference());
+        } catch (IOException e) {
+            log.error("Read failed on path {}: {}", key, e);
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Override
+    public <T> void storeObject(String accountName, ObjectType objectType, String objectKey, T obj) {
+        String key = makekey(accountName, objectType, objectKey);
+        log.info("Writing key {}", key);
+        try {
+            String json = objectMapper.writeValueAsString(obj);
+            entries.put(key, json);
+        } catch (IOException e) {
+            log.error("Update failed on path {}: {}", key, e);
+            throw new IllegalStateException(e);
+        }
+    }
+
+    @Override
+    public void deleteObject(String accountName, ObjectType objectType, String objectKey) {
+        String key = makekey(accountName, objectType, objectKey);
+        log.info("Deleting key {}", key);
+        String oldValue = entries.remove(key);
+        if (oldValue == null) {
+            log.error("Object named {} does not exist", key);
+            throw new IllegalStateException("Does not exist");
+        }
+    }
+
+    @Override
+    public List<Map<String, Object>> listObjectKeys(String accountName, ObjectType objectType) {
+        String prefix = makeprefix(accountName, objectType);
+
+        return entries
+                .entrySet()
+                .parallelStream()
+                .filter(e -> e.getKey().startsWith(prefix))
+                .map(e -> mapFrom(e.getKey()))
+                .collect(Collectors.toList());
+    }
+
+    private Map<String, Object> mapFrom(String key) {
+        Map<String, Object> ret = new HashMap<String, Object>();
+        ret.put(nameFromKey(key), metadataFor(key));
+        return ret;
+    }
+
+    private String nameFromKey(String key) {
+        return key.split(":", 3)[2];
+    }
+
+    private ObjectMetadata metadataFor(String key) {
+        return ObjectMetadata.builder().name(nameFromKey(key)).build();
+    }
+
+    private String makeprefix(String accountName, ObjectType objectType) {
+        MemoryNamedAccountCredentials credentials = (MemoryNamedAccountCredentials)accountCredentialsRepository
+                .getOne(accountName)
+                .orElseThrow(() -> new IllegalArgumentException("Unable to resolve account " + accountName + "."));
+        String namespace = credentials.getNamespace();
+        String typename = objectType.getTypeReference().getType().getTypeName();
+        return namespace + ":" + typename + ":";
+    }
+
+    private String makekey(String accountName, ObjectType objectType, String objectKey) {
+        return makeprefix(accountName, objectType) + objectKey;
+    }
+}

kayenta-web/config/kayenta.yml

@@ -33,6 +33,9 @@ kayenta:
   redis:
     enabled: false
 
+  memory:
+    enabled: false
+
 swagger:
   enabled: true
   title: Kayenta API