namespace cache for testing on circle CI (#4120)

* WIP, namespace cache for testing on circle CI

square/okhttp#6214
interceptors would be a choice to add a fragment as suggested https://square.github.io/okhttp/interceptors/ but non-logging interceptors seem broken in the kohsuke library

* My skin is not my own

* It's not who I am underneath, but what I do that defines me.

* duh, need this to actually save a new cache

* Badge will never work since we now push images

Let's me test the performance of the new cache

.circleci/config.yml

@@ -157,7 +157,7 @@ jobs:
     steps: # a collection of executable commands
       - checkout
       - restore_cache:
-          key: dockstore-web-cache-unit-test-{{ .Environment.CACHE_VERSION }}
+          key: dockstore-web-cache-unit-test-{{ .Environment.CACHE_SEGMENTED_VERSION }}-segmented
       - run:
           name: Install yq
           command: |
@@ -213,7 +213,7 @@ jobs:
             - ~/.m2
           key: dockstore-java-{{ checksum "pom.xml" }}
       - save_cache:
-          key: dockstore-web-cache-unit-test-{{ .Environment.CACHE_VERSION }}
+          key: dockstore-web-cache-unit-test-{{ .Environment.CACHE_SEGMENTED_VERSION }}-segmented
           paths:
             - /tmp/dockstore-web-cache
       - run:
@@ -315,7 +315,7 @@ commands:
             sudo apt update
             sudo apt install -y postgresql-client
       - restore_cache:
-          key: dockstore-web-cache-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CACHE_VERSION }}
+          key: dockstore-web-cache-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CACHE_SEGMENTED_VERSION }}-segmented
   setup_integration_test:
     steps:
       - run:
@@ -359,7 +359,7 @@ commands:
   save_test_results:
     steps:
       - save_cache:
-          key: dockstore-web-cache-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CACHE_VERSION }}
+          key: dockstore-web-cache-{{ .Environment.CIRCLE_JOB }}-{{ .Environment.CACHE_SEGMENTED_VERSION }}-segmented
           paths:
             - /tmp/dockstore-web-cache
       - run:

README.md

@@ -8,7 +8,6 @@
 [![license](https://img.shields.io/hexpm/l/plug.svg?maxAge=2592000)](LICENSE)
 [![CircleCI](https://circleci.com/gh/dockstore/dockstore/tree/develop.svg?style=svg)](https://circleci.com/gh/dockstore/dockstore/tree/develop)
 [![Documentation Status](https://readthedocs.org/projects/dockstore/badge/?version=develop)](https://dockstore.readthedocs.io/en/develop/?badge=develop)
-[![Docker Repository on Quay](https://quay.io/repository/dockstore/dockstore-webservice/status "Docker Repository on Quay")](https://quay.io/repository/dockstore/dockstore-webservice)
 
 
 # Dockstore

dockstore-webservice/src/main/java/io/dockstore/webservice/DockstoreWebserviceApplication.java

@@ -189,8 +189,12 @@ public static void main(String[] args) throws Exception {
         new DockstoreWebserviceApplication().run(args);
     }
 
-    public static Cache getCache() {
-        return cache;
+    public static Cache getCache(String cacheNamespace) {
+        if (cacheNamespace == null) {
+            return cache;
+        } else {
+            return generateCache(cacheNamespace);
+        }
     }
 
     @Override
@@ -220,18 +224,7 @@ public DataSourceFactory getDataSourceFactory(DockstoreWebserviceConfiguration c
         bootstrap.addBundle(new MultiPartBundle());
 
         if (cache == null) {
-            int cacheSize = CACHE_IN_MB * BYTES_IN_KILOBYTE * KILOBYTES_IN_MEGABYTE; // 100 MiB
-            final File cacheDir;
-            try {
-                // let's try using the same cache each time
-                // not sure how corruptible/non-curruptable the cache is
-                // https://github.com/square/okhttp/blob/parent-3.10.0/okhttp/src/main/java/okhttp3/internal/cache/DiskLruCache.java#L82 looks promising
-                cacheDir = Files.createDirectories(Paths.get(DOCKSTORE_WEB_CACHE)).toFile();
-            } catch (IOException e) {
-                LOG.error("Could no create or re-use web cache", e);
-                throw new RuntimeException(e);
-            }
-            cache = new Cache(cacheDir, cacheSize);
+            cache = generateCache(null);
         }
         try {
             cache.initialize();
@@ -259,6 +252,21 @@ public DataSourceFactory getDataSourceFactory(DockstoreWebserviceConfiguration c
         }
     }
 
+    private static Cache generateCache(String suffix) {
+        int cacheSize = CACHE_IN_MB * BYTES_IN_KILOBYTE * KILOBYTES_IN_MEGABYTE; // 100 MiB
+        final File cacheDir;
+        try {
+            // let's try using the same cache each time
+            // not sure how corruptible/non-curruptable the cache is
+            // namespace cache when testing on circle ci
+            cacheDir = Files.createDirectories(Paths.get(DOCKSTORE_WEB_CACHE + (suffix == null ? "" : "/" + suffix))).toFile();
+        } catch (IOException e) {
+            LOG.error("Could no create or re-use web cache", e);
+            throw new RuntimeException(e);
+        }
+        return new Cache(cacheDir, cacheSize);
+    }
+
     private static void configureMapper(ObjectMapper objectMapper) {
         objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
         objectMapper.registerModule(new Hibernate5Module());

dockstore-webservice/src/main/java/io/dockstore/webservice/helpers/AbstractImageRegistry.java

@@ -194,7 +194,7 @@ public List<Tool> refreshTools(final long userId, final UserDAO userDAO, final T
             List<Tag> toolTags = getTags(tool);
             final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory
                 .createSourceCodeRepo(tool.getGitUrl(), client, bitbucketToken == null ? null : bitbucketToken.getContent(),
-                    gitlabToken == null ? null : gitlabToken.getContent(), githubToken == null ? null : githubToken.getContent());
+                    gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
             updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
         }
 
@@ -241,7 +241,7 @@ public List<Tool> refreshTool(final long userId, final UserDAO userDAO, final To
             List<Tag> toolTags = getTags(tool);
             final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory
                 .createSourceCodeRepo(tool.getGitUrl(), client, bitbucketToken == null ? null : bitbucketToken.getContent(),
-                    gitlabToken == null ? null : gitlabToken.getContent(), githubToken.getContent());
+                    gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
             updateTags(toolTags, tool, sourceCodeRepo, tagDAO, fileDAO, toolDAO, fileFormatDAO, eventDAO, user);
         }
 

dockstore-webservice/src/main/java/io/dockstore/webservice/helpers/GitHubSourceCodeRepo.java

@@ -102,19 +102,30 @@ public class GitHubSourceCodeRepo extends SourceCodeRepoInterface {
     private static final Logger LOG = LoggerFactory.getLogger(GitHubSourceCodeRepo.class);
     private final GitHub github;
 
-    public GitHubSourceCodeRepo(String gitUsername, String githubTokenContent) {
+    /**
+     *
+     * @param githubTokenUsername the username for githubTokenContent
+     * @param gitUsername deprecate this, this is more accurately, the github organization for a specific repository when this class was used for one repo at a time, weird
+     * @param githubTokenContent authorization token
+     */
+    public GitHubSourceCodeRepo(@Deprecated String gitUsername, String githubTokenContent, String githubTokenUsername) {
         this.gitUsername = gitUsername;
         // this code is duplicate from DockstoreWebserviceApplication, except this is a lot faster for unknown reasons ...
-        OkHttpClient.Builder builder = new OkHttpClient().newBuilder().cache(DockstoreWebserviceApplication.getCache());
+        OkHttpClient.Builder builder = new OkHttpClient().newBuilder();
+        builder.eventListener(new CacheHitListener(GitHubSourceCodeRepo.class.getSimpleName(), githubTokenUsername));
         if (System.getenv("CIRCLE_SHA1") != null) {
-            builder.eventListener(new CacheHitListener(GitHubSourceCodeRepo.class.getSimpleName(), gitUsername));
+            // namespace cache by user when testing
+            builder.cache(DockstoreWebserviceApplication.getCache(githubTokenUsername));
+        } else {
+            // use general cache
+            builder.cache(DockstoreWebserviceApplication.getCache(null));
         }
         OkHttpClient build = builder.build();
         ObsoleteUrlFactory obsoleteUrlFactory = new ObsoleteUrlFactory(build);
 
         HttpConnector okHttp3Connector = new ImpatientHttpConnector(obsoleteUrlFactory::open);
         try {
-            this.github = new GitHubBuilder().withOAuthToken(githubTokenContent, gitUsername).withRateLimitHandler(new WaitReporter(gitUsername))
+            this.github = new GitHubBuilder().withOAuthToken(githubTokenContent, githubTokenUsername).withRateLimitHandler(new WaitReporter(githubTokenUsername))
                     .withAbuseLimitHandler(AbuseLimitHandler.WAIT).withConnector(okHttp3Connector).build();
         } catch (IOException e) {
             throw new RuntimeException(e);

dockstore-webservice/src/main/java/io/dockstore/webservice/helpers/SourceCodeRepoFactory.java

@@ -44,13 +44,13 @@ private SourceCodeRepoFactory() {
 
     public static SourceCodeRepoInterface createGitHubAppRepo(String token) {
         // The gitUsername doesn't seem to matter
-        return new GitHubSourceCodeRepo("dockstore", token);
+        return new GitHubSourceCodeRepo("dockstore", token, "JWT");
     }
 
     public static SourceCodeRepoInterface createSourceCodeRepo(Token token) {
         SourceCodeRepoInterface repo;
         if (Objects.equals(token.getTokenSource(), TokenType.GITHUB_COM)) {
-            repo = new GitHubSourceCodeRepo(token.getUsername(), token.getContent());
+            repo = new GitHubSourceCodeRepo(token.getUsername(), token.getContent(), token.getUsername());
         } else if (Objects.equals(token.getTokenSource(), TokenType.BITBUCKET_ORG)) {
             repo = new BitBucketSourceCodeRepo(token.getUsername(), token.getContent());
         } else if (Objects.equals(token.getTokenSource(), TokenType.GITLAB_COM)) {
@@ -65,7 +65,7 @@ public static SourceCodeRepoInterface createSourceCodeRepo(Token token) {
     }
 
     public static SourceCodeRepoInterface createSourceCodeRepo(String gitUrl, HttpClient client, String bitbucketTokenContent,
-            String gitlabTokenContent, String githubTokenContent) {
+            String gitlabTokenContent, Token githubToken) {
 
         Map<String, String> repoUrlMap = parseGitUrl(gitUrl);
 
@@ -78,7 +78,7 @@ public static SourceCodeRepoInterface createSourceCodeRepo(String gitUrl, HttpCl
 
         SourceCodeRepoInterface repo;
         if (SourceControl.GITHUB.toString().equals(source)) {
-            repo = new GitHubSourceCodeRepo(gitUsername, githubTokenContent);
+            repo = new GitHubSourceCodeRepo(gitUsername, githubToken.getContent(), githubToken.getUsername());
         } else if (SourceControl.BITBUCKET.toString().equals(source)) {
             if (bitbucketTokenContent != null) {
                 repo = new BitBucketSourceCodeRepo(gitUsername, bitbucketTokenContent);
@@ -106,19 +106,19 @@ public static SourceCodeRepoInterface createSourceCodeRepo(String gitUrl, HttpCl
      * @param gitUrl    Git URL to identify which SourceCodeRepo to return
      * @param bitbucketTokenContent The user's Bitbucket token if it exists, null otherwise
      * @param gitlabTokenContent    The user's GitLab token if it exists, null otherwise
-     * @param githubTokenContent    The user's GitHub token if it exists, null otherwise
+     * @param githubToken    The user's GitHub token if it exists, null otherwise
      * @return  a SourceCode repo if a token exists, null otherwise
      */
     public static SourceCodeRepoInterface createSourceCodeRepo(String gitUrl, String bitbucketTokenContent,
-            String gitlabTokenContent, String githubTokenContent) {
+            String gitlabTokenContent, Token githubToken) {
         Map<String, String> repoUrlMap = parseGitUrl(gitUrl);
         if (repoUrlMap == null) {
             return null;
         }
         String source = repoUrlMap.get("Source");
         String gitUsername = repoUrlMap.get("Username");
         if (SourceControl.GITHUB.toString().equals(source)) {
-            return githubTokenContent != null ? new GitHubSourceCodeRepo(gitUsername, githubTokenContent) : null;
+            return githubToken != null ? new GitHubSourceCodeRepo(gitUsername, githubToken.getContent(), githubToken.getUsername()) : null;
         } else if (SourceControl.BITBUCKET.toString().equals(source)) {
             return bitbucketTokenContent != null ? new BitBucketSourceCodeRepo(gitUsername, bitbucketTokenContent) : null;
         } else if (SourceControl.GITLAB.toString().equals(source)) {

dockstore-webservice/src/main/java/io/dockstore/webservice/helpers/SourceCodeRepoInterface.java

@@ -68,6 +68,7 @@
 public abstract class SourceCodeRepoInterface {
     public static final Logger LOG = LoggerFactory.getLogger(SourceCodeRepoInterface.class);
     public static final int BYTES_IN_KB = 1024;
+    @Deprecated
     String gitUsername;
 
     /**

dockstore-webservice/src/main/java/io/dockstore/webservice/resources/AbstractWorkflowResource.java

@@ -134,11 +134,11 @@ protected SourceCodeRepoInterface getSourceCodeRepoInterface(String gitUrl, User
         List<Token> tokens = getAndRefreshTokens(user, tokenDAO, client, bitbucketClientID, bitbucketClientSecret);
 
         final String bitbucketTokenContent = getToken(tokens, TokenType.BITBUCKET_ORG);
-        final String gitHubTokenContent = getToken(tokens, TokenType.GITHUB_COM);
+        Token gitHubToken = Token.extractToken(tokens, TokenType.GITHUB_COM);
         final String gitlabTokenContent = getToken(tokens, TokenType.GITLAB_COM);
 
         final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory
-            .createSourceCodeRepo(gitUrl, client, bitbucketTokenContent, gitlabTokenContent, gitHubTokenContent);
+            .createSourceCodeRepo(gitUrl, client, bitbucketTokenContent, gitlabTokenContent, gitHubToken);
         if (sourceCodeRepo == null) {
             throw new CustomWebApplicationException("Git tokens invalid, please re-link your git accounts.", HttpStatus.SC_BAD_REQUEST);
         }

dockstore-webservice/src/main/java/io/dockstore/webservice/resources/DockerRepoResource.java

@@ -270,7 +270,7 @@ private Tool refreshContainer(final long containerId, final long userId) {
 
         final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory
             .createSourceCodeRepo(tool.getGitUrl(), client, bitbucketToken == null ? null : bitbucketToken.getContent(),
-                gitlabToken == null ? null : gitlabToken.getContent(), githubToken == null ? null : githubToken.getContent());
+                gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
 
         // Get all registries
         ImageRegistryFactory factory = new ImageRegistryFactory(quayToken);
@@ -589,7 +589,7 @@ private void setToolLicenseInformation(User user, Tool tool) {
         Token bitbucketToken = Token.extractToken(tokens, TokenType.BITBUCKET_ORG);
         final SourceCodeRepoInterface sourceCodeRepo = SourceCodeRepoFactory
                 .createSourceCodeRepo(tool.getGitUrl(), bitbucketToken == null ? null : bitbucketToken.getContent(),
-                        gitlabToken == null ? null : gitlabToken.getContent(), githubToken == null ? null : githubToken.getContent());
+                        gitlabToken == null ? null : gitlabToken.getContent(), githubToken);
         if (sourceCodeRepo != null) {
             sourceCodeRepo.checkSourceCodeValidity();
             String gitRepositoryFromGitUrl = AbstractImageRegistry.getGitRepositoryFromGitUrl(tool.getGitUrl());

dockstore-webservice/src/main/java/io/dockstore/webservice/resources/MetadataResource.java

@@ -358,7 +358,7 @@ public List<DescriptorLanguage.DescriptorLanguageBean> getDescriptorLanguages()
     @ApiResponse(description = "Cache performance information", content = @Content(mediaType = "application/json"))
     @ApiOperation(value = "Get measures of cache performance.", notes = "NO authentication", response = Map.class)
     public Map<String, String> getCachePerformance() {
-        Cache cache = DockstoreWebserviceApplication.getCache();
+        Cache cache = DockstoreWebserviceApplication.getCache(null);
         Map<String, String> results = new HashMap<>();
         results.put("requestCount", String.valueOf(cache.requestCount()));
         results.put("networkCount", String.valueOf(cache.networkCount()));