Add multi-platform container builds for amd64 and arm64

VERSION

@@ -1 +1 @@
-1.32.5
+1.33.0-B1

src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy

@@ -58,6 +58,16 @@ interface RegistryAuthService {
      */
     String getAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException
 
+    /**
+     * Get the authorization header for push operations (pull,push scope).
+     *
+     * @param image The image name for which the authorisation is needed
+     * @param auth The {@link RegistryAuth} information modelling the target registry
+     * @param creds The user credentials
+     * @return The authorization header including the 'Basic' or 'Bearer' prefix
+     */
+    String getAuthorizationForPush(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException
+
     /**
      * Invalidate a cached authorization token
      *

src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy

@@ -110,6 +110,15 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
 
     private LoadingCache<CacheKey, String> cacheTokens
 
+    private CacheLoader<CacheKey, String> pushLoader = new CacheLoader<CacheKey, String>() {
+        @Override
+        String load(CacheKey key) throws Exception {
+            return getPushToken(key)
+        }
+    }
+
+    private LoadingCache<CacheKey, String> cachePushTokens
+
     @Inject
     private RegistryLookupService lookupService
 
@@ -124,6 +133,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
                 .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS)
                 .executor(ioExecutor)
                 .build(loader)
+
+        cachePushTokens = Caffeine
+                .newBuilder()
+                .maximumSize(1_000)
+                .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS)
+                .executor(ioExecutor)
+                .build(pushLoader)
     }
 
     /**
@@ -215,23 +231,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
      */
     @Override
     String getAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException {
-        if( !auth )
-            throw new RegistryUnauthorizedAccessException("Missing authentication credentials")
-
-        if( !auth.type )
-            return null
-
-        if( auth.type == RegistryAuth.Type.Bearer ) {
-            final token = getAuthToken(image, auth, creds)
-            return "Bearer $token"
-        }
-
-        if( auth.type == RegistryAuth.Type.Basic ) {
-            final String basic = creds ? "$creds.username:$creds.password".bytes.encodeBase64() : null
-            return basic ? "Basic $basic" : null
-        }
-
-        throw new RegistryUnauthorizedAccessException("Unknown authentication type: $auth.type")
+        return getAuthorization0(image, auth, creds, false)
     }
 
     /**
@@ -242,9 +242,12 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
      * @return The resulting bearer token to authorise a pull request
      */
     protected String getToken0(CacheKey key) {
+        return fetchToken(buildLoginUrl(key.auth.realm, key.image, key.auth.service), key.creds)
+    }
+
+    private String fetchToken(String login, RegistryCredentials creds) {
         final httpClient = HttpClientFactory.followRedirectsHttpClient()
-        final login = buildLoginUrl(key.auth.realm, key.image, key.auth.service)
-        final req = makeRequest(login, key.creds)
+        final req = makeRequest(login, creds)
         log.trace "Token request=$req"
 
         // retry strategy
@@ -272,18 +275,30 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         throw new RegistryForwardException("Unexpected response acquiring token for '$login' [${response.statusCode()}]", response)
     }
 
-    String buildLoginUrl(URI realm, String image, String service){
-        String result = "${realm}?scope=repository:${image}:pull"
-        if(service) {
+    String buildLoginUrl(URI realm, String image, String service, String scope='pull') {
+        String result = "${realm}?scope=repository:${image}:${scope}"
+        if( service ) {
             result += "&service=$service"
         }
         return result
     }
 
     protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) {
+        return getCachedToken(cacheTokens, image, auth, creds)
+    }
+
+    protected String getPushToken(CacheKey key) {
+        return fetchToken(buildLoginUrl(key.auth.realm, key.image, key.auth.service, 'pull,push'), key.creds)
+    }
+
+    protected String getPushAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) {
+        return getCachedToken(cachePushTokens, image, auth, creds)
+    }
+
+    private String getCachedToken(LoadingCache<CacheKey, String> cache, String image, RegistryAuth auth, RegistryCredentials creds) {
         final key = new CacheKey(image, auth, creds)
         try {
-            return cacheTokens.get(key)
+            return cache.get(key)
         }
         catch (CompletionException e) {
             // this catches the exception thrown in the cache loader lookup
@@ -292,6 +307,31 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
         }
     }
 
+    @Override
+    String getAuthorizationForPush(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException {
+        return getAuthorization0(image, auth, creds, true)
+    }
+
+    private String getAuthorization0(String image, RegistryAuth auth, RegistryCredentials creds, boolean push) {
+        if( !auth )
+            throw new RegistryUnauthorizedAccessException("Missing authentication credentials")
+
+        if( !auth.type )
+            return null
+
+        if( auth.type == RegistryAuth.Type.Bearer ) {
+            final token = push ? getPushAuthToken(image, auth, creds) : getAuthToken(image, auth, creds)
+            return "Bearer $token"
+        }
+
+        if( auth.type == RegistryAuth.Type.Basic ) {
+            final String basic = creds ? "$creds.username:$creds.password".bytes.encodeBase64() : null
+            return basic ? "Basic $basic" : null
+        }
+
+        throw new RegistryUnauthorizedAccessException("Unknown authentication type: $auth.type")
+    }
+
     /**
      * Invalidate a cached authorization token
      *
@@ -302,6 +342,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService {
     void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) {
         final key = new CacheKey(image, auth, creds)
         cacheTokens.invalidate(key)
+        cachePushTokens.invalidate(key)
         tokenStore.remove(getStableKey(key))
     }
 

src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy

@@ -152,6 +152,9 @@ class BuildConfig {
     @Value('${wave.build.logs.maxLength:100000}')
     long maxLength
 
+    @Value('${wave.build.skip-cache:false}')
+    boolean skipCache
+
     @PostConstruct
     private void init() {
         log.info("Builder config: " +

src/main/groovy/io/seqera/wave/controller/ContainerController.groovy

@@ -63,6 +63,7 @@ import io.seqera.wave.service.builder.BuildRequest
 import io.seqera.wave.service.builder.BuildTrack
 import io.seqera.wave.service.builder.ContainerBuildService
 import io.seqera.wave.service.builder.FreezeService
+import io.seqera.wave.service.builder.MultiPlatformBuildService
 import io.seqera.wave.service.inclusion.ContainerInclusionService
 import io.seqera.wave.service.inspect.ContainerInspectService
 import io.seqera.wave.service.mirror.ContainerMirrorService
@@ -76,6 +77,7 @@ import io.seqera.wave.service.request.ContainerRequestService
 import io.seqera.wave.service.request.ContainerStatusService
 import io.seqera.wave.service.request.TokenData
 import io.seqera.wave.service.scan.ContainerScanService
+import io.seqera.wave.core.ChildRefs
 import io.seqera.wave.service.validation.ValidationService
 import io.seqera.wave.service.validation.ValidationServiceImpl
 import io.seqera.wave.tower.PlatformId
@@ -92,6 +94,7 @@ import static io.seqera.wave.util.ContainerHelper.condaFileFromRequest
 import static io.seqera.wave.util.ContainerHelper.containerFileFromRequest
 import static io.seqera.wave.util.ContainerHelper.decodeBase64OrFail
 import static io.seqera.wave.util.ContainerHelper.makeContainerId
+import static io.seqera.wave.util.ContainerHelper.makeMultiPlatformContainerId
 import static io.seqera.wave.util.ContainerHelper.makeResponseV1
 import static io.seqera.wave.util.ContainerHelper.makeResponseV2
 import static io.seqera.wave.util.ContainerHelper.makeTargetImage
@@ -178,6 +181,10 @@ class ContainerController {
     @Nullable
     private ContainerScanService scanService
 
+    @Inject
+    @Nullable
+    private MultiPlatformBuildService multiPlatformBuildService
+
     @PostConstruct
     private void init() {
         log.info "Wave server url: $serverUrl; allowAnonymous: $allowAnonymous; tower-endpoint-url: $towerEndpointUrl; default-build-repo: ${buildConfig?.defaultBuildRepository}; default-cache-repo: ${buildConfig?.defaultCacheRepository}; default-public-repo: ${buildConfig?.defaultPublicRepository}"
@@ -258,10 +265,23 @@ class ContainerController {
             final generated = containerFileFromRequest(req)
             req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString())
         }
-        // make sure container platform is defined 
+        // make sure container platform is defined
         if( !req.containerPlatform )
             req.containerPlatform = ContainerPlatform.DEFAULT.toString()
 
+        // multi-platform validation
+        final parsedPlatform = ContainerPlatform.of(req.containerPlatform)
+        if( parsedPlatform.isMultiArch() ) {
+            if( parsedPlatform != ContainerPlatform.MULTI_PLATFORM )
+                throw new BadRequestException("Only linux/amd64,linux/arm64 multi-platform combination is currently supported")
+            if( !req.containerFile && !req.packages )
+                throw new BadRequestException("Multi-platform builds require either 'containerFile' or 'packages' attribute")
+            if( req.formatSingularity() )
+                throw new BadRequestException("Multi-platform builds are not supported for Singularity format")
+            if( !multiPlatformBuildService )
+                throw new UnsupportedBuildServiceException()
+        }
+
         final ip = addressResolver.resolve(httpRequest)
         // check the rate limit before continuing
         if( rateLimiterService )
@@ -388,6 +408,21 @@ class ContainerController {
         )
     }
 
+    protected ChildRefs makeChildScanIds(BuildRequest build, SubmitContainerTokenRequest req) {
+        if( !scanService || !build.platform.isMultiArch() )
+            return null
+        final multiPlatform = build.platform
+        final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async
+        final scanIdByPlatform = new LinkedHashMap<String, String>()
+        for( ContainerPlatform.Platform p : multiPlatform.platforms ) {
+            final platform = p.toString()
+            final id = scanService.getScanId("${build.targetImage}#${platform}", null, scanMode, req.format)
+            if( id )
+                scanIdByPlatform.put(id, platform)
+        }
+        return ChildRefs.of(scanIdByPlatform)
+    }
+
     protected BuildTrack checkBuild(BuildRequest build, boolean dryRun) {
         final digest = registryProxyService.getImageDigest(build)
         // check for dry-run execution
@@ -408,6 +443,37 @@ class ContainerController {
         }
     }
 
+    protected BuildTrack checkMultiPlatformBuild(BuildRequest templateBuild, SubmitContainerTokenRequest req, PlatformId identity, boolean dryRun) {
+        final containerSpec = templateBuild.containerFile
+        final condaContent = templateBuild.condaFile
+        final buildRepository = ContainerCoordinates.parse(templateBuild.targetImage).repository
+
+        // compute multi-platform container ID and target image
+        final containerId = makeMultiPlatformContainerId(containerSpec, condaContent, buildRepository, req.buildContext, req.freeze ? req.containerConfig : null)
+        final targetImage = makeTargetImage(templateBuild.format, buildRepository, containerId, condaContent, req.nameStrategy)
+
+        // check for dry-run execution
+        if( dryRun ) {
+            log.debug "== Dry-run multi-platform build request for $targetImage"
+            final dryId = containerId + BuildRequest.SEP + '0'
+            final digest = registryProxyService.getImageDigest(targetImage, identity)
+            return new BuildTrack(dryId, targetImage, digest!=null, true)
+        }
+
+        // check if the multi-platform image already exists
+        if( !buildConfig.skipCache ) {
+            final digest = registryProxyService.getImageDigest(targetImage, identity)
+            if( digest ) {
+                log.debug "== Found cached multi-platform build for $targetImage"
+                final cache = persistenceService.loadBuildSucceed(targetImage, digest)
+                return new BuildTrack(cache?.buildId, targetImage, true, true)
+            }
+        }
+
+        // delegate to multi-platform build service
+        return multiPlatformBuildService.buildMultiPlatformImage(templateBuild, containerId, targetImage, identity)
+    }
+
     protected String getContainerDigest(String containerImage, PlatformId identity) {
         containerImage
                 ? registryProxyService.getImageDigest(containerImage, identity)
@@ -446,8 +512,26 @@ class ContainerController {
         String buildId
         boolean buildNew
         String scanId
+        ChildRefs scanChildIds = null
         Boolean succeeded
-        if( req.containerFile ) {
+        if( req.containerFile && ContainerPlatform.of(req.containerPlatform).isMultiArch() ) {
+            if( !buildService ) throw new UnsupportedBuildServiceException()
+            final build0 = makeBuildRequest(req, identity, ip)
+            // create per-platform scan IDs for multi-arch builds
+            final childScans = makeChildScanIds(build0, req)
+            final build = childScans ? build0.withChildScanIds(childScans) : build0
+            final track = checkMultiPlatformBuild(build, req, identity, req.dryRun)
+            targetImage = track.targetImage
+            targetContent = build.containerFile
+            condaContent = build.condaFile
+            buildId = track.id
+            buildNew = !track.cached
+            scanId = build.scanId
+            scanChildIds = build.scanChildIds
+            succeeded = track.succeeded
+            type = ContainerRequest.Type.Build
+        }
+        else if( req.containerFile ) {
             if( !buildService ) throw new UnsupportedBuildServiceException()
             final build = makeBuildRequest(req, identity, ip)
             final track = checkBuild(build, req.dryRun)
@@ -501,6 +585,7 @@ class ContainerController {
                 buildNew,
                 req.freeze,
                 scanId,
+                scanChildIds,
                 req.scanMode,
                 req.scanLevels,
                 req.dryRun,

src/main/groovy/io/seqera/wave/controller/InspectController.groovy

@@ -32,6 +32,7 @@ import io.micronaut.scheduling.TaskExecutors
 import io.micronaut.scheduling.annotation.ExecuteOn
 import io.seqera.wave.api.ContainerInspectRequest
 import io.seqera.wave.api.ContainerInspectResponse
+import io.seqera.wave.core.ContainerPlatform
 import io.seqera.wave.exception.BadRequestException
 import io.seqera.wave.service.UserService
 import io.seqera.wave.service.inspect.ContainerInspectService
@@ -74,6 +75,9 @@ class InspectController {
         if( !req.containerImage )
             throw new BadRequestException("Missing 'containerImage' attribute")
 
+        // multi-platform values are not allowed for inspect requests
+        ContainerPlatform.validateSinglePlatform(platform)
+
         // this is needed for backward compatibility with old clients
         if( !req.towerEndpoint ) {
             req.towerEndpoint = towerEndpointUrl

src/main/groovy/io/seqera/wave/controller/ViewController.groovy

@@ -56,6 +56,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord
 import io.seqera.wave.service.persistence.WaveScanRecord
 import io.seqera.wave.service.scan.ContainerScanService
 import io.seqera.wave.service.scan.ScanEntry
+import io.seqera.wave.core.ChildRefs
 import io.seqera.wave.service.scan.ScanType
 import io.seqera.wave.service.scan.ScanVulnerability
 import io.seqera.wave.util.JacksonHelper
@@ -137,8 +138,7 @@ class ViewController {
         binding.mirror_digest = result.digest ?: '-'
         binding.mirror_user = result.userName ?: '-'
         binding.put('server_url', serverUrl)
-        binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null
-        binding.scan_id = result.scanId
+        ChildRefs.populateScanBinding(binding, result.scanId, null, result.succeeded(), serverUrl)
         return binding
     }
 
@@ -254,8 +254,8 @@ class ViewController {
         binding.build_condafile = result.condaFile
         binding.build_digest = result.digest ?: '-'
         binding.put('server_url', serverUrl)
-        binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null
-        binding.scan_id = result.scanId
+        ChildRefs.populateScanBinding(binding, result.scanId, result.scanChildIds, result.succeeded(), serverUrl)
+        ChildRefs.populateBuildBinding(binding, result.buildChildIds, serverUrl)
         // inspect uri
         binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null
         // configure build logs when available
@@ -314,8 +314,7 @@ class ViewController {
         binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : null
         binding.fusion_version = data.fusionVersion ?: '-'
 
-        binding.scan_id = data.scanId
-        binding.scan_url = data.scanId ? "$serverUrl/view/scans/${data.scanId}" : null
+        ChildRefs.populateScanBinding(binding, data.scanId, data.scanChildIds, true, serverUrl)
 
         binding.mirror_id = data.mirror ? data.buildId : null
         binding.mirror_url =  data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null