Skip to content

Commit

Permalink
Docker registry regex (#5139)
Browse files Browse the repository at this point in the history 
* fix(dynamicAccounts): Included java files in source set

* feat(docker/registry): Filter repositories using a regex

* feat(docker/registry): added restriction, regex may not be specified at the same time than repositories or catalogFile

* feat(docker/registry): replace property name from regex to repositoriesRegex

* feat(docker/registry): fix test cases and add a new one in order to Filtering by regular expression.

* feat(docker/registry): fix test case DockerRegistryClient uses correct user agent.

* feat(docker/registry): remove ignore annotation.

* feat(docker/registry): Modified test cases using a Stub with all the necessary calls

* feat(docker/registry): Modified the constructor for testing purposes, refactor test cases using new constructor.

Co-authored-by: German Muzquiz <german.muzquiz@armory.io>
  • Loading branch information
@jorgebee65
jorgebee65 and German Muzquiz committed Dec 22, 2020
1 parent 9bf5bba commit 1060024
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 25 deletions.
40 changes: 35 additions & 5 deletions ...m/netflix/spinnaker/clouddriver/docker/registry/api/v2/client/DockerRegistryClient.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class DockerRegistryClient {
long clientTimeoutMillis
int paginateSize
String catalogFile
String repositoriesRegex
boolean insecureRegistry
DockerOkClientProvider okClientProvider

Expand Down Expand Up @@ -105,6 +106,12 @@ class DockerRegistryClient {
return this
}

Builder repositoriesRegex(String regex) {
this.repositoriesRegex = regex
return this
}


Builder insecureRegistry(boolean insecureRegistry) {
this.insecureRegistry = insecureRegistry
return this
Expand All @@ -121,11 +128,11 @@ class DockerRegistryClient {
throw new IllegalArgumentException('Error, at most one of "password", "passwordFile", "passwordCommand" or "dockerconfigFile" can be specified')
}
if (password || passwordCommand) {
return new DockerRegistryClient(address, email, username, password, passwordCommand, clientTimeoutMillis, paginateSize, catalogFile, insecureRegistry, okClientProvider)
return new DockerRegistryClient(address, email, username, password, passwordCommand, clientTimeoutMillis, paginateSize, catalogFile, repositoriesRegex, insecureRegistry, okClientProvider)
} else if (passwordFile) {
return new DockerRegistryClient(address, email, username, passwordFile, clientTimeoutMillis, paginateSize, catalogFile, insecureRegistry, okClientProvider)
return new DockerRegistryClient(address, email, username, passwordFile, clientTimeoutMillis, paginateSize, catalogFile, repositoriesRegex, insecureRegistry, okClientProvider)
} else {
return new DockerRegistryClient(address, clientTimeoutMillis, paginateSize, catalogFile, insecureRegistry, okClientProvider)
return new DockerRegistryClient(address, clientTimeoutMillis, paginateSize, catalogFile, repositoriesRegex, insecureRegistry, okClientProvider)
}
}

Expand All @@ -140,6 +147,7 @@ class DockerRegistryClient {
DockerRegistryService registryService
GsonConverter converter
String catalogFile
String repositoriesRegex

final static String userAgent = DockerUserAgent.getUserAgent()
final int paginateSize
Expand All @@ -152,6 +160,7 @@ class DockerRegistryClient {
long clientTimeoutMillis,
int paginateSize,
String catalogFile,
String repositoriesRegex,
boolean insecureRegistry,
DockerOkClientProvider okClientProvider) {

Expand All @@ -167,6 +176,7 @@ class DockerRegistryClient {
this.converter = new GsonConverter(new GsonBuilder().create())
this.address = address
this.catalogFile = catalogFile
this.repositoriesRegex = repositoriesRegex
}

DockerRegistryClient(String address,
Expand All @@ -177,23 +187,40 @@ class DockerRegistryClient {
long clientTimeoutMillis,
int paginateSize,
String catalogFile,
String repositoriesRegex,
boolean insecureRegistry,
DockerOkClientProvider okClientProvider) {
this(address, clientTimeoutMillis, paginateSize, catalogFile, insecureRegistry, okClientProvider)
this(address, clientTimeoutMillis, paginateSize, catalogFile, repositoriesRegex, insecureRegistry, okClientProvider)
this.tokenService = new DockerBearerTokenService(username, password, passwordCommand)
this.email = email
}

DockerRegistryClient(String address,
int paginateSize,
String catalogFile,
String repositoriesRegex,
DockerRegistryService dockerRegistryService,
DockerBearerTokenService dockerBearerTokenService) {
this.paginateSize = paginateSize
this.converter = new GsonConverter(new GsonBuilder().create())
this.address = address
this.catalogFile = catalogFile
this.repositoriesRegex = repositoriesRegex
this.tokenService = dockerBearerTokenService
this.registryService = dockerRegistryService;
}

DockerRegistryClient(String address,
String email,
String username,
File passwordFile,
long clientTimeoutMillis,
int paginateSize,
String catalogFile,
String repositoriesRegex,
boolean insecureRegistry,
DockerOkClientProvider okClientProvider) {
this(address, clientTimeoutMillis, paginateSize, catalogFile, insecureRegistry, okClientProvider)
this(address, clientTimeoutMillis, paginateSize, catalogFile, repositoriesRegex, insecureRegistry, okClientProvider)
this.tokenService = new DockerBearerTokenService(username, passwordFile)
this.email = email
}
Expand Down Expand Up @@ -343,6 +370,9 @@ class DockerRegistryClient {
def nextPath = findNextLink(response?.headers)
def catalog = (DockerRegistryCatalog) converter.fromBody(response.body, DockerRegistryCatalog)

if(repositoriesRegex) {
catalog.repositories = catalog.repositories.findAll { it ==~ repositoriesRegex }
}
if (nextPath) {
def nextCatalog = getCatalog(nextPath)
catalog.repositories.addAll(nextCatalog.repositories)
Expand Down
2 changes: 2 additions & 0 deletions ...spinnaker/clouddriver/docker/registry/config/DockerRegistryConfigurationProperties.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ class DockerRegistryConfigurationProperties {
List<String> skip
// a file listing all repositories to index
String catalogFile
// Allow filter the repositories by a regular expression
String repositoriesRegex
}

List<ManagedAccount> accounts = []
Expand Down
3 changes: 2 additions & 1 deletion ...pinnaker/clouddriver/docker/registry/security/DockerRegistryCredentialsInitializer.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class DockerRegistryCredentialsInitializer {
AccountCredentialsRepository accountCredentialsRepository,
CatsModule catsModule,
DockerOkClientProvider dockerOkClientProvider) {

def (ArrayList<DockerRegistryConfigurationProperties.ManagedAccount> accountsToAdd, List<String> namesOfDeletedAccounts) =
ProviderUtils.calculateAccountDeltas(accountCredentialsRepository, DockerRegistryNamedAccountCredentials,
dockerRegistryConfigurationProperties.accounts)
Expand All @@ -69,6 +69,7 @@ class DockerRegistryCredentialsInitializer {
.email(managedAccount.email)
.passwordFile(managedAccount.passwordFile)
.catalogFile(managedAccount.catalogFile)
.repositoriesRegex(managedAccount.repositoriesRegex)
.dockerconfigFile(managedAccount.dockerconfigFile)
.cacheThreads(managedAccount.cacheThreads)
.cacheIntervalSeconds(managedAccount.cacheIntervalSeconds)
Expand Down
17 changes: 17 additions & 0 deletions ...innaker/clouddriver/docker/registry/security/DockerRegistryNamedAccountCredentials.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
List<String> repositories
List<String> skip
String catalogFile
String repositoriesRegex
DockerOkClientProvider dockerOkClientProvider

Builder() {}
Expand Down Expand Up @@ -166,6 +167,11 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
return this
}

Builder repositoriesRegex(String repositoriesRegex) {
this.repositoriesRegex = repositoriesRegex
return this
}

Builder dockerOkClientProvider(DockerOkClientProvider dockerOkClientProvider) {
this.dockerOkClientProvider = dockerOkClientProvider
return this
Expand All @@ -191,6 +197,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
trackDigests,
sortTagsByDate,
catalogFile,
repositoriesRegex,
insecureRegistry,
dockerOkClientProvider)
}
Expand All @@ -215,6 +222,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
boolean trackDigests,
boolean sortTagsByDate,
String catalogFile,
String repositoriesRegex,
boolean insecureRegistry,
DockerOkClientProvider dockerOkClientProvider) {
this(accountName,
Expand All @@ -236,6 +244,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
trackDigests,
sortTagsByDate,
catalogFile,
repositoriesRegex,
insecureRegistry,
null,
dockerOkClientProvider)
Expand All @@ -260,6 +269,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
boolean trackDigests,
boolean sortTagsByDate,
String catalogFile,
String repositoriesRegex,
boolean insecureRegistry,
List<String> requiredGroupMembership,
DockerOkClientProvider dockerOkClientProvider) {
Expand All @@ -271,6 +281,10 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
throw new IllegalArgumentException("repositories and catalogFile may not be specified together.")
}

if(repositoriesRegex && (repositories || catalogFile)){
throw new IllegalArgumentException("repositoriesRegex may not be specified at the same time than repositories or catalogFile.")
}

this.accountName = accountName
this.environment = environment
this.accountType = accountType
Expand Down Expand Up @@ -307,6 +321,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
this.username = username
this.password = password
this.email = email
this.repositoriesRegex = repositoriesRegex
this.trackDigests = trackDigests
this.sortTagsByDate = sortTagsByDate
this.insecureRegistry = insecureRegistry;
Expand Down Expand Up @@ -371,6 +386,7 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
.clientTimeoutMillis(clientTimeoutMillis)
.paginateSize(paginateSize)
.catalogFile(catalogFile)
.repositoriesRegex(repositoriesRegex)
.insecureRegistry(insecureRegistry)
.okClientProvider(dockerOkClientProvider)
.build()
Expand Down Expand Up @@ -410,5 +426,6 @@ class DockerRegistryNamedAccountCredentials extends AbstractAccountCredentials<D
final List<String> requiredGroupMembership
final List<String> skip
final String catalogFile
final String repositoriesRegex
final DockerOkClientProvider dockerOkClientProvider
}
70 changes: 51 additions & 19 deletions ...tflix/spinnaker/clouddriver/docker/registry/api/v2/client/DockerRegistryClientSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@

package com.netflix.spinnaker.clouddriver.docker.registry.api.v2.client

import spock.lang.Ignore
import com.netflix.spinnaker.clouddriver.docker.registry.api.v2.auth.DockerBearerTokenService
import retrofit.client.Response
import retrofit.mime.TypedByteArray
import retrofit.mime.TypedInput
import spock.lang.Shared
import spock.lang.Specification

Expand All @@ -27,57 +30,86 @@ import java.util.concurrent.TimeUnit
* with an exception indicating a network or HTTP error, or will fail to load data
* from dockerhub.
*/
@Ignore
class DockerRegistryClientSpec extends Specification {
private static final REPOSITORY1 = "library/ubuntu"

@Shared
DockerRegistryClient client
def dockerBearerTokenService = Mock(DockerBearerTokenService)

def stubbedRegistryService = Stub(DockerRegistryClient.DockerRegistryService){
String tagsJson = "{\"name\":\"library/ubuntu\",\"tags\":[\"latest\",\"xenial\",\"rolling\"]}"
TypedInput tagsTypedInput = new TypedByteArray("application/json", tagsJson.getBytes())
Response tagsResponse = new Response("/v2/{repository}/tags/list",200, "nothing", Collections.EMPTY_LIST, tagsTypedInput)
getTags(_,_,_) >> tagsResponse

String checkJson = "{}"
TypedInput checkTypedInput = new TypedByteArray("application/json", checkJson.getBytes())
Response checkResponse = new Response("/v2/",200, "nothing", Collections.EMPTY_LIST, checkTypedInput)
checkVersion(_,_) >> checkResponse

String json = "{\"repositories\":[\"armory-io/armorycommons\",\"armory/aquascan\",\"other/keel\"]}"
TypedInput catalogTypedInput = new TypedByteArray("application/json", json.getBytes())
Response catalogResponse = new Response("/v2/_catalog/",200, "nothing", Collections.EMPTY_LIST, catalogTypedInput)
getCatalog(_,_,_) >> catalogResponse
}

def setupSpec() {

}

void "DockerRegistryClient should request a real set of tags."() {
when:
client = new DockerRegistryClient("https://index.docker.io", "", "", "", TimeUnit.MINUTES.toMillis(1), 100, "", false)
DockerRegistryTags result = client.getTags(REPOSITORY1)
client = new DockerRegistryClient("https://index.docker.io",100,"","",stubbedRegistryService, dockerBearerTokenService)
def result = client.getTags(REPOSITORY1)

then:
result.name == REPOSITORY1
result.tags.size() > 0
result.name == REPOSITORY1
result.tags.size() > 0
}

void "DockerRegistryClient should validate that it is pointing at a v2 endpoint."() {
when:
client = new DockerRegistryClient("https://index.docker.io", "", "", "", TimeUnit.MINUTES.toMillis(1), 100, "", false)
// Can only fail due to an exception thrown here.
client.checkV2Availability()
client = new DockerRegistryClient("https://index.docker.io",100,"","",stubbedRegistryService, dockerBearerTokenService)
// Can only fail due to an exception thrown here.
client.checkV2Availability()

then:
true
true
}

void "DockerRegistryClient invoked with insecureRegistry=true"() {
when:
client = new DockerRegistryClient("https://index.docker.io", "", "", "", TimeUnit.MINUTES.toMillis(1), 100, "", true)
DockerRegistryTags result = client.getTags(REPOSITORY1)
client = new DockerRegistryClient("https://index.docker.io",100,"","",stubbedRegistryService, dockerBearerTokenService)
DockerRegistryTags result = client.getTags(REPOSITORY1)

then:
result.name == REPOSITORY1
result.tags.size() > 0
result.name == REPOSITORY1
result.tags.size() > 0
}

void "DockerRegistryClient uses correct user agent"() {
when:
client = new DockerRegistryClient("https://index.docker.io", "", "", "", TimeUnit.MINUTES.toMillis(1), 100, "", true)
client.registryService = Mock(DockerRegistryClient.DockerRegistryService)
def mockService = Mock(DockerRegistryClient.DockerRegistryService);
client = new DockerRegistryClient("https://index.docker.io",100,"","",mockService, dockerBearerTokenService)

when:
client.checkV2Availability()
def userAgent = client.userAgent
client.getTags(REPOSITORY1)

then:
userAgent.startsWith("Spinnaker")
1 * client.registryService.getTags(_, _, userAgent)
1 * mockService.checkVersion(_,_)
}

void "DockerRegistryClient should filter repositories by regular expression."() {
when:
client = new DockerRegistryClient("https://index.docker.io",100,"","",stubbedRegistryService, dockerBearerTokenService)
def original = client.getCatalog().repositories.size()
client = new DockerRegistryClient("https://index.docker.io",100,"","armory\\/.*",stubbedRegistryService, dockerBearerTokenService)
def filtered = client.getCatalog().repositories.size()

then:
filtered < original
}

}

0 comments on commit 1060024

Please sign in to comment.