Skip to content
Permalink
Browse files

scanner: Use the generic file storage to store scan results

Remove the specific implementations for `ArtifactoryStorage` and
`LocalFileStorage` and let `FileBasedStorage` instead use a `FileStorage`
from the utils module.

This adds more flexibility as any implementation of `FileStorage` can
now be used for storing scan results.

Signed-off-by: Martin Nonnenmacher <martin.nonnenmacher@here.com>
  • Loading branch information...
mnonnenmacher committed Oct 1, 2019
1 parent b8ef207 commit da7e3a814fc0e6301bf3ed394eba1a661e4d88d7
Showing with 278 additions and 558 deletions.
  1. +18 −10 README.md
  2. +7 −6 cli/src/main/kotlin/commands/ScannerCommand.kt
  3. +32 −0 model/src/main/kotlin/config/FileBasedStorageConfiguration.kt
  4. +10 −14 model/src/main/kotlin/config/{ArtifactoryStorageConfiguration.kt → FileStorageConfiguration.kt}
  5. +41 −0 model/src/main/kotlin/config/HttpFileStorageConfiguration.kt
  6. +2 −10 model/src/main/kotlin/config/ScannerConfiguration.kt
  7. +13 −7 model/src/main/resources/reference.conf
  8. +0 −2 model/src/test/kotlin/ScannerRunTest.kt
  9. +0 −73 model/src/test/kotlin/config/ArtifactoryStorageConfigurationTest.kt
  10. +41 −0 model/src/test/kotlin/config/HttpFileStorageConfigurationTest.kt
  11. +15 −9 model/src/test/kotlin/config/OrtConfigurationTest.kt
  12. +1 −2 reporter/src/funTest/assets/NPM-is-windows-1.0.2-scan-result.json
  13. +1 −2 reporter/src/funTest/assets/npm-test-with-exclude-scan-results.yml
  14. +1 −2 reporter/src/funTest/assets/npm-test-without-exclude-scan-results.yml
  15. +1 −2 reporter/src/funTest/assets/static-html-reporter-test-input.yml
  16. +1 −2 scanner/src/funTest/assets/file-counter-expected-output-for-analyzer-result.yml
  17. +0 −77 scanner/src/funTest/kotlin/storages/ArtifactoryStorageTest.kt
  18. +5 −4 scanner/src/funTest/kotlin/storages/{LocalFileStorageTest.kt → FileBasedStorageTest.kt}
  19. +16 −26 scanner/src/main/kotlin/ScanResultsStorage.kt
  20. +0 −136 scanner/src/main/kotlin/storages/ArtifactoryStorage.kt
  21. +73 −1 scanner/src/main/kotlin/storages/FileBasedStorage.kt
  22. +0 −83 scanner/src/main/kotlin/storages/LocalFileStorage.kt
  23. +0 −90 scanner/src/test/kotlin/ScanResultsStorageTest.kt
@@ -235,31 +235,39 @@ file (`-c`) that contains a respective local file storage configuration:
```hocon
ort {
scanner {
localFileStorage {
directory = "/tmp/ort/scan-results"
fileBasedStorage {
backend {
localFileStorage {
directory = "/tmp/ort/scan-results"
}
}
}
}
}
```

### Artifactory Storage
### HTTP Storage

To use Artifactory to store scan results, use the following configuration:
Any HTTP file server can be used to store scan results. Custom headers can be configured to provide authentication
credentials. For example, to use Artifactory to store scan results, use the following configuration:

```hocon
ort {
scanner {
artifactoryStorage {
url = "https://artifactory.domain.com/artifactory"
repository = "generic-repository-name"
apiToken = "api-token"
fileBasedStorage {
backend {
httpFileStorage {
url = "https://artifactory.domain.com/artifactory/repository/scan-results"
headers {
X-JFrog-Art-Api = "api-token"
}
}
}
}
}
}
```

The scanner creates a directory "scan-results" in the configured repository and uses it to store scan results.

### PostgreSQL Storage

To use PostgreSQL to store scan results, use the following configuration:
@@ -36,13 +36,14 @@ import com.here.ort.scanner.Scanner
import com.here.ort.scanner.ScannerFactory
import com.here.ort.scanner.TOOL_NAME
import com.here.ort.scanner.scanners.ScanCode
import com.here.ort.scanner.storages.LocalFileStorage
import com.here.ort.scanner.storages.FileBasedStorage
import com.here.ort.scanner.storages.SCAN_RESULTS_FILE_NAME
import com.here.ort.utils.PARAMETER_ORDER_MANDATORY
import com.here.ort.utils.PARAMETER_ORDER_OPTIONAL
import com.here.ort.utils.expandTilde
import com.here.ort.utils.getUserOrtDirectory
import com.here.ort.utils.log
import com.here.ort.utils.storage.LocalFileStorage

import java.io.File

@@ -117,12 +118,12 @@ object ScannerCommand : CommandWithHelp() {

// By default use a file based scan results storage.
val localFileStorage = LocalFileStorage(getUserOrtDirectory().resolve(TOOL_NAME))
ScanResultsStorage.storage = localFileStorage
val fileBasedStorage = FileBasedStorage(localFileStorage)
ScanResultsStorage.storage = fileBasedStorage

// Allow to override the default scan results storage.
val configuredStorages = listOfNotNull(
config.artifactoryStorage,
config.localFileStorage,
config.fileBasedStorage,
config.postgresStorage
)

@@ -138,7 +139,7 @@ object ScannerCommand : CommandWithHelp() {

val localFileStorageLogFunction: ((String) -> Unit)? = when {
// If the local file storage is in use, log about it already at info level.
log.delegate.isInfoEnabled && ScanResultsStorage.storage == localFileStorage -> log::info
log.delegate.isInfoEnabled && ScanResultsStorage.storage == fileBasedStorage -> log::info

// Otherwise log about the local file storage only at debug level.
log.delegate.isDebugEnabled -> log::debug
@@ -147,7 +148,7 @@ object ScannerCommand : CommandWithHelp() {
}

if (localFileStorageLogFunction != null) {
val fileCount = localFileStorage.scanResultsDirectory.walk().filter {
val fileCount = localFileStorage.directory.walk().filter {
it.isFile && it.name == SCAN_RESULTS_FILE_NAME
}.count()

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2019 HERE Europe B.V.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.model.config

import com.here.ort.utils.storage.FileStorage

/**
* The configuration model of a file based storage.
*/
data class FileBasedStorageConfiguration(
/**
* The configuration of the [FileStorage] used to store the files.
*/
val backend: FileStorageConfiguration
)
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 HERE Europe B.V.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,25 +19,21 @@

package com.here.ort.model.config

import com.fasterxml.jackson.annotation.JsonProperty
import com.here.ort.utils.storage.FileStorage
import com.here.ort.utils.storage.HttpFileStorage
import com.here.ort.utils.storage.LocalFileStorage

/**
* A class to hold the configuration for using Artifactory as a storage.
* The configuration model for a [FileStorage]. Only one of the storage options can be configured.
*/
data class ArtifactoryStorageConfiguration(
data class FileStorageConfiguration(
/**
* The URL of the Artifactory server, e.g. "https://example.com/artifactory".
* The configuration of a [HttpFileStorage].
*/
val url: String,
val httpFileStorage: HttpFileStorageConfiguration? = null,

/**
* The name of the Artifactory repository to use as a storage.
* The configuration of a [LocalFileStorage].
*/
val repository: String,

/**
* An Artifactory API token with read/write access to [repository].
*/
@JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
val apiToken: String = ""
val localFileStorage: LocalFileStorageConfiguration? = null
)
@@ -0,0 +1,41 @@
/*
* Copyright (C) 2019 HERE Europe B.V.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.model.config

import com.fasterxml.jackson.databind.annotation.JsonSerialize
import com.fasterxml.jackson.databind.util.StdConverter

data class HttpFileStorageConfiguration(
/**
* The URL of the HTTP server, e.g. "https://example.com/storage".
*/
val url: String,

/**
* Custom headers that are added to all HTTP requests. As headers are likely to contain sensitive information like
* credentials, values are masked when this class is serialized with Jackson.
*/
@JsonSerialize(contentConverter = MaskStringConverter::class)
val headers: Map<String, String>
)

class MaskStringConverter : StdConverter<String, String>() {
override fun convert(value: String) = "***"
}
@@ -19,22 +19,14 @@

package com.here.ort.model.config

import com.fasterxml.jackson.annotation.JsonAlias

/**
* The configuration model of the scanner.
*/
data class ScannerConfiguration(
/**
* Configuration of the Artifactory scan results storage.
*/
@JsonAlias("artifactory_cache")
val artifactoryStorage: ArtifactoryStorageConfiguration? = null,

/**
* Configuration of the local file scan results storage.
* Configuration of a file based storage.
*/
val localFileStorage: LocalFileStorageConfiguration? = null,
val fileBasedStorage: FileBasedStorageConfiguration? = null,

/**
* Configuration of the PostgreSQL scan results storage.
@@ -2,14 +2,20 @@
// exclusive, so this file is only used to show all options and to validate the configuration.
ort {
scanner {
artifactoryStorage {
url = "https://your-artifactory-server"
repository = repository
apiToken = apiToken
}
fileBasedStorage {
backend {
httpFileStorage {
url = "https://your-http-server"
headers {
key1 = "value1"
key2 = "value2"
}
}

localFileStorage {
directory = ~/.ort/scanner
localFileStorage {
directory = ~/.ort/scanner
}
}
}

postgresStorage {
@@ -35,7 +35,6 @@ class ScannerRunTest : StringSpec() {
os: "Linux"
tool_versions: {}
config:
artifactory_storage: null
scanner: null
results:
scanned_scopes: []
@@ -61,7 +60,6 @@ class ScannerRunTest : StringSpec() {
os: "Linux"
tool_versions: {}
config:
artifactory_storage: null
scanner: null
results:
scanned_scopes: []

This file was deleted.

0 comments on commit da7e3a8

Please sign in to comment.
You can’t perform that action at this time.