From 99cfa5f70c41352d751a8d52d6cf2edfab1ff8d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Thu, 25 Feb 2021 18:19:47 +0100 Subject: [PATCH 01/13] Start to include sanitization --- README.md | 2 ++ src/main/resources/config.ini | 9 ++++++--- src/main/scala/gatling/Config.scala | 6 ++++++ src/main/scala/gatling/ScanSimulation.scala | 14 +++++++++----- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e88bcb7..fdbd216 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ ApiKey: OPSWAT MetaDefender Cloud apikey WaitBeforePolling: waiting time between push file and start to polling (ms) ``` +TODO + To run the jar (in the `target` folder where the `.jar` file is generated): java -cp metadefender-gatling-1.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index 1da62f7..2b9c56a 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -1,16 +1,19 @@ [general] BaseUrl=https://api.metadefender.com/v4/file ;constant concurrent users -ConstantUsers=5 +ConstantUsers=1 ;time to run, (s) -TestDuration=10 +TestDuration=1 ;time before full filling, (s) ScanWorkflow=multiscan ;dataset folder -LocalPath=C:\OPSWAT\data +LocalPath=/home/aniko.bartos/Documents/tester_files ;sleep time between each polling scan result (ms) PollingIntervals=500 ;OPSWAT MetaDefender Cloud apikey ApiKey=opswat_metadefender_api ;waiting time between push file and start to polling (ms) WaitBeforePolling=1000 +SilentScan=true +Scan=true +Sanitization=true \ No newline at end of file diff --git a/src/main/scala/gatling/Config.scala b/src/main/scala/gatling/Config.scala index 4b1d9b6..49ab6fc 100644 --- a/src/main/scala/gatling/Config.scala +++ b/src/main/scala/gatling/Config.scala @@ -14,6 +14,9 @@ class Config { var localPath = "" var apikey = "" var waitBeforePolling = 1000 + var silentScan = true + var scan = true + var sanitization = true } object Config { @@ -29,6 +32,9 @@ object Config { v.pollingIntervals = ini.get("general", "PollingIntervals", classOf[Int]) v.apikey = ini.get("general", "ApiKey", classOf[String]) v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int]) + v.silentScan = ini.get("general", "SilentScan", classOf[Boolean]) + v.scan = ini.get("general", "Scan", classOf[Boolean]) + v.sanitization = ini.get("general", "Sanitization", classOf[Boolean]) v } } diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index 8695691..e785372 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -4,6 +4,8 @@ import io.gatling.core.body.RawFileBody import io.gatling.core.structure.{ChainBuilder, ScenarioBuilder} import io.gatling.http.Predef._ import io.gatling.http.protocol.HttpProtocolBuilder +import io.gatling.http.request.builder.HttpRequestBuilder + import scala.concurrent.duration._ @@ -34,11 +36,11 @@ class ScanSimulation extends Simulation { http("get-scan-result") .get(config.baseUrl + "/${dataId}") .header("apikey", config.apikey) - .silent .check(status.is(200)) .check(jsonPath("$..process_info.progress_percentage").find.saveAs("progress")) .check(jsonPath("$..process_info.post_processing.actions_ran").optional.saveAs("sanitization")) + val action: ChainBuilder = exec(_.set("progress", "0")) .doIf(session => session("dataId").asOption[String].isDefined) { @@ -49,7 +51,7 @@ class ScanSimulation extends Simulation { } - + val pipeline: ScenarioBuilder = scenario("scan-pipeline") .feed(localFiles.feeder) .exec(Scan.submitFile) .pause(config.waitBeforePolling.milliseconds) - .exec(ScanProgress.action) - //.exec(GetSanitized.action) + .doIf(config.scan){ + exec(ScanProgress.action) + } + .doIf(config.sanitization){GetSanitized.action} setUp( pipeline From 47dd33f6cd669249008ba97b6c58abd19c6c3bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Fri, 26 Feb 2021 20:11:57 +0100 Subject: [PATCH 02/13] Separate sanitization and scan --- src/main/resources/config.ini | 4 +- src/main/scala/gatling/Config.scala | 23 +++++-- src/main/scala/gatling/ScanSimulation.scala | 67 ++++++++++++++------- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index 2b9c56a..bbda5f9 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -3,7 +3,7 @@ BaseUrl=https://api.metadefender.com/v4/file ;constant concurrent users ConstantUsers=1 ;time to run, (s) -TestDuration=1 +TestDuration=3 ;time before full filling, (s) ScanWorkflow=multiscan ;dataset folder @@ -15,5 +15,5 @@ ApiKey=opswat_metadefender_api ;waiting time between push file and start to polling (ms) WaitBeforePolling=1000 SilentScan=true -Scan=true +Scan=false Sanitization=true \ No newline at end of file diff --git a/src/main/scala/gatling/Config.scala b/src/main/scala/gatling/Config.scala index 49ab6fc..7c0eee0 100644 --- a/src/main/scala/gatling/Config.scala +++ b/src/main/scala/gatling/Config.scala @@ -16,15 +16,18 @@ class Config { var waitBeforePolling = 1000 var silentScan = true var scan = true - var sanitization = true + var sanitization = false } object Config { def parseConfigure(filePath: String): Config = { + + var rule = "sanitize" + var scanFlow = "" val v = new Config() val ini = new Wini(new File(filePath)) v.baseUrl = ini.get("general", "BaseUrl", classOf[String]) - v.scanWorkflow = ini.get("general", "ScanWorkflow", classOf[String]) + scanFlow = ini.get("general", "ScanWorkflow", classOf[String]) v.constantUser = ini.get("general", "ConstantUsers", classOf[Int]) v.testDuration = ini.get("general", "TestDuration", classOf[Int]) v.rampupDuration = ini.get("general", "RampupDuration", classOf[Int]) @@ -33,8 +36,20 @@ object Config { v.apikey = ini.get("general", "ApiKey", classOf[String]) v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int]) v.silentScan = ini.get("general", "SilentScan", classOf[Boolean]) - v.scan = ini.get("general", "Scan", classOf[Boolean]) - v.sanitization = ini.get("general", "Sanitization", classOf[Boolean]) + + val scan = ini.get("general", "Scan", classOf[Boolean]) + val sanitization = ini.get("general", "Sanitization", classOf[Boolean]) + + if (scan && sanitization){ + rule =scanFlow+",sanitize" + } + else if(scan){ + rule=scanFlow + } + + v.scan = scan + v.sanitization = sanitization + v.scanWorkflow = rule v } } diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index e785372..0ab5764 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -19,7 +19,7 @@ class ScanSimulation extends Simulation { .acceptHeader("*/*") .disableCaching - object Scan { + object FileUpload { val submitFile = http("submit-file") .post(config.baseUrl) .headers(Map("filename" -> "${filename}", "rule" -> config.scanWorkflow)) @@ -29,49 +29,72 @@ class ScanSimulation extends Simulation { .check(status.is(200)) .check(jsonPath("$..data_id").find.exists) .check(jsonPath("$..data_id").find.saveAs("dataId")) + } object ScanProgress { + private val getScanProgress = http("get-scan-result") .get(config.baseUrl + "/${dataId}") .header("apikey", config.apikey) .check(status.is(200)) - .check(jsonPath("$..process_info.progress_percentage").find.saveAs("progress")) - .check(jsonPath("$..process_info.post_processing.actions_ran").optional.saveAs("sanitization")) + .check(jsonPath("$..scan_results.progress_percentage").optional.saveAs("progress")) + .check(jsonPath("$..sanitized.result").optional.saveAs("sanitization")) + .check( jsonPath( "$" ).saveAs( "RESPONSE_DATA" ) ) - val action: ChainBuilder = - exec(_.set("progress", "0")) + val actionSum: ChainBuilder = + exec(_.set("sanitization", "Processing").set("progress", "0")) .doIf(session => session("dataId").asOption[String].isDefined) { - asLongAs(session => session("progress").as[String] != "100") { - pause(config.pollingIntervals.millis).exec(getScanProgress) + if (config.scan && config.sanitization){ + println("Scan and Sanitization") + asLongAs(session => session("progress").as[String] != "100" || + session("sanitization").as[String] == "Processing") { + pause(config.pollingIntervals.millis) + .exec(getScanProgress)} + } + else if(config.scan){ + println("Scan") + asLongAs(session => session("progress").as[String] != "100") { + pause(config.pollingIntervals.millis).exec(getScanProgress)} + } + else { + println("Sanitization") + asLongAs(session => session("sanitization").as[String] == "Processing") { + pause(config.pollingIntervals.millis) + .exec(getScanProgress) + .exec( session => { + println( "Some Restful Service:" ) + println( session( "RESPONSE_DATA" ).as[String] ) + session + }) + } } } + } - object GetSanitized { - private val getSanitizedFile = - http("get-sanitized-file") - .get(config.baseUrl + "/converted/${dataId}") - .check(status.is(200)) - - val action: ChainBuilder = doIf(session => session("sanitization").asOption[String].contains("Sanitized")) { - exec(getSanitizedFile) - } - } +// object GetSanitized { +// private val getSanitizedFile = +// http("get-sanitized-file") +// .get(config.baseUrl + "/converted/${dataId}") +// .header("apikey", config.apikey) +// .check(status.is(200)) +// +// val action: ChainBuilder = doIf(session => session("sanitization").asOption[String].contains("Sanitized")) { +// exec(getSanitizedFile) +// } +// } val pipeline: ScenarioBuilder = scenario("scan-pipeline") .feed(localFiles.feeder) - .exec(Scan.submitFile) + .exec(FileUpload.submitFile) .pause(config.waitBeforePolling.milliseconds) - .doIf(config.scan){ - exec(ScanProgress.action) - } - .doIf(config.sanitization){GetSanitized.action} + .exec(ScanProgress.actionSum) setUp( pipeline From 1c42abf24b56619ffdcbcc2e48a9a35108aabbd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Mon, 1 Mar 2021 15:28:21 +0100 Subject: [PATCH 03/13] silent as option --- src/main/scala/gatling/ScanSimulation.scala | 53 +++++++++++---------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index 0ab5764..9a05667 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -34,14 +34,21 @@ class ScanSimulation extends Simulation { object ScanProgress { - private val getScanProgress = - http("get-scan-result") + def initScanProgress () : HttpRequestBuilder = { + var base = http("get-scan-result") .get(config.baseUrl + "/${dataId}") .header("apikey", config.apikey) .check(status.is(200)) + if(config.silentScan){ + base = base.silent + } + return base .check(jsonPath("$..scan_results.progress_percentage").optional.saveAs("progress")) .check(jsonPath("$..sanitized.result").optional.saveAs("sanitization")) .check( jsonPath( "$" ).saveAs( "RESPONSE_DATA" ) ) + } + + private val getScanProgress = initScanProgress() val actionSum: ChainBuilder = @@ -52,43 +59,41 @@ class ScanSimulation extends Simulation { asLongAs(session => session("progress").as[String] != "100" || session("sanitization").as[String] == "Processing") { pause(config.pollingIntervals.millis) - .exec(getScanProgress)} + .exec(getScanProgress) + .exec( session => { + println( "Respond: S+S:" ) + println( session( "RESPONSE_DATA" ).as[String] ) + session + }) + } } else if(config.scan){ println("Scan") asLongAs(session => session("progress").as[String] != "100") { - pause(config.pollingIntervals.millis).exec(getScanProgress)} + pause(config.pollingIntervals.millis) + .exec(getScanProgress) + .exec( session => { + println( "Response Scan:" ) + println( session( "RESPONSE_DATA" ).as[String] ) + session + }) + } } else { println("Sanitization") asLongAs(session => session("sanitization").as[String] == "Processing") { pause(config.pollingIntervals.millis) .exec(getScanProgress) - .exec( session => { - println( "Some Restful Service:" ) - println( session( "RESPONSE_DATA" ).as[String] ) - session - }) + .exec( session => { + println( "Response Sanit:" ) + println( session( "RESPONSE_DATA" ).as[String] ) + session + }) } } } - } - - -// object GetSanitized { -// private val getSanitizedFile = -// http("get-sanitized-file") -// .get(config.baseUrl + "/converted/${dataId}") -// .header("apikey", config.apikey) -// .check(status.is(200)) -// -// val action: ChainBuilder = doIf(session => session("sanitization").asOption[String].contains("Sanitized")) { -// exec(getSanitizedFile) -// } -// } - val pipeline: ScenarioBuilder = scenario("scan-pipeline") .feed(localFiles.feeder) From 6e29f40c2c3c12d879def0d874d7f3a627ac16dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Mon, 1 Mar 2021 19:35:21 +0100 Subject: [PATCH 04/13] readme.md ready --- README.md | 66 +++++++++++++++++---- pom.xml | 2 +- src/main/resources/config.ini | 16 +++-- src/main/scala/gatling/Config.scala | 31 +++++----- src/main/scala/gatling/ScanSimulation.scala | 39 +++++------- 5 files changed, 95 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index fdbd216..352f6b6 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Using Gatling to perform MetaDefender performance test Documentation * [MetaDefender Core developer guide](https://onlinehelp.opswat.com/corev4/9._%28NEW%29_MetaDefender_Core_Developer_Guide.html) +* [File scanning and sanitization API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) * [Gatling](https://gatling.io/) To build the jar, make sure you have Maven: @@ -13,20 +14,65 @@ The install step will copy `src/main/resources/config.ini` to `target/config.ini You can modify the settings in this configuration file: `target/config.ini` +Example: ``` [general] -BaseUrl: MetaDefender REST URL (e.g.: http://localhost:8008/file) -ConstantUsers: the number of constant concurrent users -TestDuration: time to run the test (s) -ScanWorkflow: MetaDefender workflow name -LocalPath: Dataset folder path -PollingIntervals: sleep time between each polling scan result (ms) -ApiKey: OPSWAT MetaDefender Cloud apikey -WaitBeforePolling: waiting time between push file and start to polling (ms) +BaseUrl: https://api.metadefender.com/v4/file +ConstantUsers: 5 +TestDuration: 10 +ScanWorkflow: multiscan +LocalPath: home/opswatuser/testfiles +PollingIntervals: 500 +ApiKey: 1234567890abcdefghijklmnopqrstuv +WaitBeforePolling: 1000 +SilentScan=true +Scan=true +Sanitization=false ``` -TODO + +***BaseURL:*** +MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file). Actual endpoint information for file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) + +***ConstantUsers:*** +The number of constant concurrent users. The script injects users at a constant -this number- rate, defined in users per second. + +***TestDuration:*** +Time to run the test. (s) + +***ScanWorkflow:*** +MetaDefender workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows in one. It can be ```multiscan```, ```sanitize``` or```unarchive```. More details: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* + +***LocalPath:*** +The folder path where the files to be tested are located. Eg.: */home/user1/tester* + +***PollingIntervals:*** +Sleep time between each polling scan result. (ms) + +***ApiKey:*** +OPSWAT MetaDefender Cloud apikey. It can be found at [metadefender.opswat.com](https://metadefender.opswat.com/account) -> *API key information and limits* -> *API key*. (Registration required.) + +***WaitBeforePolling:*** +Waiting time between push file and start to polling. (ms) + +***SilentScan:*** +A silent request is issued but not logged nor reported. If SilentScan is true the get-scan-result method (polling part) still executed but not logged or reported the errors. (true/false) + +***Scan:*** +Enable or disable the file scanning method. If it is enable, the program asks the result (by polling) until the scan process is not 100% (true/false) + +***Sanitization:*** +Enable or disable the file sanitization method. If it is enable, the program asks the result (by polling) until the sanitization process is not 100% (true/false) + + +*****DeveloperMode:*** +It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) + + + +
To run the jar (in the `target` folder where the `.jar` file is generated): - java -cp metadefender-gatling-1.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation + java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation + diff --git a/pom.xml b/pom.xml index 4cd439a..4fef450 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.opswat metadefender-gatling jar - 1.0.0-SNAPSHOT + 2.0.0-SNAPSHOT metadefender-gatling http://maven.apache.org diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index bbda5f9..9acb41c 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -1,19 +1,23 @@ [general] +;OPSWAT filescsan API endpoint BaseUrl=https://api.metadefender.com/v4/file ;constant concurrent users -ConstantUsers=1 +ConstantUsers=5 ;time to run, (s) -TestDuration=3 -;time before full filling, (s) +TestDuration=10 +;file scanning rule, autocomplete when sanitization ScanWorkflow=multiscan ;dataset folder -LocalPath=/home/aniko.bartos/Documents/tester_files -;sleep time between each polling scan result (ms) +LocalPath=C:\OPSWAT\data +;sleep time between each polling scan/sanitization result (ms) PollingIntervals=500 ;OPSWAT MetaDefender Cloud apikey ApiKey=opswat_metadefender_api ;waiting time between push file and start to polling (ms) WaitBeforePolling=1000 +;turn off scan-reporting, eg:errors. (true/false) SilentScan=true +;file-scan (true/false) Scan=false -Sanitization=true \ No newline at end of file +;file-sanitization (true/false) +Sanitization=true diff --git a/src/main/scala/gatling/Config.scala b/src/main/scala/gatling/Config.scala index 7c0eee0..3fdedae 100644 --- a/src/main/scala/gatling/Config.scala +++ b/src/main/scala/gatling/Config.scala @@ -6,17 +6,17 @@ import org.ini4j.Wini class Config { var baseUrl = "" - var scanWorkflow = "" var constantUser = 2 var testDuration = 5 - var rampupDuration = 5 - var pollingIntervals = 500 + var scanWorkflow = "" var localPath = "" + var pollingIntervals = 500 var apikey = "" var waitBeforePolling = 1000 var silentScan = true var scan = true var sanitization = false + var developerMode = false } object Config { @@ -26,30 +26,27 @@ object Config { var scanFlow = "" val v = new Config() val ini = new Wini(new File(filePath)) + v.baseUrl = ini.get("general", "BaseUrl", classOf[String]) - scanFlow = ini.get("general", "ScanWorkflow", classOf[String]) v.constantUser = ini.get("general", "ConstantUsers", classOf[Int]) v.testDuration = ini.get("general", "TestDuration", classOf[Int]) - v.rampupDuration = ini.get("general", "RampupDuration", classOf[Int]) + + // Add sanitize to rule if Sanitization is true + scanFlow = ini.get("general", "ScanWorkflow", classOf[String]) + val scan = ini.get("general", "Scan", classOf[Boolean]) + val sanitization = ini.get("general", "DeveloperMode", classOf[Boolean]) + if (scan && sanitization){rule =scanFlow+",sanitize"} + else if(scan){rule=scanFlow} + v.scanWorkflow = rule + v.localPath = ini.get("general", "LocalPath", classOf[String]) v.pollingIntervals = ini.get("general", "PollingIntervals", classOf[Int]) v.apikey = ini.get("general", "ApiKey", classOf[String]) v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int]) v.silentScan = ini.get("general", "SilentScan", classOf[Boolean]) - - val scan = ini.get("general", "Scan", classOf[Boolean]) - val sanitization = ini.get("general", "Sanitization", classOf[Boolean]) - - if (scan && sanitization){ - rule =scanFlow+",sanitize" - } - else if(scan){ - rule=scanFlow - } - v.scan = scan v.sanitization = sanitization - v.scanWorkflow = rule + v.developerMode = ini.get("general", "DeveloperMode", classOf[Boolean]) v } } diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index 9a05667..95f266e 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -5,7 +5,6 @@ import io.gatling.core.structure.{ChainBuilder, ScenarioBuilder} import io.gatling.http.Predef._ import io.gatling.http.protocol.HttpProtocolBuilder import io.gatling.http.request.builder.HttpRequestBuilder - import scala.concurrent.duration._ @@ -29,11 +28,9 @@ class ScanSimulation extends Simulation { .check(status.is(200)) .check(jsonPath("$..data_id").find.exists) .check(jsonPath("$..data_id").find.saveAs("dataId")) - } object ScanProgress { - def initScanProgress () : HttpRequestBuilder = { var base = http("get-scan-result") .get(config.baseUrl + "/${dataId}") @@ -42,53 +39,45 @@ class ScanSimulation extends Simulation { if(config.silentScan){ base = base.silent } - return base + base .check(jsonPath("$..scan_results.progress_percentage").optional.saveAs("progress")) .check(jsonPath("$..sanitized.result").optional.saveAs("sanitization")) .check( jsonPath( "$" ).saveAs( "RESPONSE_DATA" ) ) } - private val getScanProgress = initScanProgress() + def printResponse(): ChainBuilder = { + exec( session => { + println("Response:") + println(session( "RESPONSE_DATA").as[String]) + session + }) + } + private val getScanProgress = initScanProgress() - val actionSum: ChainBuilder = + val action: ChainBuilder = exec(_.set("sanitization", "Processing").set("progress", "0")) .doIf(session => session("dataId").asOption[String].isDefined) { if (config.scan && config.sanitization){ - println("Scan and Sanitization") asLongAs(session => session("progress").as[String] != "100" || session("sanitization").as[String] == "Processing") { pause(config.pollingIntervals.millis) .exec(getScanProgress) - .exec( session => { - println( "Respond: S+S:" ) - println( session( "RESPONSE_DATA" ).as[String] ) - session - }) + .doIf(config.developerMode){printResponse()} } } else if(config.scan){ - println("Scan") asLongAs(session => session("progress").as[String] != "100") { pause(config.pollingIntervals.millis) .exec(getScanProgress) - .exec( session => { - println( "Response Scan:" ) - println( session( "RESPONSE_DATA" ).as[String] ) - session - }) + .doIf(config.developerMode){printResponse()} } } else { - println("Sanitization") asLongAs(session => session("sanitization").as[String] == "Processing") { pause(config.pollingIntervals.millis) .exec(getScanProgress) - .exec( session => { - println( "Response Sanit:" ) - println( session( "RESPONSE_DATA" ).as[String] ) - session - }) + .doIf(config.developerMode){printResponse()} } } } @@ -99,7 +88,7 @@ class ScanSimulation extends Simulation { .feed(localFiles.feeder) .exec(FileUpload.submitFile) .pause(config.waitBeforePolling.milliseconds) - .exec(ScanProgress.actionSum) + .exec(ScanProgress.action) setUp( pipeline From cd9c4cac8129f73434ee8d87fca558ea239c7810 Mon Sep 17 00:00:00 2001 From: anikobartos Date: Tue, 2 Mar 2021 12:53:20 +0100 Subject: [PATCH 05/13] Update README.md --- README.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 352f6b6..9eccebb 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The install step will copy `src/main/resources/config.ini` to `target/config.ini You can modify the settings in this configuration file: `target/config.ini` -Example: +Example configuration: ``` [general] BaseUrl: https://api.metadefender.com/v4/file @@ -32,40 +32,53 @@ Sanitization=false ***BaseURL:*** -MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file). Actual endpoint information for file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) + +MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file). More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) ***ConstantUsers:*** -The number of constant concurrent users. The script injects users at a constant -this number- rate, defined in users per second. + +The number of constant concurrent users for the test. Each simulated user will submit a randomly +selected file for scanning, wait for the scan result, then select a new file for scanning. ***TestDuration:*** -Time to run the test. (s) + +Total test duration while the simulated users will continue to submit files. (seconds) ***ScanWorkflow:*** -MetaDefender workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows in one. It can be ```multiscan```, ```sanitize``` or```unarchive```. More details: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* + +MetaDefender scanning workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. It can be `multiscan`, `sanitize` or `unarchive`. If *Sanitization* is true the program will automatically complete it with `sanitize`. More details: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* ***LocalPath:*** -The folder path where the files to be tested are located. Eg.: */home/user1/tester* + +The folder path where the files to be tested are located, e.g.: */home/user1/tester* ***PollingIntervals:*** -Sleep time between each polling scan result. (ms) + +Sleep time between polling scan results. (milliseconds) ***ApiKey:*** -OPSWAT MetaDefender Cloud apikey. It can be found at [metadefender.opswat.com](https://metadefender.opswat.com/account) -> *API key information and limits* -> *API key*. (Registration required.) + +OPSWAT MetaDefender Cloud API key. You can find your key at [metadefender.opswat.com](https://metadefender.opswat.com/account) -> *API key information and limits* -> *API key*. (Registration required.) ***WaitBeforePolling:*** -Waiting time between push file and start to polling. (ms) + +Waiting time after file submission to start polling. (milliseconds) ***SilentScan:*** + A silent request is issued but not logged nor reported. If SilentScan is true the get-scan-result method (polling part) still executed but not logged or reported the errors. (true/false) ***Scan:*** + Enable or disable the file scanning method. If it is enable, the program asks the result (by polling) until the scan process is not 100% (true/false) ***Sanitization:*** + Enable or disable the file sanitization method. If it is enable, the program asks the result (by polling) until the sanitization process is not 100% (true/false) *****DeveloperMode:*** + It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) From 9d49e7faf07041564f9bdbd60244c70e9e3c8f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Tue, 2 Mar 2021 18:13:03 +0100 Subject: [PATCH 06/13] Sanitize correction --- README.md | 16 +++--- src/main/resources/config.ini | 6 +- src/main/scala/gatling/Config.scala | 18 +----- src/main/scala/gatling/ScanSimulation.scala | 62 +++++++++++++-------- 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 9eccebb..61d4df3 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,14 @@ You can modify the settings in this configuration file: `target/config.ini` Example configuration: ``` [general] -BaseUrl: https://api.metadefender.com/v4/file -ConstantUsers: 5 -TestDuration: 10 -ScanWorkflow: multiscan -LocalPath: home/opswatuser/testfiles -PollingIntervals: 500 -ApiKey: 1234567890abcdefghijklmnopqrstuv -WaitBeforePolling: 1000 +BaseUrl=https://api.metadefender.com/v4/file +ConstantUsers=5 +TestDuration=10 +ScanWorkflow=multiscan +LocalPath=/home/opswatuser/testfiles +PollingIntervals=500 +ApiKey=1234567890abcdefghijklmnopqrstuv +WaitBeforePolling=1000 SilentScan=true Scan=true Sanitization=false diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index 9acb41c..badcee9 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -17,7 +17,5 @@ ApiKey=opswat_metadefender_api WaitBeforePolling=1000 ;turn off scan-reporting, eg:errors. (true/false) SilentScan=true -;file-scan (true/false) -Scan=false -;file-sanitization (true/false) -Sanitization=true +;check file-sanitization (true/false) +CheckSanitizationResult=true diff --git a/src/main/scala/gatling/Config.scala b/src/main/scala/gatling/Config.scala index 3fdedae..60fee0e 100644 --- a/src/main/scala/gatling/Config.scala +++ b/src/main/scala/gatling/Config.scala @@ -14,38 +14,26 @@ class Config { var apikey = "" var waitBeforePolling = 1000 var silentScan = true - var scan = true - var sanitization = false + var checkSanitization = false var developerMode = false } object Config { def parseConfigure(filePath: String): Config = { - var rule = "sanitize" - var scanFlow = "" val v = new Config() val ini = new Wini(new File(filePath)) v.baseUrl = ini.get("general", "BaseUrl", classOf[String]) v.constantUser = ini.get("general", "ConstantUsers", classOf[Int]) v.testDuration = ini.get("general", "TestDuration", classOf[Int]) - - // Add sanitize to rule if Sanitization is true - scanFlow = ini.get("general", "ScanWorkflow", classOf[String]) - val scan = ini.get("general", "Scan", classOf[Boolean]) - val sanitization = ini.get("general", "DeveloperMode", classOf[Boolean]) - if (scan && sanitization){rule =scanFlow+",sanitize"} - else if(scan){rule=scanFlow} - v.scanWorkflow = rule - + v.scanWorkflow = ini.get("general", "ScanWorkflow", classOf[String]) v.localPath = ini.get("general", "LocalPath", classOf[String]) v.pollingIntervals = ini.get("general", "PollingIntervals", classOf[Int]) v.apikey = ini.get("general", "ApiKey", classOf[String]) v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int]) v.silentScan = ini.get("general", "SilentScan", classOf[Boolean]) - v.scan = scan - v.sanitization = sanitization + v.checkSanitization = ini.get("general", "CheckSanitizationResult", classOf[Boolean]) v.developerMode = ini.get("general", "DeveloperMode", classOf[Boolean]) v } diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index 95f266e..4356e23 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -19,27 +19,45 @@ class ScanSimulation extends Simulation { .disableCaching object FileUpload { - val submitFile = http("submit-file") - .post(config.baseUrl) - .headers(Map("filename" -> "${filename}", "rule" -> config.scanWorkflow)) - .header("apikey", config.apikey) - .header("Content-Type","application/octet-stream") - .body(RawFileBody("${filepath}")) - .check(status.is(200)) - .check(jsonPath("$..data_id").find.exists) - .check(jsonPath("$..data_id").find.saveAs("dataId")) + def initFileUpload () : HttpRequestBuilder = { + var base = http("submit-file") + .post(config.baseUrl) + .header("filename","${filename}") + + if(config.scanWorkflow != ""){ + base = base.header("rule", config.scanWorkflow) + } + + if(config.apikey != ""){ + base = base.header("apikey", config.apikey) + } + + base + .header("Content-Type","application/octet-stream") + .body(RawFileBody("${filepath}")) + .check(status.is(200)) + .check(jsonPath("$..data_id").find.exists) + .check(jsonPath("$..data_id").find.saveAs("dataId")) + } + + val submitFile = initFileUpload() } object ScanProgress { def initScanProgress () : HttpRequestBuilder = { var base = http("get-scan-result") .get(config.baseUrl + "/${dataId}") - .header("apikey", config.apikey) - .check(status.is(200)) + + if(config.apikey != ""){ + base = base.header("apikey", config.apikey) + } + if(config.silentScan){ base = base.silent } + base + .check(status.is(200)) .check(jsonPath("$..scan_results.progress_percentage").optional.saveAs("progress")) .check(jsonPath("$..sanitized.result").optional.saveAs("sanitization")) .check( jsonPath( "$" ).saveAs( "RESPONSE_DATA" ) ) @@ -47,7 +65,6 @@ class ScanSimulation extends Simulation { def printResponse(): ChainBuilder = { exec( session => { - println("Response:") println(session( "RESPONSE_DATA").as[String]) session }) @@ -58,26 +75,25 @@ class ScanSimulation extends Simulation { val action: ChainBuilder = exec(_.set("sanitization", "Processing").set("progress", "0")) .doIf(session => session("dataId").asOption[String].isDefined) { - if (config.scan && config.sanitization){ + if (config.checkSanitization){ asLongAs(session => session("progress").as[String] != "100" || session("sanitization").as[String] == "Processing") { pause(config.pollingIntervals.millis) .exec(getScanProgress) - .doIf(config.developerMode){printResponse()} - } - } - else if(config.scan){ - asLongAs(session => session("progress").as[String] != "100") { - pause(config.pollingIntervals.millis) - .exec(getScanProgress) - .doIf(config.developerMode){printResponse()} + .doIf(config.developerMode){ + println("Scan and sanitization response:") + printResponse() + } } } else { - asLongAs(session => session("sanitization").as[String] == "Processing") { + asLongAs(session => session("progress").as[String] != "100") { pause(config.pollingIntervals.millis) .exec(getScanProgress) - .doIf(config.developerMode){printResponse()} + .doIf(config.developerMode){ + println("Scan response:") + printResponse() + } } } } From 06f0fde4691441dd2ab8f7761b68aa7afd01fc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Tue, 2 Mar 2021 20:37:13 +0100 Subject: [PATCH 07/13] Local usage + readme.md --- README.md | 40 ++++++++++++--------- src/main/resources/config.ini | 4 +-- src/main/scala/gatling/ScanSimulation.scala | 9 +++-- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 61d4df3..8ac4818 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Using Gatling to perform MetaDefender performance test Documentation * [MetaDefender Core developer guide](https://onlinehelp.opswat.com/corev4/9._%28NEW%29_MetaDefender_Core_Developer_Guide.html) -* [File scanning and sanitization API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) +* [File scanning and sanitization API (cloud)](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) * [Gatling](https://gatling.io/) To build the jar, make sure you have Maven: @@ -14,26 +14,38 @@ The install step will copy `src/main/resources/config.ini` to `target/config.ini You can modify the settings in this configuration file: `target/config.ini` -Example configuration: +Example configuration for **local** usage: +``` +[general] +BaseUrl=http://localhost:8008/file +ConstantUsers=5 +TestDuration=10 +LocalPath=/home/opswatuser/testfiles +PollingIntervals=500 +WaitBeforePolling=1000 +SilentScan=true +``` + + +Example configuration for **cloud** usage: ``` [general] BaseUrl=https://api.metadefender.com/v4/file ConstantUsers=5 TestDuration=10 -ScanWorkflow=multiscan +ScanWorkflow=multiscan,sanitize LocalPath=/home/opswatuser/testfiles PollingIntervals=500 ApiKey=1234567890abcdefghijklmnopqrstuv WaitBeforePolling=1000 SilentScan=true -Scan=true -Sanitization=false +CheckSanitizationResult=true ``` ***BaseURL:*** -MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file). More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) +MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file (cloud) or http://localhost:8008/file (local)). More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) ***ConstantUsers:*** @@ -44,9 +56,9 @@ selected file for scanning, wait for the scan result, then select a new file for Total test duration while the simulated users will continue to submit files. (seconds) -***ScanWorkflow:*** +***ScanWorkflow [optional]:*** -MetaDefender scanning workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. It can be `multiscan`, `sanitize` or `unarchive`. If *Sanitization* is true the program will automatically complete it with `sanitize`. More details: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* +MetaDefender scanning workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. For cloud-based testing, it can be `multiscan`, `sanitize` or `unarchive`. If `CheckSanitizationResult` is true, and the testing process is cloud-based, it needs to be set. More details to cloud usage: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* ***LocalPath:*** @@ -56,7 +68,7 @@ The folder path where the files to be tested are located, e.g.: */home/user1/tes Sleep time between polling scan results. (milliseconds) -***ApiKey:*** +***ApiKey [cloud usage only]:*** OPSWAT MetaDefender Cloud API key. You can find your key at [metadefender.opswat.com](https://metadefender.opswat.com/account) -> *API key information and limits* -> *API key*. (Registration required.) @@ -68,18 +80,14 @@ Waiting time after file submission to start polling. (milliseconds) A silent request is issued but not logged nor reported. If SilentScan is true the get-scan-result method (polling part) still executed but not logged or reported the errors. (true/false) -***Scan:*** - -Enable or disable the file scanning method. If it is enable, the program asks the result (by polling) until the scan process is not 100% (true/false) - -***Sanitization:*** +***CheckSanitizationResult:*** -Enable or disable the file sanitization method. If it is enable, the program asks the result (by polling) until the sanitization process is not 100% (true/false) +If it is true, the program asks for the result (by polling) until the sanitization process is not 100%. For cloud-based usage, it needs to be set in `ScanWorkflow` also. (true/false) *****DeveloperMode:*** -It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) +It can be set a DeveloperMode, which print the HTTP-responses to the terminal. It is also print (before the response) what will be check: Scan result or scan results with polling result. By default, it is hidden and false. (true/false) diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index badcee9..30e9780 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -17,5 +17,5 @@ ApiKey=opswat_metadefender_api WaitBeforePolling=1000 ;turn off scan-reporting, eg:errors. (true/false) SilentScan=true -;check file-sanitization (true/false) -CheckSanitizationResult=true +;check file-sanitization results(true/false) +CheckSanitizationResult=false diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index 4356e23..e67e9f9 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -63,8 +63,9 @@ class ScanSimulation extends Simulation { .check( jsonPath( "$" ).saveAs( "RESPONSE_DATA" ) ) } - def printResponse(): ChainBuilder = { + def printResponse ( scanTypeAsString:String ): ChainBuilder = { exec( session => { + println(scanTypeAsString) println(session( "RESPONSE_DATA").as[String]) session }) @@ -81,8 +82,7 @@ class ScanSimulation extends Simulation { pause(config.pollingIntervals.millis) .exec(getScanProgress) .doIf(config.developerMode){ - println("Scan and sanitization response:") - printResponse() + printResponse("Check scan and sanitization process:") } } } @@ -91,8 +91,7 @@ class ScanSimulation extends Simulation { pause(config.pollingIntervals.millis) .exec(getScanProgress) .doIf(config.developerMode){ - println("Scan response:") - printResponse() + printResponse("Check scan process:") } } } From 4acce83700cdda40c8d91683e842a2361e23d5c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Wed, 3 Mar 2021 19:11:11 +0100 Subject: [PATCH 08/13] check only process_info -> progress_percentage --- README.md | 9 ++--- src/main/resources/config.ini | 12 +++---- src/main/scala/gatling/Config.scala | 14 +++++--- src/main/scala/gatling/ScanSimulation.scala | 40 +++++++-------------- 4 files changed, 30 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 8ac4818..0c6b0ad 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ PollingIntervals=500 ApiKey=1234567890abcdefghijklmnopqrstuv WaitBeforePolling=1000 SilentScan=true -CheckSanitizationResult=true ``` @@ -58,7 +57,7 @@ Total test duration while the simulated users will continue to submit files. (se ***ScanWorkflow [optional]:*** -MetaDefender scanning workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. For cloud-based testing, it can be `multiscan`, `sanitize` or `unarchive`. If `CheckSanitizationResult` is true, and the testing process is cloud-based, it needs to be set. More details to cloud usage: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* +MetaDefender scanning workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. For cloud-based testing, it can be `multiscan`, `sanitize` or `unarchive`. More details to cloud usage: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* ***LocalPath:*** @@ -80,14 +79,10 @@ Waiting time after file submission to start polling. (milliseconds) A silent request is issued but not logged nor reported. If SilentScan is true the get-scan-result method (polling part) still executed but not logged or reported the errors. (true/false) -***CheckSanitizationResult:*** - -If it is true, the program asks for the result (by polling) until the sanitization process is not 100%. For cloud-based usage, it needs to be set in `ScanWorkflow` also. (true/false) - *****DeveloperMode:*** -It can be set a DeveloperMode, which print the HTTP-responses to the terminal. It is also print (before the response) what will be check: Scan result or scan results with polling result. By default, it is hidden and false. (true/false) +It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index 30e9780..f646995 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -1,15 +1,15 @@ [general] -;OPSWAT filescsan API endpoint +;OPSWAT filescan API endpoint BaseUrl=https://api.metadefender.com/v4/file ;constant concurrent users -ConstantUsers=5 +ConstantUsers=1 ;time to run, (s) TestDuration=10 -;file scanning rule, autocomplete when sanitization -ScanWorkflow=multiscan +;file scanning rule +ScanWorkflow=multiscan,sanitize ;dataset folder LocalPath=C:\OPSWAT\data -;sleep time between each polling scan/sanitization result (ms) +;sleep time between each polling scan result (ms) PollingIntervals=500 ;OPSWAT MetaDefender Cloud apikey ApiKey=opswat_metadefender_api @@ -17,5 +17,3 @@ ApiKey=opswat_metadefender_api WaitBeforePolling=1000 ;turn off scan-reporting, eg:errors. (true/false) SilentScan=true -;check file-sanitization results(true/false) -CheckSanitizationResult=false diff --git a/src/main/scala/gatling/Config.scala b/src/main/scala/gatling/Config.scala index 60fee0e..ae66cb5 100644 --- a/src/main/scala/gatling/Config.scala +++ b/src/main/scala/gatling/Config.scala @@ -1,7 +1,7 @@ -import java.io.File - +import java.io.{File, IOException} import org.ini4j.Wini +import java.nio.file.{Files, Paths} class Config { @@ -14,7 +14,6 @@ class Config { var apikey = "" var waitBeforePolling = 1000 var silentScan = true - var checkSanitization = false var developerMode = false } @@ -29,11 +28,18 @@ object Config { v.testDuration = ini.get("general", "TestDuration", classOf[Int]) v.scanWorkflow = ini.get("general", "ScanWorkflow", classOf[String]) v.localPath = ini.get("general", "LocalPath", classOf[String]) + + try{ + assert(Files.list(Paths.get(v.localPath)).count()>0) + } + catch { + case _:Throwable => throw new IOException("WRONG PATH OR EMPTY DIRECTORY: "+v.localPath) + } + v.pollingIntervals = ini.get("general", "PollingIntervals", classOf[Int]) v.apikey = ini.get("general", "ApiKey", classOf[String]) v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int]) v.silentScan = ini.get("general", "SilentScan", classOf[Boolean]) - v.checkSanitization = ini.get("general", "CheckSanitizationResult", classOf[Boolean]) v.developerMode = ini.get("general", "DeveloperMode", classOf[Boolean]) v } diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index e67e9f9..304bd06 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -58,15 +58,14 @@ class ScanSimulation extends Simulation { base .check(status.is(200)) - .check(jsonPath("$..scan_results.progress_percentage").optional.saveAs("progress")) - .check(jsonPath("$..sanitized.result").optional.saveAs("sanitization")) - .check( jsonPath( "$" ).saveAs( "RESPONSE_DATA" ) ) + .check(jsonPath("$..process_info.progress_percentage").optional.saveAs("progress")) + .check( jsonPath( "$" ).saveAs( "response_data" ) ) } - def printResponse ( scanTypeAsString:String ): ChainBuilder = { + def printResponse (): ChainBuilder = { exec( session => { - println(scanTypeAsString) - println(session( "RESPONSE_DATA").as[String]) + println("Response:") + println(session( "response_data").as[String]) session }) } @@ -74,31 +73,18 @@ class ScanSimulation extends Simulation { private val getScanProgress = initScanProgress() val action: ChainBuilder = - exec(_.set("sanitization", "Processing").set("progress", "0")) - .doIf(session => session("dataId").asOption[String].isDefined) { - if (config.checkSanitization){ - asLongAs(session => session("progress").as[String] != "100" || - session("sanitization").as[String] == "Processing") { - pause(config.pollingIntervals.millis) - .exec(getScanProgress) - .doIf(config.developerMode){ - printResponse("Check scan and sanitization process:") - } + exec(_.set("progress", "0")) + .doIf(session => session("dataId").asOption[String].isDefined) { + asLongAs(session => session("progress").as[String] != "100") { + pause(config.pollingIntervals.millis) + .exec(getScanProgress) + .doIf(config.developerMode){ + printResponse() } } - else { - asLongAs(session => session("progress").as[String] != "100") { - pause(config.pollingIntervals.millis) - .exec(getScanProgress) - .doIf(config.developerMode){ - printResponse("Check scan process:") - } - } - } - } + } } - val pipeline: ScenarioBuilder = scenario("scan-pipeline") .feed(localFiles.feeder) .exec(FileUpload.submitFile) From 5742f3c99bbb3bc7de2d0373250908789fb050d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Wed, 3 Mar 2021 20:07:37 +0100 Subject: [PATCH 09/13] Readme correction --- README.md | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0c6b0ad..82cb48b 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,54 @@ -Using Gatling to perform MetaDefender performance test +# MetaDefender Gatling performance test -Documentation +Useful links: * [MetaDefender Core developer guide](https://onlinehelp.opswat.com/corev4/9._%28NEW%29_MetaDefender_Core_Developer_Guide.html) * [File scanning and sanitization API (cloud)](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) * [Gatling](https://gatling.io/) +## Requirements + +- installed Java 11 + +## Usage +### Build + To build the jar, make sure you have Maven: mvn clean install The install step will copy `src/main/resources/config.ini` to `target/config.ini`. +### Configuration + You can modify the settings in this configuration file: `target/config.ini` -Example configuration for **local** usage: + +Example configuration for **Metadefender Cloud** usage: ``` [general] -BaseUrl=http://localhost:8008/file +BaseUrl=https://api.metadefender.com/v4/file ConstantUsers=5 TestDuration=10 +ScanWorkflow=multiscan,sanitize LocalPath=/home/opswatuser/testfiles PollingIntervals=500 +ApiKey=1234567890abcdefghijklmnopqrstuv WaitBeforePolling=1000 -SilentScan=true ``` -Example configuration for **cloud** usage: +Example configuration for **local MetaDefender Core** usage: ``` [general] -BaseUrl=https://api.metadefender.com/v4/file +BaseUrl=http://localhost:8008/file ConstantUsers=5 TestDuration=10 -ScanWorkflow=multiscan,sanitize LocalPath=/home/opswatuser/testfiles PollingIntervals=500 -ApiKey=1234567890abcdefghijklmnopqrstuv WaitBeforePolling=1000 -SilentScan=true ``` - ***BaseURL:*** MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file (cloud) or http://localhost:8008/file (local)). More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) @@ -85,10 +92,9 @@ A silent request is issued but not logged nor reported. If SilentScan is true th It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) +### Run a test -
- -To run the jar (in the `target` folder where the `.jar` file is generated): +`metadefender-gatling-2.0.0-SNAPSHOT.jar` file and `config.ini` file are required to run the test. In case of building `metadefender-gatling-2.0.0-SNAPSHOT.jar` is generated in `target` folder. Command to run: java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation From 68cb8954f31ecc3eb61ce628d78c25feda6f55ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Thu, 4 Mar 2021 21:07:13 +0100 Subject: [PATCH 10/13] Config correction --- README.md | 12 +++----- src/main/resources/config.ini | 5 +-- src/main/scala/gatling/Config.scala | 34 +++++++++++++++------ src/main/scala/gatling/ScanSimulation.scala | 10 +++--- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 82cb48b..e473bfb 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ Useful links: - installed Java 11 -## Usage -### Build +## Build To build the jar, make sure you have Maven: @@ -19,7 +18,7 @@ To build the jar, make sure you have Maven: The install step will copy `src/main/resources/config.ini` to `target/config.ini`. -### Configuration +## Configuration You can modify the settings in this configuration file: `target/config.ini` @@ -82,17 +81,16 @@ OPSWAT MetaDefender Cloud API key. You can find your key at [metadefender.opswat Waiting time after file submission to start polling. (milliseconds) -***SilentScan:*** - -A silent request is issued but not logged nor reported. If SilentScan is true the get-scan-result method (polling part) still executed but not logged or reported the errors. (true/false) +*****ShowPollingDetails:*** +If ShowPollingDetails is true the software shows and logs the polling details. By default it is false. (true/false) *****DeveloperMode:*** It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) -### Run a test +## Run a test `metadefender-gatling-2.0.0-SNAPSHOT.jar` file and `config.ini` file are required to run the test. In case of building `metadefender-gatling-2.0.0-SNAPSHOT.jar` is generated in `target` folder. Command to run: diff --git a/src/main/resources/config.ini b/src/main/resources/config.ini index f646995..3927bec 100644 --- a/src/main/resources/config.ini +++ b/src/main/resources/config.ini @@ -1,5 +1,4 @@ [general] -;OPSWAT filescan API endpoint BaseUrl=https://api.metadefender.com/v4/file ;constant concurrent users ConstantUsers=1 @@ -14,6 +13,4 @@ PollingIntervals=500 ;OPSWAT MetaDefender Cloud apikey ApiKey=opswat_metadefender_api ;waiting time between push file and start to polling (ms) -WaitBeforePolling=1000 -;turn off scan-reporting, eg:errors. (true/false) -SilentScan=true +WaitBeforePolling=1000 \ No newline at end of file diff --git a/src/main/scala/gatling/Config.scala b/src/main/scala/gatling/Config.scala index ae66cb5..dd24ffb 100644 --- a/src/main/scala/gatling/Config.scala +++ b/src/main/scala/gatling/Config.scala @@ -13,11 +13,29 @@ class Config { var pollingIntervals = 500 var apikey = "" var waitBeforePolling = 1000 - var silentScan = true + var pollingDetails = false var developerMode = false } object Config { + def checkValues(config: Config): Unit ={ + if(config.baseUrl==null||config.baseUrl==""){throw new IOException("MISSING ENDPOINT")} + if(config.constantUser<1){config.constantUser=2} + if(config.testDuration<1){config.testDuration=5} + if(config.scanWorkflow==null){config.scanWorkflow=""} + + try{ + assert(Files.list(Paths.get(config.localPath)).count()>0) + } + catch { + case _:Throwable => throw new IOException("WRONG PATH OR EMPTY DIRECTORY: "+config.localPath) + } + + if(config.pollingIntervals<1){config.pollingIntervals=500} + if(config.apikey==null){config.apikey=""} + if(config.waitBeforePolling<1){config.waitBeforePolling=1000} + } + def parseConfigure(filePath: String): Config = { val v = new Config() @@ -28,19 +46,15 @@ object Config { v.testDuration = ini.get("general", "TestDuration", classOf[Int]) v.scanWorkflow = ini.get("general", "ScanWorkflow", classOf[String]) v.localPath = ini.get("general", "LocalPath", classOf[String]) - - try{ - assert(Files.list(Paths.get(v.localPath)).count()>0) - } - catch { - case _:Throwable => throw new IOException("WRONG PATH OR EMPTY DIRECTORY: "+v.localPath) - } - v.pollingIntervals = ini.get("general", "PollingIntervals", classOf[Int]) v.apikey = ini.get("general", "ApiKey", classOf[String]) + v.apikey=ini.get("general", "ApiKey", classOf[String]) v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int]) - v.silentScan = ini.get("general", "SilentScan", classOf[Boolean]) + v.pollingDetails = ini.get("general", "ShowPollingDetails", classOf[Boolean]) v.developerMode = ini.get("general", "DeveloperMode", classOf[Boolean]) + + checkValues(v) + v } } diff --git a/src/main/scala/gatling/ScanSimulation.scala b/src/main/scala/gatling/ScanSimulation.scala index 304bd06..0b66965 100644 --- a/src/main/scala/gatling/ScanSimulation.scala +++ b/src/main/scala/gatling/ScanSimulation.scala @@ -52,20 +52,22 @@ class ScanSimulation extends Simulation { base = base.header("apikey", config.apikey) } - if(config.silentScan){ + if(!config.pollingDetails){ base = base.silent } base .check(status.is(200)) .check(jsonPath("$..process_info.progress_percentage").optional.saveAs("progress")) - .check( jsonPath( "$" ).saveAs( "response_data" ) ) + .check( jsonPath( "$" ).optional.saveAs( "response_data" ) ) } def printResponse (): ChainBuilder = { exec( session => { - println("Response:") - println(session( "response_data").as[String]) + if(session.contains("response_data")) { + println("Response:") + println(session( "response_data").as[String]) + } session }) } From 2d36429c7b5c8daa24eb5658decabb6ff12a0ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartos=20Anik=C3=B3?= Date: Fri, 5 Mar 2021 15:02:41 +0100 Subject: [PATCH 11/13] added start.sh --- README.md | 10 ++++++++-- start.sh | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100755 start.sh diff --git a/README.md b/README.md index e473bfb..8a72cad 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,13 @@ It can be set a DeveloperMode, which print the HTTP-responses to the terminal. B ## Run a test -`metadefender-gatling-2.0.0-SNAPSHOT.jar` file and `config.ini` file are required to run the test. In case of building `metadefender-gatling-2.0.0-SNAPSHOT.jar` is generated in `target` folder. Command to run: +`metadefender-gatling-2.0.0-SNAPSHOT.jar` file and `config.ini` file are required to run the test. In case of building `metadefender-gatling-2.0.0-SNAPSHOT.jar` is generated in `target` folder. - java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation +Run + +`./start.sh` + +to testing. + + diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..3410155 --- /dev/null +++ b/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cd target +java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation +cd .. \ No newline at end of file From 1a5ac2b6be1d8b0d64e3ca9d6a8afec4cf8314c4 Mon Sep 17 00:00:00 2001 From: Peter Vingelmann Date: Fri, 5 Mar 2021 19:41:09 +0100 Subject: [PATCH 12/13] Update README.md --- README.md | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8a72cad..4e14eb0 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Useful links: ## Requirements -- installed Java 11 +- Java 11 installed (Java 8 is not supported) ## Build @@ -36,7 +36,6 @@ ApiKey=1234567890abcdefghijklmnopqrstuv WaitBeforePolling=1000 ``` - Example configuration for **local MetaDefender Core** usage: ``` [general] @@ -50,7 +49,8 @@ WaitBeforePolling=1000 ***BaseURL:*** -MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file (cloud) or http://localhost:8008/file (local)). More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) +MetaDefender REST URL, e.g.: https://api.metadefender.com/v4/file (Cloud) or http://localhost:8008/file (local). +More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) ***ConstantUsers:*** @@ -63,7 +63,9 @@ Total test duration while the simulated users will continue to submit files. (se ***ScanWorkflow [optional]:*** -MetaDefender scanning workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. For cloud-based testing, it can be `multiscan`, `sanitize` or `unarchive`. More details to cloud usage: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* +MetaDefender scan workflow rule to activate. Multiple values can be separated by "," to combine multiple workflows. For Cloud-based testing, it can be `multiscan`, `sanitize` or `unarchive`. More details about Cloud workflows: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule* + +For testing local MetaDefender Core, you are free to specify any custom workflow that is available on your installation. ***LocalPath:*** @@ -73,7 +75,7 @@ The folder path where the files to be tested are located, e.g.: */home/user1/tes Sleep time between polling scan results. (milliseconds) -***ApiKey [cloud usage only]:*** +***ApiKey [Cloud usage only]:*** OPSWAT MetaDefender Cloud API key. You can find your key at [metadefender.opswat.com](https://metadefender.opswat.com/account) -> *API key information and limits* -> *API key*. (Registration required.) @@ -83,22 +85,23 @@ Waiting time after file submission to start polling. (milliseconds) *****ShowPollingDetails:*** -If ShowPollingDetails is true the software shows and logs the polling details. By default it is false. (true/false) +If true, Gatling will also show the polling details as separate requests. By default, it is false. (true/false) *****DeveloperMode:*** -It can be set a DeveloperMode, which print the HTTP-responses to the terminal. By default, it is hidden and false. (true/false) +It true, Gatling will print the HTTP responses to the console. By default, it is false. (true/false) -## Run a test +## Running a test -`metadefender-gatling-2.0.0-SNAPSHOT.jar` file and `config.ini` file are required to run the test. In case of building `metadefender-gatling-2.0.0-SNAPSHOT.jar` is generated in `target` folder. +The `metadefender-gatling-2.0.0-SNAPSHOT.jar` and `config.ini` files are required to run the test. +Maven will install these files in the `target` folder. -Run +Run a test using the helper script: -`./start.sh` + ./start.sh -to testing. +The script runs the jar file with the following parameters: - + java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation From e6736750268191400c7f8101ebbbbdb8958351de Mon Sep 17 00:00:00 2001 From: Peter Vingelmann Date: Fri, 5 Mar 2021 19:42:30 +0100 Subject: [PATCH 13/13] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4e14eb0..521ebb2 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ For testing local MetaDefender Core, you are free to specify any custom workflow The folder path where the files to be tested are located, e.g.: */home/user1/tester* +This path must point to a valid folder that contains at least one file! + ***PollingIntervals:*** Sleep time between polling scan results. (milliseconds)