Merge pull request #5 from OPSWAT/features/Sanitization

Features/sanitization

README.md

@@ -1,67 +1,109 @@
-Using Gatling to perform MetaDefender performance test
+# MetaDefender Gatling performance test
 
-Documentation
+Useful links:
 
 * [MetaDefender Core developer guide](https://onlinehelp.opswat.com/corev4/9._%28NEW%29_MetaDefender_Core_Developer_Guide.html)
-* [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html)
+* [File scanning and sanitization API (cloud)](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html)
 * [Gatling](https://gatling.io/)
 
+## Requirements
+
+- Java 11 installed (Java 8 is not supported)
+
+## Build
+
 To build the jar, make sure you have Maven:
 
 	mvn clean install
 
 The install step will copy `src/main/resources/config.ini` to `target/config.ini`.
 
+## Configuration
+
 You can modify the settings in this configuration file: `target/config.ini`
 
-Example configuration:
+
+Example configuration for **Metadefender Cloud** usage:
 ```
 [general]
-BaseUrl: https://api.metadefender.com/v4/file
-ConstantUsers: 5
-TestDuration: 10
-ScanWorkflow: multiscan
-LocalPath: /home/user/testfiles
-PollingIntervals: 500
-ApiKey: 1234567890abcdefghijklmnopqrstuv
-WaitBeforePolling: 1000
+BaseUrl=https://api.metadefender.com/v4/file
+ConstantUsers=5
+TestDuration=10
+ScanWorkflow=multiscan,sanitize
+LocalPath=/home/opswatuser/testfiles
+PollingIntervals=500
+ApiKey=1234567890abcdefghijklmnopqrstuv
+WaitBeforePolling=1000
 ```
 
-***BaseURL***
+Example configuration for **local MetaDefender Core** usage:
+```
+[general]
+BaseUrl=http://localhost:8008/file
+ConstantUsers=5
+TestDuration=10
+LocalPath=/home/opswatuser/testfiles
+PollingIntervals=500
+WaitBeforePolling=1000
+```
 
-MetaDefender REST URL (e.g.: https://api.metadefender.com/v4/file). More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html)
+***BaseURL:***
 
-***ConstantUsers***
+MetaDefender REST URL, e.g.: https://api.metadefender.com/v4/file (Cloud) or http://localhost:8008/file (local). 
+More information about file scanning: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html)
+
+***ConstantUsers:***
 
 The number of constant concurrent users for the test. Each simulated user will submit a randomly 
 selected file for scanning, wait for the scan result, then select a new file for scanning.
 
-***TestDuration***
+***TestDuration:***
 
 Total test duration while the simulated users will continue to submit files. (seconds)
 
-***ScanWorkflow***
+***ScanWorkflow [optional]:***
+
+MetaDefender scan workflow rule to activate. Multiple values can be separated by "," to combine multiple workflows. For Cloud-based testing, it can be `multiscan`, `sanitize` or `unarchive`. More details about Cloud workflows: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule*
 
-MetaDefender workflow rule to activate. Multiple values can be sent separated by "," to combine multiple workflows. It can be `multiscan`, `sanitize` or `unarchive`. More details: [File scanning API](https://onlinehelp.opswat.com/mdcloud/2.1_Scanning_a_file_by_file_upload.html) -> *Request* -> *rule*
+For testing local MetaDefender Core, you are free to specify any custom workflow that is available on your installation.
 
-***LocalPath***
+***LocalPath:***
 
 The folder path where the files to be tested are located, e.g.: */home/user1/tester*
 
-***PollingIntervals***
+This path must point to a valid folder that contains at least one file!
+
+***PollingIntervals:***
 
 Sleep time between polling scan results. (milliseconds)
 
-***ApiKey***
+***ApiKey [Cloud usage only]:***
 
 OPSWAT MetaDefender Cloud API key. You can find your key at [metadefender.opswat.com](https://metadefender.opswat.com/account) -> *API key information and limits* -> *API key*. (Registration required.)
 
-***WaitBeforePolling***
+***WaitBeforePolling:***
 
 Waiting time after file submission to start polling. (milliseconds)
 
-<br>
+*****ShowPollingDetails:***
+
+If true, Gatling will also show the polling details as separate requests. By default, it is false. (true/false)
+
+*****DeveloperMode:***
+
+It true, Gatling will print the HTTP responses to the console. By default, it is false. (true/false)
+
+
+## Running a test
+
+The `metadefender-gatling-2.0.0-SNAPSHOT.jar` and `config.ini` files are required to run the test. 
+Maven will install these files in the `target` folder. 
+
+Run a test using the helper script:
+
+	./start.sh
+
+The script runs the jar file with the following parameters:
 
-To run the jar (in the `target` folder where the `.jar` file is generated):
+	java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation
 
-	java -cp metadefender-gatling-1.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation

pom.xml

@@ -4,7 +4,7 @@
   <groupId>com.opswat</groupId>
   <artifactId>metadefender-gatling</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.0-SNAPSHOT</version>
+  <version>2.0.0-SNAPSHOT</version>
   <name>metadefender-gatling</name>
   <url>http://maven.apache.org</url>
 

src/main/resources/config.ini

@@ -1,16 +1,16 @@
 [general]
 BaseUrl=https://api.metadefender.com/v4/file
 ;constant concurrent users
-ConstantUsers=5
+ConstantUsers=1
 ;time to run, (s)
 TestDuration=10
-;time before full filling, (s)
-ScanWorkflow=multiscan
+;file scanning rule
+ScanWorkflow=multiscan,sanitize
 ;dataset folder
 LocalPath=C:\OPSWAT\data
 ;sleep time between each polling scan result (ms)
 PollingIntervals=500
 ;OPSWAT MetaDefender Cloud apikey
 ApiKey=opswat_metadefender_api
 ;waiting time between push file and start to polling (ms)
-WaitBeforePolling=1000
+WaitBeforePolling=1000

src/main/scala/gatling/Config.scala

@@ -1,34 +1,60 @@
 
-import java.io.File
-
+import java.io.{File, IOException}
 import org.ini4j.Wini
+import java.nio.file.{Files, Paths}
 
 
 class Config {
   var baseUrl = ""
-  var scanWorkflow = ""
   var constantUser = 2
   var testDuration = 5
-  var rampupDuration = 5
-  var pollingIntervals = 500
+  var scanWorkflow = ""
   var localPath = ""
+  var pollingIntervals = 500
   var apikey = ""
   var waitBeforePolling = 1000
+  var pollingDetails = false
+  var developerMode = false
 }
 
 object Config {
+  def checkValues(config: Config): Unit ={
+    if(config.baseUrl==null||config.baseUrl==""){throw new IOException("MISSING ENDPOINT")}
+    if(config.constantUser<1){config.constantUser=2}
+    if(config.testDuration<1){config.testDuration=5}
+    if(config.scanWorkflow==null){config.scanWorkflow=""}
+
+    try{
+      assert(Files.list(Paths.get(config.localPath)).count()>0)
+    }
+    catch {
+      case _:Throwable => throw new IOException("WRONG PATH OR EMPTY DIRECTORY: "+config.localPath)
+    }
+
+    if(config.pollingIntervals<1){config.pollingIntervals=500}
+    if(config.apikey==null){config.apikey=""}
+    if(config.waitBeforePolling<1){config.waitBeforePolling=1000}
+  }
+
   def parseConfigure(filePath: String): Config = {
+
     val v = new Config()
     val ini = new Wini(new File(filePath))
+
     v.baseUrl = ini.get("general", "BaseUrl", classOf[String])
-    v.scanWorkflow = ini.get("general", "ScanWorkflow", classOf[String])
     v.constantUser = ini.get("general", "ConstantUsers", classOf[Int])
     v.testDuration = ini.get("general", "TestDuration", classOf[Int])
-    v.rampupDuration = ini.get("general", "RampupDuration", classOf[Int])
+    v.scanWorkflow = ini.get("general", "ScanWorkflow", classOf[String])
     v.localPath = ini.get("general", "LocalPath", classOf[String])
     v.pollingIntervals = ini.get("general", "PollingIntervals", classOf[Int])
     v.apikey = ini.get("general", "ApiKey", classOf[String])
+    v.apikey=ini.get("general", "ApiKey", classOf[String])
     v.waitBeforePolling = ini.get("general", "WaitBeforePolling", classOf[Int])
+    v.pollingDetails = ini.get("general", "ShowPollingDetails", classOf[Boolean])
+    v.developerMode = ini.get("general", "DeveloperMode", classOf[Boolean])
+
+    checkValues(v)
+
     v
   }
 }

src/main/scala/gatling/ScanSimulation.scala

@@ -4,6 +4,7 @@ import io.gatling.core.body.RawFileBody
 import io.gatling.core.structure.{ChainBuilder, ScenarioBuilder}
 import io.gatling.http.Predef._
 import io.gatling.http.protocol.HttpProtocolBuilder
+import io.gatling.http.request.builder.HttpRequestBuilder
 import scala.concurrent.duration._
 
 
@@ -17,57 +18,80 @@ class ScanSimulation extends Simulation {
     .acceptHeader("*/*")
     .disableCaching
 
-  object Scan {
-    val submitFile = http("submit-file")
-      .post(config.baseUrl)
-      .headers(Map("filename" -> "${filename}", "rule" -> config.scanWorkflow))
-      .header("apikey", config.apikey)
-      .header("Content-Type","application/octet-stream")
-      .body(RawFileBody("${filepath}"))
-      .check(status.is(200))
-      .check(jsonPath("$..data_id").find.exists)
-      .check(jsonPath("$..data_id").find.saveAs("dataId"))
+  object FileUpload {
+    def initFileUpload () : HttpRequestBuilder = {
+      var base = http("submit-file")
+        .post(config.baseUrl)
+        .header("filename","${filename}")
+
+      if(config.scanWorkflow != ""){
+        base = base.header("rule", config.scanWorkflow)
+      }
+
+      if(config.apikey != ""){
+        base = base.header("apikey", config.apikey)
+      }
+
+      base
+        .header("Content-Type","application/octet-stream")
+        .body(RawFileBody("${filepath}"))
+        .check(status.is(200))
+        .check(jsonPath("$..data_id").find.exists)
+        .check(jsonPath("$..data_id").find.saveAs("dataId"))
+    }
+
+    val submitFile = initFileUpload()
   }
 
   object ScanProgress {
-    private val getScanProgress =
-      http("get-scan-result")
+    def initScanProgress () : HttpRequestBuilder = {
+      var base = http("get-scan-result")
         .get(config.baseUrl + "/${dataId}")
-        .header("apikey", config.apikey)
-        .silent
+
+      if(config.apikey != ""){
+        base = base.header("apikey", config.apikey)
+      }
+
+      if(!config.pollingDetails){
+        base = base.silent
+      }
+
+      base
         .check(status.is(200))
-        .check(jsonPath("$..process_info.progress_percentage").find.saveAs("progress"))
-        .check(jsonPath("$..process_info.post_processing.actions_ran").optional.saveAs("sanitization"))
+        .check(jsonPath("$..process_info.progress_percentage").optional.saveAs("progress"))
+        .check( jsonPath( "$" ).optional.saveAs( "response_data" ) )
+    }
+
+    def printResponse (): ChainBuilder = {
+      exec( session => {
+        if(session.contains("response_data")) {
+          println("Response:")
+          println(session( "response_data").as[String])
+        }
+        session
+      })
+    }
+
+    private val getScanProgress = initScanProgress()
 
     val action: ChainBuilder =
       exec(_.set("progress", "0"))
-        .doIf(session => session("dataId").asOption[String].isDefined) {
+      .doIf(session => session("dataId").asOption[String].isDefined) {
           asLongAs(session => session("progress").as[String] != "100") {
-            pause(config.pollingIntervals.millis).exec(getScanProgress)
+            pause(config.pollingIntervals.millis)
+            .exec(getScanProgress)
+            .doIf(config.developerMode){
+              printResponse()
+            }
           }
-        }
-  }
-
-  <!-- uncomment to if you want to check sanitization result -->
-  <!--
-  object GetSanitized {
-    private val getSanitizedFile =
-      http("get-sanitized-file")
-        .get(config.baseUrl + "/converted/${dataId}")
-        .check(status.is(200))
-
-    val action: ChainBuilder = doIf(session => session("sanitization").asOption[String].contains("Sanitized")) {
-      exec(getSanitizedFile)
-    }
+      }
   }
-  -->
 
   val pipeline: ScenarioBuilder = scenario("scan-pipeline")
     .feed(localFiles.feeder)
-    .exec(Scan.submitFile)
+    .exec(FileUpload.submitFile)
     .pause(config.waitBeforePolling.milliseconds)
     .exec(ScanProgress.action)
-    //.exec(GetSanitized.action)
 
   setUp(
     pipeline

start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cd target
+java -cp metadefender-gatling-2.0.0-SNAPSHOT.jar io.gatling.app.Gatling -s ScanSimulation
+cd ..