-
Notifications
You must be signed in to change notification settings - Fork 179
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
GitHub-issue#253 : Implemented GeoIP processor integration test #2927
Changes from 3 commits
607bba5
063851e
c2357f4
d4a3d31
4099f54
248d527
a0dbffd
53d7e26
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.dataprepper.plugins.processor; | ||
|
||
public class GeoIPInputJson { | ||
Peer PeerObject; | ||
private String status; | ||
|
||
// Getter Methods | ||
public Peer getPeer() { | ||
return PeerObject; | ||
} | ||
public String getStatus() { | ||
return status; | ||
} | ||
// Setter Methods | ||
public void setPeer( Peer peerObject ) { | ||
this.PeerObject = peerObject; | ||
} | ||
public void setStatus( String status ) { | ||
this.status = status; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.dataprepper.plugins.processor; | ||
|
||
import com.fasterxml.jackson.core.JsonProcessingException; | ||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; | ||
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; | ||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; | ||
import org.junit.jupiter.api.BeforeEach; | ||
import org.junit.jupiter.api.Test; | ||
import org.junit.jupiter.api.extension.ExtendWith; | ||
import org.mockito.junit.jupiter.MockitoExtension; | ||
import org.opensearch.dataprepper.plugins.processor.utils.IPValidationcheck; | ||
|
||
import java.io.File; | ||
import java.net.InetAddress; | ||
import java.net.UnknownHostException; | ||
import java.util.HashMap; | ||
import java.util.Map; | ||
|
||
import static org.hamcrest.MatcherAssert.assertThat; | ||
import static org.hamcrest.Matchers.equalTo; | ||
|
||
@ExtendWith(MockitoExtension.class) | ||
public class GeoIPProcessorUrlServiceIT { | ||
|
||
private ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS)); | ||
private String tempPath; | ||
private GeoIPProcessorConfig geoIPProcessorConfig; | ||
private String ipAddress; | ||
private String maxmindLicenseKey; | ||
private GeoIPProcessorService geoIPProcessorService; | ||
private GeoIPInputJson geoIPInputJson; | ||
private String jsonInput; | ||
private static final String TEMP_PATH_FOLDER = "GeoIP"; | ||
public static final String DATABASE_1 = "first_database"; | ||
public static final String URL_SUFFIX = "&suffix=tar.gz"; | ||
|
||
@BeforeEach | ||
public void setUp() throws JsonProcessingException { | ||
|
||
maxmindLicenseKey = System.getProperty("tests.geoipProcessor.maxmindLicenseKey"); | ||
|
||
jsonInput = "{\"peer\": {\"ip\": \"8.8.8.8\", \"host\": \"example.org\" }, \"status\": \"success\"}"; | ||
|
||
geoIPInputJson = objectMapper.readValue(jsonInput, GeoIPInputJson.class); | ||
tempPath = System.getProperty("java.io.tmpdir")+ File.separator + TEMP_PATH_FOLDER; | ||
|
||
String asnUrl = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=" + maxmindLicenseKey + URL_SUFFIX; | ||
String cityUrl = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=" + maxmindLicenseKey + URL_SUFFIX; | ||
String countryUrl = "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=" + maxmindLicenseKey + URL_SUFFIX; | ||
|
||
String pipelineConfig = " aws:\n" + | ||
" region: us-east-2\n" + | ||
" sts_role_arn: \"arn:aws:iam::123456789:role/data-prepper-execution-role\"\n" + | ||
" keys:\n" + | ||
" - key:\n" + | ||
" source: \"/peer/ip\"\n" + | ||
" target: \"target1\"\n" + | ||
" - key:\n" + | ||
" source: \"/peer/ip2\"\n" + | ||
" target: \"target2\"\n" + | ||
" attributes: [\"city_name\",\"country_name\"]\n" + | ||
" service_type:\n" + | ||
" maxmind:\n" + | ||
" database_path:\n" + | ||
" - url: " + asnUrl + "\n" + | ||
" - url: " + cityUrl + "\n" + | ||
" - url: " + countryUrl + "\n" + | ||
" load_type: \"cache\"\n" + | ||
" cache_size: 8192\n" + | ||
" cache_refresh_schedule: PT3M"; | ||
|
||
objectMapper.registerModule(new JavaTimeModule()); | ||
this.geoIPProcessorConfig = objectMapper.readValue(pipelineConfig, GeoIPProcessorConfig.class); | ||
} | ||
|
||
public GeoIPProcessorService createObjectUnderTest() { | ||
return new GeoIPProcessorService(geoIPProcessorConfig, tempPath); | ||
} | ||
|
||
@Test | ||
void verify_enrichment_of_data_from_maxmind_url() throws UnknownHostException { | ||
|
||
Map<String, Object> geoData = new HashMap<>(); | ||
this.geoIPProcessorService = createObjectUnderTest(); | ||
String ipAddress = geoIPInputJson.getPeer().getIp(); | ||
if (IPValidationcheck.isPublicIpAddress(ipAddress)) { | ||
InetAddress inetAddress = InetAddress.getByName(ipAddress); | ||
//All attributes are considered by default with null | ||
geoData = geoIPProcessorService.getGeoData(inetAddress, null); | ||
|
||
assertThat(geoData.get("country_iso_code"), equalTo("US")); | ||
assertThat(geoData.get("ip"), equalTo("/8.8.8.8")); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why is it "/8.8.8.8" and not "8.8.8.8"? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Updated the code to get the valid IP in PR#2925 also updated assertion statement with "8.8.8.8" instead of "/8.8.8.8". |
||
assertThat(geoData.get("country_name"), equalTo("United States")); | ||
assertThat(geoData.get("organization_name"), equalTo("GOOGLE")); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.dataprepper.plugins.processor; | ||
|
||
public class Peer { | ||
private String ip; | ||
private String host; | ||
|
||
// Getter Methods | ||
public String getIp() { | ||
return ip; | ||
} | ||
public String getHost() { | ||
return host; | ||
} | ||
// Setter Methods | ||
public void setIp( String ip ) { | ||
this.ip = ip; | ||
} | ||
public void setHost( String host ) { | ||
this.host = host; | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Usually, you can do this by Mocking the config. There are many IT.java files where you can see how to do this. Take a look at
./data-prepper-plugins/aggregate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessorIT.java
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As per David suggestion, we have added yaml input (configuration) in other plugins like S3Source.
For example in S3Source: S3SelectCSVOptionTest.java, S3ScanScanOptionsTest.java.
In case integration testing, the mocking should be avoided as much as possible.