-
Notifications
You must be signed in to change notification settings - Fork 217
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
'#1487 First version of nominating task that queries a configured
nominatim server for a related address of a georeferenced item.
- Loading branch information
1 parent
3ae5c93
commit 967fbd9
Showing
7 changed files
with
299 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| #host name of the nominatim server | ||
| hostName=10.65.6.2 | ||
|
|
||
| #port of the nominatim server service | ||
| hostPort=8080 | ||
|
|
||
| #protocol of the nominatim server service (http or https) | ||
| protocol=http | ||
|
|
||
| #protocol of the nominatim server service (http or https) | ||
| connectionPoolSize=auto | ||
|
|
||
| #Query to be passed via URL to test if service is available at task init | ||
| serviceTestUrlQuery=/?country=brazil |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 86 additions & 0 deletions
86
iped-geo/src/main/java/iped/geo/nominatim/NominatimConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| package iped.geo.nominatim; | ||
|
|
||
| import iped.engine.config.AbstractTaskPropertiesConfig; | ||
| import iped.utils.UTF8Properties; | ||
|
|
||
| public class NominatimConfig extends AbstractTaskPropertiesConfig { | ||
| private static final long serialVersionUID = 1L; | ||
|
|
||
| public static final String ENABLE_PARAM = "enableNominatimLocationResolving"; | ||
| private static final String CONF_FILE = "NominatimConfig.txt"; | ||
|
|
||
| String protocol = "http"; | ||
| String hostName = "10.65.6.2"; | ||
| String hostPort = "8080"; | ||
| int connectionPoolSize; | ||
| String serviceTestUrlQuery = "/"; | ||
|
|
||
| private boolean enableNominatim = false; | ||
|
|
||
| @Override | ||
| public void processProperties(UTF8Properties properties) { | ||
| String value = properties.getProperty("protocol"); //$NON-NLS-1$ | ||
| if (value != null) { | ||
| protocol = value.trim(); | ||
| } | ||
|
|
||
| value = properties.getProperty("hostName"); //$NON-NLS-1$ | ||
| if (value != null) { | ||
| hostName = value.trim(); | ||
| } | ||
|
|
||
| value = properties.getProperty("serviceTestUrlQuery"); //$NON-NLS-1$ | ||
| if (value != null) { | ||
| serviceTestUrlQuery = value.trim(); | ||
| } | ||
|
|
||
| value = properties.getProperty("hostPort"); //$NON-NLS-1$ | ||
| if (value != null) { | ||
| hostPort = Integer.toString(Integer.valueOf(value.trim())); | ||
| } | ||
|
|
||
| value = properties.getProperty("connectionPoolSize"); // $NON-NLS-1$ | ||
| if (value != null && !value.trim().equalsIgnoreCase("auto")) { //$NON-NLS-1$ | ||
| connectionPoolSize = Integer.valueOf(value.trim()); | ||
| } else { | ||
| connectionPoolSize = Math.max((int) Math.ceil((float) Runtime.getRuntime().availableProcessors()), 2); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String getTaskEnableProperty() { | ||
| // TODO Auto-generated method stub | ||
| return ENABLE_PARAM; | ||
| } | ||
|
|
||
| @Override | ||
| public String getTaskConfigFileName() { | ||
| // TODO Auto-generated method stub | ||
| return CONF_FILE; | ||
| } | ||
|
|
||
| public String getProtocol() { | ||
| return protocol; | ||
| } | ||
|
|
||
| public String getHostName() { | ||
| return hostName; | ||
| } | ||
|
|
||
| public String getHostPort() { | ||
| return hostPort; | ||
| } | ||
|
|
||
| public int getConnectionPoolSize() { | ||
| return connectionPoolSize; | ||
| } | ||
|
|
||
| public boolean isEnableNominatim() { | ||
| return enableNominatim; | ||
| } | ||
|
|
||
| public String getServiceTestUrlQuery() { | ||
| return serviceTestUrlQuery; | ||
| } | ||
|
|
||
| } |
190 changes: 190 additions & 0 deletions
190
iped-geo/src/main/java/iped/geo/nominatim/NominatimTask.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,190 @@ | ||
| package iped.geo.nominatim; | ||
|
|
||
| import java.io.BufferedReader; | ||
| import java.io.InputStreamReader; | ||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.concurrent.atomic.AtomicInteger; | ||
|
|
||
| import org.apache.http.HeaderElement; | ||
| import org.apache.http.HeaderElementIterator; | ||
| import org.apache.http.HttpResponse; | ||
| import org.apache.http.client.ClientProtocolException; | ||
| import org.apache.http.client.methods.CloseableHttpResponse; | ||
| import org.apache.http.client.methods.HttpGet; | ||
| import org.apache.http.conn.ConnectionKeepAliveStrategy; | ||
| import org.apache.http.conn.HttpClientConnectionManager; | ||
| import org.apache.http.impl.client.CloseableHttpClient; | ||
| import org.apache.http.impl.client.HttpClients; | ||
| import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; | ||
| import org.apache.http.message.BasicHeaderElementIterator; | ||
| import org.apache.http.protocol.HTTP; | ||
| import org.apache.http.protocol.HttpContext; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import iped.configuration.Configurable; | ||
| import iped.data.IItem; | ||
| import iped.engine.config.ConfigurationManager; | ||
| import iped.engine.task.AbstractTask; | ||
| import iped.geo.parsers.GeofileParser; | ||
| import iped.properties.ExtraProperties; | ||
|
|
||
| public class NominatimTask extends AbstractTask { | ||
|
|
||
| private static Logger LOGGER = LoggerFactory.getLogger(NominatimTask.class); | ||
|
|
||
| static final String NOMINATIM_METADATA = "nominatim:geojson"; | ||
| protected static final long CUSTOM_KEEP_ALIVE = 5000; | ||
|
|
||
| /* The http connection pool is static to be used application wide */ | ||
| static PoolingHttpClientConnectionManager cm = null; | ||
| static ConnectionKeepAliveStrategy myStrategy; | ||
| static IdleConnectionMonitorThread staleMonitor; | ||
| static CloseableHttpClient httpClient; | ||
|
|
||
| AtomicInteger count = new AtomicInteger(0); | ||
|
|
||
| static String baseUrl = ""; | ||
|
|
||
| static boolean unresolvedServer = true; | ||
|
|
||
| private NominatimConfig nominatimConfig; | ||
|
|
||
| @Override | ||
| public List<Configurable<?>> getConfigurables() { | ||
| return Arrays.asList(new NominatimConfig()); | ||
| } | ||
|
|
||
| @Override | ||
| public void init(ConfigurationManager configurationManager) throws Exception { | ||
| count.incrementAndGet(); | ||
| if (cm == null) { | ||
| nominatimConfig = (NominatimConfig) configurationManager.findObject(NominatimConfig.class); | ||
|
|
||
| cm = new PoolingHttpClientConnectionManager(); | ||
| cm.setMaxTotal(200); | ||
| cm.setDefaultMaxPerRoute(nominatimConfig.getConnectionPoolSize()); | ||
| myStrategy = new ConnectionKeepAliveStrategy() { | ||
| @Override | ||
| public long getKeepAliveDuration(HttpResponse response, HttpContext context) { | ||
| HeaderElementIterator it = new BasicHeaderElementIterator( | ||
| response.headerIterator(HTTP.CONN_KEEP_ALIVE)); | ||
| while (it.hasNext()) { | ||
| HeaderElement he = it.nextElement(); | ||
| String param = he.getName(); | ||
| String value = he.getValue(); | ||
| if (value != null && param.equalsIgnoreCase("timeout")) { | ||
| return Long.parseLong(value) * 1000; | ||
| } | ||
| } | ||
| return CUSTOM_KEEP_ALIVE; | ||
| } | ||
| }; | ||
|
|
||
| try { | ||
| httpClient = HttpClients.custom().setKeepAliveStrategy(myStrategy) | ||
| .setConnectionManager(cm).build(); | ||
| String defaultTestCountry = "brazil"; | ||
| baseUrl = nominatimConfig.getProtocol() + "://" + nominatimConfig.getHostName() + ":" | ||
| + nominatimConfig.getHostPort(); | ||
| HttpGet get = new HttpGet(baseUrl + nominatimConfig.getServiceTestUrlQuery()); | ||
| try (CloseableHttpResponse response = httpClient.execute(get)) { | ||
| unresolvedServer = false; | ||
| } catch (ClientProtocolException cpe) { | ||
| unresolvedServer = true; | ||
| } | ||
| } catch (Exception e) { | ||
| unresolvedServer = true; | ||
| } | ||
|
|
||
| /* thread to monitor connections closed by the host */ | ||
| staleMonitor = new IdleConnectionMonitorThread(cm); | ||
| staleMonitor.start(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void finish() throws Exception { | ||
| int remaining = count.decrementAndGet(); | ||
|
|
||
| if (remaining == 0) { | ||
| httpClient.close(); | ||
| staleMonitor.shutdown(); | ||
| cm.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| protected void process(IItem evidence) throws Exception { | ||
| if (!unresolvedServer) { | ||
| String featureString = evidence.getMetadata().get(GeofileParser.FEATURE_STRING); | ||
|
|
||
| if (featureString == null || featureString.length() < 5) { | ||
| String location = evidence.getMetadata().get(ExtraProperties.LOCATIONS); | ||
|
|
||
| if (location != null && location.length() >= 1) { | ||
| String[] locs = location.split(";"); //$NON-NLS-1$ | ||
| String lat = locs[0].trim(); | ||
| String longit = locs[1].trim(); | ||
|
|
||
| try { | ||
| HttpGet get = new HttpGet(baseUrl + "/reverse?format=geojson&lat=" + lat + "&lon=" + longit); | ||
| try (CloseableHttpResponse response = httpClient.execute(get)) { | ||
| try (BufferedReader in = new BufferedReader( | ||
| new InputStreamReader(response.getEntity().getContent()))) { | ||
| String inputLine; | ||
| StringBuffer content = new StringBuffer(); | ||
| while ((inputLine = in.readLine()) != null) { | ||
| content.append(inputLine); | ||
| } | ||
|
|
||
| evidence.getMetadata().add(NOMINATIM_METADATA, content.toString()); | ||
| } | ||
| } | ||
| } catch (ClientProtocolException cpe) { | ||
| cpe.printStackTrace(); | ||
| LOGGER.warn(cpe.getMessage()); | ||
| } catch (Exception e) { | ||
| LOGGER.warn(e.getMessage()); | ||
| e.printStackTrace(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| class IdleConnectionMonitorThread extends Thread { | ||
| private final HttpClientConnectionManager connMgr; | ||
| private volatile boolean shutdown; | ||
|
|
||
| public IdleConnectionMonitorThread(PoolingHttpClientConnectionManager connMgr) { | ||
| super(); | ||
| this.connMgr = connMgr; | ||
| } | ||
|
|
||
| @Override | ||
| public void run() { | ||
| try { | ||
| while (!shutdown) { | ||
| synchronized (this) { | ||
| wait(CUSTOM_KEEP_ALIVE); | ||
| connMgr.closeExpiredConnections(); | ||
| connMgr.closeIdleConnections(30, TimeUnit.SECONDS); | ||
| } | ||
| } | ||
| } catch (InterruptedException ex) { | ||
| shutdown(); | ||
| } | ||
| } | ||
|
|
||
| public void shutdown() { | ||
| shutdown = true; | ||
| synchronized (this) { | ||
| notifyAll(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |