#14351 updated geo and Characters

dotCMS/dependencies.gradle

@@ -231,7 +231,8 @@ dependencies {
 
     compile group: 'redis.clients', name: 'jedis', version:'2.7.3'
     compile group: 'org.apache.commons', name: 'commons-pool2', version:'2.0'
-    compile group: 'com.dotcms.lib', name: 'dot.maxmind-db', version:'1.0.0_2'
+    compile 'com.maxmind.geoip2:geoip2:2.12.0'
+
     compile group: 'org.openjdk.jmh', name: 'jmh-core', version:'0.1'
     compile group: 'com.dotcms.lib', name: 'dot.geoip2', version:'2.1.0_2'
     compile group: 'com.dotcms.lib', name: 'dot.aopalliance-repackaged', version:'2.4.0-b10_2'

dotCMS/src/main/java/com/dotcms/util/GeoIp2CityDbUtil.java

@@ -12,16 +12,18 @@
 
 import javax.servlet.http.HttpServletRequest;
 
-import com.dotcms.repackage.com.maxmind.geoip2.DatabaseReader;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
-import com.dotcms.repackage.com.maxmind.geoip2.model.CityResponse;
-import com.dotcms.repackage.com.maxmind.geoip2.record.City;
-import com.dotcms.repackage.com.maxmind.geoip2.record.Country;
-import com.dotcms.repackage.com.maxmind.geoip2.record.Subdivision;
+
 import com.dotmarketing.exception.DotRuntimeException;
 import com.dotmarketing.portlets.rules.conditionlet.Location;
 import com.dotmarketing.util.Config;
 import com.dotmarketing.util.Logger;
+import com.maxmind.db.CHMCache;
+import com.maxmind.geoip2.DatabaseReader;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.maxmind.geoip2.model.CityResponse;
+import com.maxmind.geoip2.record.City;
+import com.maxmind.geoip2.record.Country;
+import com.maxmind.geoip2.record.Subdivision;
 
 /**
  * Provides utility methods to interact with the GeoIP2 API City Database. This
@@ -113,7 +115,7 @@ private static void connectToDatabase(File database) {
 			if (databaseReader != null) {
 				databaseReader.close();
 			}
-			databaseReader = new DatabaseReader.Builder(database).build();
+			databaseReader = new DatabaseReader.Builder(database).withCache(new CHMCache()).build();
 			lastModified = database.lastModified();
 		} catch (IOException e) {
 			Logger.error(GeoIp2CityDbUtil.class,
@@ -219,7 +221,7 @@ public String getContinent(String ipAddress) throws IOException, GeoIp2Exception
 	}
 
 
-	private com.dotcms.repackage.com.maxmind.geoip2.record.Location getLocation(String ipAddress)
+	private com.maxmind.geoip2.record.Location getLocation(String ipAddress)
 			throws IOException, GeoIp2Exception {
 		InetAddress inetAddress = InetAddress.getByName(ipAddress);
 		CityResponse city = getDatabaseReader().city(inetAddress);
@@ -236,7 +238,7 @@ private com.dotcms.repackage.com.maxmind.geoip2.record.Location getLocation(Stri
      */
 	public Location getLocationByIp(String ipAddress)
 			throws IOException, GeoIp2Exception {
-		com.dotcms.repackage.com.maxmind.geoip2.record.Location location = getLocation(ipAddress);
+		com.maxmind.geoip2.record.Location location = getLocation(ipAddress);
 		return new Location(location.getLatitude(), location.getLongitude());
 	}
 
@@ -252,7 +254,7 @@ public Location getLocationByIp(String ipAddress)
 	 */
 	public String getLocationAsString(String ipAddress) throws IOException, GeoIp2Exception {
 
-		com.dotcms.repackage.com.maxmind.geoip2.record.Location location = getLocation(ipAddress);
+		com.maxmind.geoip2.record.Location location = getLocation(ipAddress);
 		if(location==null) return null;
 		StringWriter sw = new StringWriter();
 		try {

dotCMS/src/main/java/com/dotcms/visitor/filter/characteristics/BaseCharacter.java

@@ -18,6 +18,8 @@
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
+import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -26,19 +28,13 @@
 
 public class BaseCharacter extends AbstractCharacter {
 
-    private final static String CLUSTER_ID;
-    private final static String SERVER_ID;
-    private final ShortyIdAPI shorty = APILocator.getShortyAPI();
-    static {
-        CLUSTER_ID = ClusterFactory.getClusterId();
-        SERVER_ID = APILocator.getServerAPI().readServerId();
-
-    }
 
 
     private BaseCharacter(final HttpServletRequest request, final HttpServletResponse response, final Visitor visitor) {
         super(request, response, visitor);
         clearMap();
+        final String CLUSTER_ID = ClusterFactory.getClusterId();
+        final String SERVER_ID = APILocator.getServerAPI().readServerId();
         final Object asset = (request.getAttribute("idInode") != null) ? request.getAttribute("idInode")
                 : (Identifier) request.getAttribute(Constants.CMS_FILTER_IDENTITY);
 
@@ -60,18 +56,22 @@ private BaseCharacter(final HttpServletRequest request, final HttpServletRespons
         final Language lang = WebAPILocator.getLanguageWebAPI().getLanguage(request);
         IAm iAm = resolveResourceType(uri, getHostNoThrow(request), lang.getId());
         final long pageProcessingTime = (Long) request.getAttribute(VisitorFilter.DOTPAGE_PROCESSING_TIME);
+        final Host host = WebAPILocator.getHostWebAPI().getCurrentHostNoThrow(request);
+
+
+
         myMap.get().put("id", UUID.randomUUID().toString());
         myMap.get().put("status", response.getStatus());
         myMap.get().put("iAm", iAm);
-        myMap.get().put("uri", uri);
+        myMap.get().put("url", request.getRequestURL() + (request.getQueryString()!=null ? "?" + request.getQueryString() : ""));
         myMap.get().put("ms", pageProcessingTime);
-
-        myMap.get().put("cluster", shorty.shortify(CLUSTER_ID));
-        myMap.get().put("server", shorty.shortify(SERVER_ID));
+        myMap.get().put("hostId", host.getIdentifier());
+        myMap.get().put("cluster", CLUSTER_ID);
+        myMap.get().put("server", SERVER_ID);
         myMap.get().put("session", request.getSession().getId());
-        myMap.get().put("sessionNew", request.getSession().isNew());
-        myMap.get().put("time", System.currentTimeMillis());
-
+        myMap.get().put("sessionNew", (request.getSession(false)!=null) && request.getSession(false).isNew());
+        myMap.get().put("time", Instant.now().getEpochSecond());
+        myMap.get().put("utc_time", Instant.now().atOffset(ZoneOffset.UTC).toInstant().getEpochSecond());
         myMap.get().put("mime", response.getContentType());
         myMap.get().put("vanityUrl", (String) request.getAttribute(VisitorFilter.VANITY_URL_ATTRIBUTE));
         myMap.get().put("referer", request.getHeader("referer"));

dotCMS/src/main/java/com/dotcms/visitor/filter/characteristics/GeoCharacter.java

@@ -22,10 +22,12 @@ public GeoCharacter(AbstractCharacter incomingCharacter) {
                 String ipAddress = visitor.getIpAddress().getHostAddress();
                 GeoIp2CityDbUtil geo = GeoIp2CityDbUtil.getInstance();
                 m = new ImmutableMap.Builder<String, String>().put("g.latLong", geo.getLocationAsString(ipAddress))
+                    .put("g.latLong", geo.getLocationAsString(ipAddress))
                     .put("g.countryCode", geo.getCountryIsoCode(ipAddress))
                     .put("g.cityName", geo.getCityName(ipAddress))
                     .put("g.continent", geo.getContinent(ipAddress))
-                    .put("g.company", geo.getCompany())
+                    .put("g.timezone", geo.getTimeZone(ipAddress).getID())
+                    .put("g.subdivisionCode", geo.getSubdivisionIsoCode(ipAddress))
                     .build();
             } catch (Exception e) {
                 m = ImmutableMap.of("g.ip", "ukn");

dotCMS/src/main/java/com/dotcms/visitor/filter/characteristics/HeaderCharacter.java

@@ -10,7 +10,7 @@ public class HeaderCharacter extends AbstractCharacter {
 
     private final Set<String> WHITELISTED_HEADERS = new HashSet<>(
             Arrays.asList(
-                    Config.getStringProperty("WHITELISTED_HEADERS", "").toLowerCase().split(",")));
+                    Config.getStringProperty("WHITELISTED_HEADERS", "User-Agent,Accept-Languag,Host,DNT").toLowerCase().split(",")));
 
 
     public HeaderCharacter(AbstractCharacter incomingCharacter) {

dotCMS/src/main/java/com/dotcms/visitor/filter/characteristics/ParamsCharacter.java

@@ -13,7 +13,7 @@ public class ParamsCharacter extends AbstractCharacter {
 
     private static final Set<String> WHITELISTED_PARAMS =
             new HashSet<>(Arrays.asList(
-                    Config.getStringProperty("WHITELISTED_PARAMS", "").toLowerCase().split(",")));
+                Config.getStringProperty("WHITELISTED_PARAMS", "q,search,language_id,tags,id").toLowerCase().split(",")));
 
     public ParamsCharacter(AbstractCharacter incomingCharacter) {
         super(incomingCharacter);

dotCMS/src/main/java/com/dotmarketing/portlets/rules/conditionlet/DateTimeConditionlet.java

@@ -1,7 +1,6 @@
 package com.dotmarketing.portlets.rules.conditionlet;
 
 import com.dotcms.repackage.com.google.common.annotations.VisibleForTesting;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
 import org.apache.logging.log4j.util.Strings;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotcms.util.HttpRequestDataUtil;
@@ -15,6 +14,7 @@
 import com.dotmarketing.portlets.rules.parameter.display.*;
 import com.dotmarketing.portlets.rules.parameter.type.*;
 import com.dotmarketing.util.Logger;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;

dotCMS/src/main/java/com/dotmarketing/portlets/rules/conditionlet/UsersCountryConditionlet.java

@@ -1,7 +1,7 @@
 package com.dotmarketing.portlets.rules.conditionlet;
 
-import com.dotcms.repackage.com.google.common.annotations.VisibleForTesting;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.google.common.annotations.VisibleForTesting;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotcms.util.HttpRequestDataUtil;
 import com.dotmarketing.portlets.rules.RuleComponentInstance;

dotCMS/src/main/java/com/dotmarketing/portlets/rules/conditionlet/VisitorsGeolocationConditionlet.java

@@ -1,7 +1,19 @@
 package com.dotmarketing.portlets.rules.conditionlet;
 
+import static com.dotcms.repackage.com.google.common.base.Preconditions.checkState;
+import static com.dotmarketing.portlets.rules.parameter.comparison.Comparison.NOT_WITHIN_DISTANCE;
+import static com.dotmarketing.portlets.rules.parameter.comparison.Comparison.WITHIN_DISTANCE;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
 import com.dotcms.repackage.com.google.common.annotations.VisibleForTesting;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotcms.util.HttpRequestDataUtil;
 import com.dotmarketing.portlets.rules.RuleComponentInstance;
@@ -19,16 +31,6 @@
 import com.dotmarketing.portlets.rules.parameter.type.TextType;
 import com.dotmarketing.util.Logger;
 
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Map;
-
-import static com.dotcms.repackage.com.google.common.base.Preconditions.checkState;
-import static com.dotmarketing.portlets.rules.parameter.comparison.Comparison.*;
-
 /**
  * Conditionlet expects all input to be in Meters. The unit type exists to inform the UI what should be displayed to the user
  */

dotCMS/src/main/java/com/dotmarketing/servlets/InitServlet.java

@@ -4,7 +4,7 @@
 import com.dotcms.cluster.business.HazelcastUtil;
 import com.dotcms.content.elasticsearch.util.ESClient;
 import com.dotcms.enterprise.LicenseUtil;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 import com.dotcms.services.VanityUrlServices;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotmarketing.beans.Host;

dotCMS/src/test/java/com/dotmarketing/portlets/rules/conditionlet/DateTimeConditionletTest.java

@@ -3,7 +3,7 @@
 import com.dotcms.UnitTestBase;
 import com.dotcms.repackage.com.google.common.collect.Lists;
 import com.dotcms.repackage.com.google.common.collect.Maps;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 import com.dotcms.unittest.TestUtil;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotmarketing.portlets.rules.exception.ComparisonNotSupportedException;

dotCMS/src/test/java/com/dotmarketing/portlets/rules/conditionlet/UsersCountryConditionletTest.java

@@ -1,9 +1,9 @@
 package com.dotmarketing.portlets.rules.conditionlet;
 
 import com.dotcms.UnitTestBase;
-import com.dotcms.repackage.com.google.common.collect.Lists;
-import com.dotcms.repackage.com.google.common.collect.Maps;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 import com.dotcms.unittest.TestUtil;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotmarketing.portlets.rules.exception.ComparisonNotSupportedException;

dotCMS/src/test/java/com/dotmarketing/portlets/rules/conditionlet/VisitorsGeolocationConditionletTest.java

@@ -1,9 +1,9 @@
 package com.dotmarketing.portlets.rules.conditionlet;
 
 import com.dotcms.UnitTestBase;
-import com.dotcms.repackage.com.google.common.collect.Lists;
-import com.dotcms.repackage.com.google.common.collect.Maps;
-import com.dotcms.repackage.com.maxmind.geoip2.exception.GeoIp2Exception;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.maxmind.geoip2.exception.GeoIp2Exception;
 import com.dotcms.unittest.TestUtil;
 import com.dotcms.util.GeoIp2CityDbUtil;
 import com.dotmarketing.portlets.rules.exception.ComparisonNotSupportedException;