Java 11 and 12 Support

pom.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
@@ -38,7 +38,8 @@
 
 
     <name>Reinforcement Learning Algorithms</name>
-    <description>Classical RL algorithms implemented in Java, including Q-Learn, R-Learn, SARSA, Actor-Critic</description>
+    <description>Classical RL algorithms implemented in Java, including Q-Learn, R-Learn, SARSA, Actor-Critic
+    </description>
     <url>https://github.com/chen0040/java-reinforcement-learning</url>
 
     <distributionManagement>
@@ -235,7 +236,6 @@
                     </plugin>
 
 
-
                     <plugin>
                         <groupId>org.codehaus.mojo</groupId>
                         <artifactId>findbugs-maven-plugin</artifactId>
@@ -361,8 +361,6 @@
                 </plugin>
 
 
-
-
             </plugins>
         </pluginManagement>
 
@@ -382,14 +380,13 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-resources-plugin</artifactId>
+                <version>2.3</version>
                 <configuration>
                     <encoding>UTF-8</encoding>
                 </configuration>
             </plugin>
 
 
-
-
         </plugins>
     </build>
 
@@ -488,23 +485,12 @@
         </dependency>
 
 
-
         <!-- Utilities -->
-
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <scope>provided</scope>
-            <version>${lombok.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>fastjson</artifactId>
             <version>1.2.41</version>
         </dependency>
-
-
     </dependencies>
 
 </project>

src/main/java/com/github/chen0040/rl/actionselection/AbstractActionSelectionStrategy.java

@@ -1,8 +1,8 @@
 package com.github.chen0040.rl.actionselection;
 
-import com.github.chen0040.rl.utils.IndexValue;
 import com.github.chen0040.rl.models.QModel;
 import com.github.chen0040.rl.models.UtilityModel;
+import com.github.chen0040.rl.utils.IndexValue;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -14,58 +14,50 @@
  */
 public abstract class AbstractActionSelectionStrategy implements ActionSelectionStrategy {
 
+    Map<String, String> attributes = new HashMap<>();
     private String prototype;
-    protected Map<String, String> attributes = new HashMap<String, String>();
 
-    public String getPrototype(){
-        return prototype;
+    @SuppressWarnings("Used-by-user")
+    public AbstractActionSelectionStrategy() {
+        this.prototype = this.getClass().getCanonicalName();
     }
 
-    public IndexValue selectAction(int stateId, QModel model, Set<Integer> actionsAtState) {
-        return new IndexValue();
+    @SuppressWarnings("Used-by-user")
+    public AbstractActionSelectionStrategy(final HashMap<String, String> attributes) {
+        this.attributes = attributes;
+        if (attributes.containsKey("prototype")) {
+            this.prototype = attributes.get("prototype");
+        }
     }
 
-    public IndexValue selectAction(int stateId, UtilityModel model, Set<Integer> actionsAtState) {
-        return new IndexValue();
+    @Override
+    public String getPrototype() {
+        return this.prototype;
     }
 
-    public AbstractActionSelectionStrategy(){
-        prototype = this.getClass().getCanonicalName();
+    @Override
+    public IndexValue selectAction(final int stateId, final QModel model, final Set<Integer> actionsAtState) {
+        return new IndexValue();
     }
 
-
-    public AbstractActionSelectionStrategy(HashMap<String, String> attributes){
-        this.attributes = attributes;
-        if(attributes.containsKey("prototype")){
-            this.prototype = attributes.get("prototype");
-        }
+    @Override
+    public IndexValue selectAction(final int stateId, final UtilityModel model, final Set<Integer> actionsAtState) {
+        return new IndexValue();
     }
 
-    public Map<String, String> getAttributes(){
-        return attributes;
+    @Override
+    public Map<String, String> getAttributes() {
+        return this.attributes;
     }
 
     @Override
-    public boolean equals(Object obj) {
-        ActionSelectionStrategy rhs = (ActionSelectionStrategy)obj;
-        if(!prototype.equalsIgnoreCase(rhs.getPrototype())) return false;
-        for(Map.Entry<String, String> entry : rhs.getAttributes().entrySet()) {
-            if(!attributes.containsKey(entry.getKey())) {
-                return false;
-            }
-            if(!attributes.get(entry.getKey()).equals(entry.getValue())){
-                return false;
-            }
-        }
-        for(Map.Entry<String, String> entry : attributes.entrySet()) {
-            if(!rhs.getAttributes().containsKey(entry.getKey())) {
-                return false;
-            }
-            if(!rhs.getAttributes().get(entry.getKey()).equals(entry.getValue())){
-                return false;
-            }
-        }
-        return true;
+    public boolean equals(final Object obj) {
+        final ActionSelectionStrategy rhs = (ActionSelectionStrategy) obj;
+        return this.prototype.equalsIgnoreCase(rhs.getPrototype()) &&
+                rhs.getAttributes().entrySet().stream().noneMatch(entry -> !this.attributes.containsKey(entry.getKey()) ||
+                        !this.attributes.get(entry.getKey()).equals(entry.getValue())) &&
+                this.attributes.entrySet().stream().noneMatch(entry -> !rhs.getAttributes().containsKey(entry.getKey()) ||
+                        !rhs.getAttributes().get(entry.getKey()).equals(entry.getValue()));
     }
 
     @Override

src/main/java/com/github/chen0040/rl/actionselection/ActionSelectionStrategyFactory.java

@@ -1,59 +1,61 @@
 package com.github.chen0040.rl.actionselection;
 
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
 
 
 /**
  * Created by xschen on 9/27/2015 0027.
  */
-public class ActionSelectionStrategyFactory {
-    public static ActionSelectionStrategy deserialize(String conf){
-        String[] comps = conf.split(";");
-
-        HashMap<String, String> attributes = new HashMap<String, String>();
-        for(int i=0; i < comps.length; ++i){
-            String comp = comps[i];
-            String[] field = comp.split("=");
-            if(field.length < 2) continue;
-            String fieldname = field[0].trim();
-            String fieldvalue = field[1].trim();
-
-            attributes.put(fieldname, fieldvalue);
+public class ActionSelectionStrategyFactory implements Serializable {
+
+    public static ActionSelectionStrategy deserialize(final String conf) {
+        final String[] comps = conf.split(";");
+
+        final HashMap<String, String> attributes = new HashMap<>();
+        for (final String comp : comps) {
+            final String[] field = comp.split("=");
+            if (field.length < 2) {
+                continue;
+            }
+
+            attributes.put(field[0].trim(), field[1].trim());
         }
-        if(attributes.isEmpty()){
+        if (attributes.isEmpty()) {
             attributes.put("prototype", conf);
         }
 
-        String prototype = attributes.get("prototype");
-        if(prototype.equals(GreedyActionSelectionStrategy.class.getCanonicalName())){
+        final String prototype = attributes.get("prototype");
+        if (prototype.equals(GreedyActionSelectionStrategy.class.getCanonicalName())) {
             return new GreedyActionSelectionStrategy();
-        } else if(prototype.equals(SoftMaxActionSelectionStrategy.class.getCanonicalName())){
+        } else if (prototype.equals(SoftMaxActionSelectionStrategy.class.getCanonicalName())) {
             return new SoftMaxActionSelectionStrategy();
-        } else if(prototype.equals(EpsilonGreedyActionSelectionStrategy.class.getCanonicalName())){
+        } else if (prototype.equals(EpsilonGreedyActionSelectionStrategy.class.getCanonicalName())) {
             return new EpsilonGreedyActionSelectionStrategy(attributes);
-        } else if(prototype.equals(GibbsSoftMaxActionSelectionStrategy.class.getCanonicalName())){
+        } else if (prototype.equals(GibbsSoftMaxActionSelectionStrategy.class.getCanonicalName())) {
             return new GibbsSoftMaxActionSelectionStrategy();
         }
 
         return null;
     }
 
-    public static String serialize(ActionSelectionStrategy strategy){
-        Map<String, String> attributes = strategy.getAttributes();
+    public static String serialize(final ActionSelectionStrategy strategy) {
+        final Map<String, String> attributes = strategy.getAttributes();
         attributes.put("prototype", strategy.getPrototype());
 
-        StringBuilder sb = new StringBuilder();
+        final StringBuilder sb = new StringBuilder();
         boolean first = true;
-        for(Map.Entry<String, String> entry : attributes.entrySet()){
-            if(first){
+        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
+            if (first) {
                 first = false;
-            }
-            else{
+            } else {
                 sb.append(";");
             }
-            sb.append(entry.getKey()+"="+entry.getValue());
+            sb.append(entry.getKey()).append("=").append(entry.getValue());
         }
         return sb.toString();
     }
+
+
 }

src/main/java/com/github/chen0040/rl/actionselection/EpsilonGreedyActionSelectionStrategy.java

@@ -1,7 +1,7 @@
 package com.github.chen0040.rl.actionselection;
 
-import com.github.chen0040.rl.utils.IndexValue;
 import com.github.chen0040.rl.models.QModel;
+import com.github.chen0040.rl.utils.IndexValue;
 
 import java.util.*;
 
@@ -10,69 +10,62 @@
  * Created by xschen on 9/27/2015 0027.
  */
 public class EpsilonGreedyActionSelectionStrategy extends AbstractActionSelectionStrategy {
-    public static final String EPSILON = "epsilon";
+    private static final String EPSILON = "epsilon";
     private Random random = new Random();
 
-    @Override
-    public Object clone(){
-        EpsilonGreedyActionSelectionStrategy clone = new EpsilonGreedyActionSelectionStrategy();
-        clone.copy(this);
-        return clone;
-    }
-
-    public void copy(EpsilonGreedyActionSelectionStrategy rhs){
-        random = rhs.random;
-        for(Map.Entry<String, String> entry : rhs.attributes.entrySet()){
-            attributes.put(entry.getKey(), entry.getValue());
-        }
+    @SuppressWarnings("Used-by-user")
+    public EpsilonGreedyActionSelectionStrategy() {
+        this.epsilon();
     }
 
-    @Override
-    public boolean equals(Object obj){
-        if(obj != null && obj instanceof EpsilonGreedyActionSelectionStrategy){
-            EpsilonGreedyActionSelectionStrategy rhs = (EpsilonGreedyActionSelectionStrategy)obj;
-            if(epsilon() != rhs.epsilon()) return false;
-           // if(!random.equals(rhs.random)) return false;
-            return true;
-        }
-        return false;
+    @SuppressWarnings("Used-by-user")
+    public EpsilonGreedyActionSelectionStrategy(final HashMap<String, String> attributes) {
+        super(attributes);
     }
 
-    private double epsilon(){
-        return Double.parseDouble(attributes.get(EPSILON));
+    @SuppressWarnings("Used-by-user")
+    public EpsilonGreedyActionSelectionStrategy(final Random random) {
+        this.random = random;
+        this.epsilon();
     }
 
-    public EpsilonGreedyActionSelectionStrategy(){
-        epsilon(0.1);
+    @Override
+    public Object clone() {
+        final EpsilonGreedyActionSelectionStrategy clone = new EpsilonGreedyActionSelectionStrategy();
+        clone.copy(this);
+        return clone;
     }
 
-    public EpsilonGreedyActionSelectionStrategy(HashMap<String, String> attributes){
-        super(attributes);
+    public void copy(final EpsilonGreedyActionSelectionStrategy rhs) {
+        this.random = rhs.random;
+        for (final Map.Entry<String, String> entry : rhs.attributes.entrySet()) {
+            this.attributes.put(entry.getKey(), entry.getValue());
+        }
     }
 
-    private void epsilon(double value){
-        attributes.put(EPSILON, "" + value);
+    @Override
+    public boolean equals(final Object obj) {
+        return obj instanceof EpsilonGreedyActionSelectionStrategy && this.epsilon() == ((EpsilonGreedyActionSelectionStrategy) obj).epsilon();
     }
 
-    public EpsilonGreedyActionSelectionStrategy(Random random){
-        this.random = random;
-        epsilon(0.1);
+    private double epsilon() {
+        return Double.parseDouble(this.attributes.get(EpsilonGreedyActionSelectionStrategy.EPSILON));
     }
 
     @Override
-    public IndexValue selectAction(int stateId, QModel model, Set<Integer> actionsAtState) {
-        if(random.nextDouble() < 1- epsilon()){
+    public IndexValue selectAction(final int stateId, final QModel model, final Set<Integer> actionsAtState) {
+        if (this.random.nextDouble() < 1 - this.epsilon()) {
             return model.actionWithMaxQAtState(stateId, actionsAtState);
-        }else{
-            int actionId;
-            if(actionsAtState != null && !actionsAtState.isEmpty()) {
-                List<Integer> actions = new ArrayList<>(actionsAtState);
-                actionId = actions.get(random.nextInt(actions.size()));
+        } else {
+            final int actionId;
+            if (actionsAtState != null && !actionsAtState.isEmpty()) {
+                final List<Integer> actions = new ArrayList<>(actionsAtState);
+                actionId = actions.get(this.random.nextInt(actions.size()));
             } else {
-                actionId = random.nextInt(model.getActionCount());
+                actionId = this.random.nextInt(model.getActionCount());
             }
 
-            double Q = model.getQ(stateId, actionId);
+            final double Q = model.getQ(stateId, actionId);
             return new IndexValue(actionId, Q);
         }
     }