@@ -3,7 +3,7 @@ */ package glueAgent; -import learners.EpisodicSarsaLearner; +import learners.SarsaLearner; import org.rlcommunity.rlglue.codec.AgentInterface; import org.rlcommunity.rlglue.codec.taskspec.TaskSpec; @@ -26,7 +26,6 @@ public class GlueAgent implements AgentInterface { // private Action action; TaskSpec TS = null; - int getStateSpaceSize() { int size = 1; @@ -72,25 +71,23 @@ public class GlueAgent implements AgentInterface { // FIXME: rrl.TaskDescription.environment = new GlueEnvironment(TSO); System.out.println("Size of qtable " + getStateSpaceSize() + "x" + getActionSpaceSize()); - learner = new EpisodicSarsaLearner(new DeterministicQTable(getStateSpaceSize(), getActionSpaceSize())); + //learner = new EpisodicSarsaLearner(new DeterministicQTable(getStateSpaceSize(), getActionSpaceSize())); //learner = new RandomLearner(new DeterministicQTable(getStateSpaceSize(), getActionSpaceSize())); - // learner = new SarsaLearner(new DeterministicQTable(getStateSpaceSize(), getActionSpaceSize())); + learner = new SarsaLearner(new DeterministicQTable(getStateSpaceSize(), getActionSpaceSize())); //learner = new SarsaLearner(new TabularModel()); // learner = new SarsaLearner(new TabularModel()); - learner.agent_init(); + learner.agentInit(); - // action = new Action(TSO.num_discrete_action_dims, 0); + // action = new Action(TSO.num_discrete_action_dims, 0); done } public Action agent_start(Observation o) { - System.out.println("START"); - //Choose and return the agent's first action State state = new GlueState(o); - rrl.Action action = learner.agent_start(state); + rrl.Action action = learner.agentStart(state); GlueAction ca = (GlueAction) action; return ca.a; @@ -103,28 +100,59 @@ public class GlueAgent implements AgentInterface { // if (r > 0) // System.out.println("Reward " + r); - rrl.Action action = learner.agent_step(r, state); + rrl.Action action = learner.agentStep(r, state); GlueAction ca = (GlueAction) action; return ca.a; } public void agent_end(double r) { - System.out.println("END"); - - learner.agent_end(r); + learner.agentEnd(r); } + + public String agent_message(String message) { + + if (message.equals("freeze learning")) { + learner.agentFreeze(); + return "message understood, policy frozen"; + } + if (message.equals("unfreeze learning")) { + learner.agentUnfreeze(); + return "message understood, policy unfrozen"; + } + if (message.equals("freeze exploring")) { + learner.agentFreezeExploring(); + return "message understood, exploring frozen"; + } + if (message.equals("unfreeze exploring")) { + learner.agentUnfreezeExploring(); + return "message understood, exploring unfrozen"; + } +// if (message.startsWith("save_policy")) { +// String[] parts = message.split(" "); +// saveValueFunction(parts[1]); +// System.out.println("Saved."); +// return "message understood, saving policy"; +// } +// if (message.startsWith("load_policy")) { +// String[] parts = message.split(" "); +// loadValueFunction(parts[1]); +// System.out.println("Loaded."); +// return "message understood, loading policy"; +// } + + return "SampleSarsaAgent(Java) does not understand your message."; + + } + public void agent_cleanup() { System.out.println("cleanup"); } - public void agent_freeze() { + public void agentFreeze() { System.out.println("freeze"); - learner.agent_freeze(); - } - - public String agent_message(String theMessage) { - return null; + learner.agentFreeze(); } + } src/glueAgent/GlueAgent.java @@ -30,42 +30,13 @@ import java.io.IOException; import java.util.Vector; import org.rlcommunity.rlglue.codec.RLGlue; -/** - * Experiment program that does some of the things that might be important when - * running an experiment. It runs an agent on the environment and periodically - * asks the agent to "freeze learning": to stop updating its policy for a number - * of episodes in order to get an estimate of the quality of what has been learned - * so far. - * - * The experiment estimates statistics such as the mean and standard deviation of - * the return gathered by the policy and writes those to a comma-separated value file - * called results.csv. - * - * This experiment also shows off some other features that can be achieved easily - * through the RL-Glue env/agent messaging system by freezing learning (described - * above), having the environment start in specific starting states, and saving - * and loading the agent's value function to/from a binary data file. - * @author Brian Tanner - */ public class GlueExperiment { - private void saveResultsToCSVFile(Vector<evaluationPoint> results, String fileName) { - try { - FileWriter FW = new FileWriter(new File(fileName)); - FW.write("#Results from SampleExperiment.java. First line is means, second line is standard deviations.\n"); - for (evaluationPoint point : results) { - FW.write("" + point.mean + ","); - } - FW.write("\n"); - for (evaluationPoint point : results) { - FW.write("" + point.standardDeviation + ","); - } - FW.write("\n"); - FW.close(); - } catch (IOException ex) { - System.out.println("Problem writing results out to file: " + fileName + " :: " + ex); - } - } + static final int numRuns = 20; + static final int numTrainingEpisodesWithSaving = 25; + static final int numTrainingEpisodesWithoutSaving = 1000; + static final int numTestingEpisodes = 25; + static String outputDirectory = "/home/valibook/eclipse/learny/"; class evaluationPoint { @@ -78,116 +49,93 @@ public class GlueExperiment { } } - /** - * Tell the agent to stop learning, then execute n episodes with his current - * policy. Estimate the mean and variance of the return over these episodes. - * @return - */ - evaluationPoint evaluateAgent() { - int i = 0; - double sum = 0; - double sum_of_squares = 0; - double this_return = 0; - double mean; - double variance; - int n = 10; - - RLGlue.RL_agent_message("freeze learning"); - for (i = 0; i < n; i++) { - /* We use a cutoff here in case the policy is bad - and will never end an episode */ - System.out.println("episode i = " + i); - RLGlue.RL_episode(5000); - this_return = RLGlue.RL_return(); - System.out.println("episode done, return " + this_return); - sum += this_return; - sum_of_squares += this_return * this_return; - } - RLGlue.RL_agent_message("unfreeze learning"); - - mean = sum / (double)n; - variance = (sum_of_squares - (double)n * mean * mean) / ((double)n - 1.0f); - return new evaluationPoint(mean, Math.sqrt(variance)); - } - - /* - This function will freeze the agent's policy and test it after every 25 episodes. - */ - void printScore(int afterEpisodes, evaluationPoint theScore) { - System.out.printf("%d\t\t%.2f\t\t%.2f\n", afterEpisodes, theScore.mean, theScore.standardDeviation); - } - - void offlineDemo() { - Vector<evaluationPoint> results = new Vector<evaluationPoint>(); - evaluationPoint initialScore = evaluateAgent(); - printScore(0, initialScore); - for (int i = 0; i < 20; i++) { - for (int j = 0; j < 25; j++) { - RLGlue.RL_episode(0); - } - evaluationPoint currentScore = evaluateAgent(); - printScore((i + 1) * 25, currentScore); - results.add(currentScore); - } - - System.out.println("The results of this experiment have been saved to a" + - " comma-separated value file called results.csv that you may open with Matlab, Octave, Excel, etc."); - - saveResultsToCSVFile(results, "results.csv"); - - } - public void runExperiment() { - RLGlue.RL_init(); - System.out.println("Starting offline demo\n----------------------------\nWill alternate learning for 25 episodes, then freeze policy and evaluate for 10 episodes.\n"); - System.out.println("After Episode\tMean Return\tStandard Deviation\n-------------------------------------------------------------------------"); - offlineDemo(); - -// System.out.println("\nNow we will save the agent's learned value function to a file...."); -// -// RLGlue.RL_agent_message("save_policy valuefunction.dat"); -// -// System.out.println("\nCalling RL_cleanup and RL_init to clear the agent's memory..."); -// -// RLGlue.RL_cleanup(); -// RLGlue.RL_init(); -// -// -// System.out.println("Evaluating the agent's default policy:\n\t\tMean Return\tStandardDeviation\n------------------------------------------------------"); -// evaluationPoint thisScore=evaluateAgent(); -// printScore(0, thisScore); -// -// System.out.println("\nLoading up the value function we saved earlier."); -// RLGlue.RL_agent_message("load_policy valuefunction.dat"); -// -// System.out.println("Evaluating the agent after loading the value function:\n\t\tMean Return\tStandardDeviation\n------------------------------------------------------"); -// thisScore=evaluateAgent(); -// printScore(0, thisScore); -// -// System.out.println("Telling the environment to use fixed start state of 2,3."); -// RLGlue.RL_env_message("set-start-state 2 3"); -// -// RLGlue.RL_start(); -// -// System.out.println("Telling the environment to print the current state to the screen."); -// RLGlue.RL_env_message("print-state"); -// -// System.out.println("Evaluating the agent a few times from a fixed start state of 2,3:\n\t\tMean Return\tStandardDeviation\n-------------------------------------------"); -// thisScore=evaluateAgent(); -// printScore(0, thisScore); -// -// System.out.println("Evaluating the agent again with the random start state:\n\t\tMean Return\tStandardDeviation\n------------------------------------------------------"); -// RLGlue.RL_env_message("set-random-start-state"); -// thisScore=evaluateAgent(); -// printScore(0, thisScore); -// -// System.out.println("\nProgram Complete."); -// RLGlue.RL_cleanup(); -// - + long[] trainingSteps = new long[numTrainingEpisodesWithSaving]; + double[] avgTrainingSteps = new double[numTrainingEpisodesWithSaving]; + double[] trainingRewards = new double[numTrainingEpisodesWithSaving]; + long[] testingSteps = new long[numTestingEpisodes]; + double[] avgTestingSteps = new double[numTestingEpisodes]; + double[] testingRewards = new double[numTestingEpisodes]; + + for (int i = 0; i < numRuns; i++) { + RLGlue.RL_init(); + + for (int j = 0; j < numTrainingEpisodesWithSaving; j++) { + trainingSteps[j] = 0; + trainingRewards[j] = 0; + } + for (int j = 0; j < numTestingEpisodes; j++) { + testingSteps[j] = 0; + testingRewards[j] = 0; + } + + // train && save statistics + for (int j = 0; j < numTrainingEpisodesWithSaving; j++) { + RLGlue.RL_episode(5000); + trainingSteps[j] += RLGlue.RL_num_steps(); + trainingRewards[j] += RLGlue.RL_return(); + } + + // just train + for (int j = 0; j < numTrainingEpisodesWithoutSaving; j++) + RLGlue.RL_episode(5000); + + + RLGlue.RL_agent_message("freeze_learning"); + // test = evaluate agent + for (int j = 0; j < numTestingEpisodes; j++) { + RLGlue.RL_episode(5000); + testingSteps[j] += RLGlue.RL_num_steps(); + testingRewards[j] += RLGlue.RL_return(); + } + + RLGlue.RL_cleanup(); + } + + for (int j = 0; j < numTrainingEpisodesWithSaving; j++) + avgTrainingSteps[j] = (double)trainingSteps[j] / (double)numRuns; + for (int j = 0; j < numTestingEpisodes; j++) + avgTestingSteps[j] = (double)testingSteps[j] / (double)numRuns; + + try { + FileWriter FW = new FileWriter(new File(outputDirectory + "/avgTrainingSteps.txt")); + for (int j = 0; j < numTrainingEpisodesWithSaving; j++) { + FW.write("" + avgTrainingSteps[j] + "\n"); + } + FW.close(); + System.out.println("Created file " + outputDirectory + "/avgTrainingSteps.txt"); + + FW = new FileWriter(new File(outputDirectory + "/avgTrainingRewards.txt")); + for (int j = 0; j < numTrainingEpisodesWithSaving; j++) { + FW.write("" + (double)trainingRewards[j] / (double)numRuns + "\n"); + } + FW.close(); + System.out.println("Created file " + outputDirectory + "/avgTrainingRewards.txt"); + + + FW = new FileWriter(new File(outputDirectory + "/avgTestingRewards.txt")); + for (int j = 0; j < numTestingEpisodes; j++) { + FW.write("" + (double)testingSteps[j] / (double)numRuns + "\n"); + } + FW.close(); + System.out.println("Created file " + outputDirectory + "/avgTestingRewards.txt"); + + FW = new FileWriter(new File(outputDirectory + "/avgTestingSteps.txt")); + for (int j = 0; j < numTestingEpisodes; j++) { + FW.write("" + avgTestingSteps[j] + "\n"); + } + FW.close(); + System.out.println("Created file " + outputDirectory + "/avgTestingSteps.txt"); + + } catch (IOException ex) { + System.out.println("Problem writing results out to file: " + ex); + } } public static void main(String[] args) { + if (args.length > 1 && args[1] != null) + outputDirectory = args[1]; + GlueExperiment theExperiment = new GlueExperiment(); theExperiment.runExperiment(); } src/glueAgent/GlueExperiment.java @@ -36,7 +36,7 @@ public class EpisodicQLearner implements Learner, Serializable { this.model = model; } - public void agent_init() { + public void agentInit() { // model = new Tree(); egreedy = new EGreedyPolicy(); episode = 0; @@ -44,12 +44,12 @@ public class EpisodicQLearner implements Learner, Serializable { EpisodeMemoryRewards = new ArrayList<Double>(); } - public void agent_cleanup() { + public void agentCleanup() { EpisodeMemoryQValues = null; EpisodeMemoryRewards = null; } - public void agent_end(double reward) { + public void agentEnd(double reward) { EpisodeMemoryRewards.add(reward); if (EpisodeMemoryRewards.size() != EpisodeMemoryQValues.size()) { @@ -80,16 +80,16 @@ public class EpisodicQLearner implements Learner, Serializable { EpisodeMemoryQValues.clear(); } - public void agent_freeze() { + public void agentFreeze() { alpha = 0f; epsilon = 0f; } - public String agent_message(String arg0) { + public String agentMessage(String arg0) { return "SarsaAgent does not implement agent_message currently"; } - public rrl.Action agent_start(State state) { + public rrl.Action agentStart(State state) { rrl.Action action = egreedy.chooseAction(model, epsilon, state); QValue qValue = model.findQValue(state, action); @@ -103,7 +103,7 @@ public class EpisodicQLearner implements Learner, Serializable { return action; } - public rrl.Action agent_step(double reward, State state) { + public rrl.Action agentStep(double reward, State state) { rrl.Action newAction = egreedy.chooseAction(model, epsilon, state); @@ -179,4 +179,31 @@ public class EpisodicQLearner implements Learner, Serializable { ex.printStackTrace(); } } + + /* (non-Javadoc) + * @see rrl.Learner#agentFreezeExploring() + */ + @Override + public void agentFreezeExploring() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreeze() + */ + @Override + public void agentUnfreeze() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreezeExploring() + */ + @Override + public void agentUnfreezeExploring() { + // TODO Auto-generated method stub + + } } src/learners/EpisodicQLearner.java @@ -8,6 +8,8 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.ArrayList; +import org.rlcommunity.rlglue.codec.RLGlue; + import rrl.DeterministicModel; import rrl.EGreedyPolicy; import rrl.Learner; @@ -16,7 +18,7 @@ import rrl.State; /** * @author slusny - * + * * SARSA learner with episodic memory. */ public class EpisodicSarsaLearner implements Learner, Serializable { @@ -36,7 +38,7 @@ public class EpisodicSarsaLearner implements Learner, Serializable { this.model = model; } - public void agent_init() { + public void agentInit() { // model = new Tree(); egreedy = new EGreedyPolicy(); episode = 0; @@ -44,23 +46,20 @@ public class EpisodicSarsaLearner implements Learner, Serializable { EpisodeMemoryRewards = new ArrayList<Double>(); } - public void agent_freeze() { + public void agentFreeze() { alpha = 0f; epsilon = 0f; } - public String agent_message(String arg0) { + public String agentMessage(String arg0) { return "SarsaAgent does not implement agent_message currently"; } - public rrl.Action agent_start(State state) { - if (EpisodeMemoryRewards.size() > 0 || EpisodeMemoryQValues.size() > 0) { - System.out.println("Internal error 3"); - System.out.println(EpisodeMemoryRewards.size()); - System.out.println(EpisodeMemoryQValues.size()); - throw new ArithmeticException(); - } - + public rrl.Action agentStart(State state) { + step_num = 0; + EpisodeMemoryRewards.clear(); + EpisodeMemoryQValues.clear(); + rrl.Action action = egreedy.chooseAction(model, epsilon, state); QValue qValue = model.findQValue(state, action); @@ -70,15 +69,11 @@ public class EpisodicSarsaLearner implements Learner, Serializable { } EpisodeMemoryQValues.add(qValue); - - step_num = 0; - -// System.out.println("AGENT START " + EpisodeMemoryRewards.size() + " " + EpisodeMemoryQValues.size()); return action; } - public rrl.Action agent_step(double reward, State state) { + public rrl.Action agentStep(double reward, State state) { rrl.Action newAction = egreedy.chooseAction(model, epsilon, state); @@ -93,33 +88,18 @@ public class EpisodicSarsaLearner implements Learner, Serializable { step_num += 1; - return newAction; - } - - public void writeInfo(String fileName) { - model.writeInfo(fileName); - } - - public void loadMe(ObjectInputStream in) { - try { - egreedy = (EGreedyPolicy) in.readObject(); - model = (DeterministicModel) in.readObject(); - alpha = in.readDouble(); - epsilon = in.readDouble(); - sarsa_gamma = in.readDouble(); - step_num = in.readInt(); - } catch(IOException ex) { - ex.printStackTrace(); - } catch(ClassNotFoundException ex){ - ex.printStackTrace(); + if (step_num % 100 == 0) { + System.out.println("STEP " + step_num); +// System.out.println("Telling the environment to print the current state to the screen."); +// RLGlue.RL_env_message("print-state"); } + + return newAction; } - public void agent_end(double reward) { + public void agentEnd(double reward) { EpisodeMemoryRewards.add(reward); - System.out.println("AGENT END " + EpisodeMemoryRewards.size() + " " + EpisodeMemoryQValues.size()); - if (EpisodeMemoryRewards.size() != EpisodeMemoryQValues.size()) { System.out.println("Internal error 2"); System.out.println(EpisodeMemoryRewards.size()); @@ -139,19 +119,33 @@ public class EpisodicSarsaLearner implements Learner, Serializable { // oldQ = oldaQ + alpha * delta oldQValue.qvalue += alpha * delta; model.updateQValue(oldQValue); -// newQValue = oldQValue; } epsilon *= .99; episode++; - - EpisodeMemoryRewards.clear(); - EpisodeMemoryQValues.clear(); - + } + + public void writeInfo(String fileName) { + model.writeInfo(fileName); } - public void agent_cleanup() { + public void loadMe(ObjectInputStream in) { + try { + egreedy = (EGreedyPolicy) in.readObject(); + model = (DeterministicModel) in.readObject(); + alpha = in.readDouble(); + epsilon = in.readDouble(); + sarsa_gamma = in.readDouble(); + step_num = in.readInt(); + } catch(IOException ex) { + ex.printStackTrace(); + } catch(ClassNotFoundException ex){ + ex.printStackTrace(); + } + } + + public void agentCleanup() { EpisodeMemoryQValues = null; EpisodeMemoryRewards = null; } @@ -196,4 +190,31 @@ public class EpisodicSarsaLearner implements Learner, Serializable { ex.printStackTrace(); } } + + /* (non-Javadoc) + * @see rrl.Learner#agentFreezeExploring() + */ + @Override + public void agentFreezeExploring() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreeze() + */ + @Override + public void agentUnfreeze() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreezeExploring() + */ + @Override + public void agentUnfreezeExploring() { + // TODO Auto-generated method stub + + } } src/learners/EpisodicSarsaLearner.java @@ -35,7 +35,7 @@ public class QLearner implements Learner, Serializable { this.model = model; } - public void agent_init() { + public void agentInit() { // model = new Tree(); egreedy = new EGreedyPolicy(); run++; @@ -44,7 +44,7 @@ public class QLearner implements Learner, Serializable { // At the end of a trial agent_cleanup will be called to allow for the deallocation of any resources created in agent_init. // Calls to agent_init and agent_cleanup should always be in a one to one ratio. - public void agent_cleanup() { + public void agentCleanup() { // TODO: delete qtree; } @@ -79,7 +79,7 @@ public class QLearner implements Learner, Serializable { // If the environment is episodic, the agent_end method will be called to allow for the experimenter // to complete the last step of his/her learning algorithm. - public void agent_end(double reward) { + public void agentEnd(double reward) { //Update last step of current episode double error = reward - sarsa_gamma * oldQValue.qvalue; oldQValue.qvalue += alpha * error; @@ -92,16 +92,16 @@ public class QLearner implements Learner, Serializable { episode++; } - public void agent_freeze() { + public void agentFreeze() { alpha = 0f; epsilon = 0f; } - public String agent_message(String arg0) { + public String agentMessage(String arg0) { return "SarsaAgent does not implement agent_message currently"; } - public rrl.Action agent_start(State state) { + public rrl.Action agentStart(State state) { rrl.Action action = egreedy.chooseAction(model, epsilon, state); oldQValue = model.findQValue(state, action); @@ -113,7 +113,7 @@ public class QLearner implements Learner, Serializable { return action; } - public rrl.Action agent_step(double reward, State state) { + public rrl.Action agentStep(double reward, State state) { rrl.Action newAction = egreedy.chooseAction(model, epsilon, state); QValue newQValue = model.findQValue(state, newAction); @@ -167,4 +167,31 @@ public class QLearner implements Learner, Serializable { ex.printStackTrace(); } } + + /* (non-Javadoc) + * @see rrl.Learner#agentFreezeExploring() + */ + @Override + public void agentFreezeExploring() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreeze() + */ + @Override + public void agentUnfreeze() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreezeExploring() + */ + @Override + public void agentUnfreezeExploring() { + // TODO Auto-generated method stub + + } } src/learners/QLearner.java @@ -36,21 +36,21 @@ public class RandomLearner implements Learner, Serializable { this.model = model; } - public void agent_init() { + public void agentInit() { // model = new Tree(); egreedy = new EGreedyPolicy(); } - public void agent_freeze() { + public void agentFreeze() { alpha = 0f; epsilon = 0f; } - public String agent_message(String arg0) { + public String agentMessage(String arg0) { return "SarsaAgent does not implement agent_message currently"; } - public rrl.Action agent_start(State state) { + public rrl.Action agentStart(State state) { rrl.Action action = egreedy.chooseAction(model, epsilon, state); QValue qValue = model.findQValue(state, action); @@ -65,7 +65,7 @@ public class RandomLearner implements Learner, Serializable { return action; } - public rrl.Action agent_step(double reward, State state) { + public rrl.Action agentStep(double reward, State state) { rrl.Action newAction = egreedy.chooseAction(model, epsilon, state); QValue qValue = model.findQValue(state, newAction); @@ -81,7 +81,7 @@ public class RandomLearner implements Learner, Serializable { return newAction; } - public void agent_end(double reward) { + public void agentEnd(double reward) { numRewards += 1; if (numRewards != numQValues) { @@ -92,6 +92,33 @@ public class RandomLearner implements Learner, Serializable { System.out.println("AGENT END " + numRewards + " qvalues " + numQValues); } - public void agent_cleanup() { + public void agentCleanup() { + } + + /* (non-Javadoc) + * @see rrl.Learner#agentFreezeExploring() + */ + @Override + public void agentFreezeExploring() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreeze() + */ + @Override + public void agentUnfreeze() { + // TODO Auto-generated method stub + + } + + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreezeExploring() + */ + @Override + public void agentUnfreezeExploring() { + // TODO Auto-generated method stub + } } src/learners/RandomLearner.java @@ -38,7 +38,7 @@ public class SarsaLearner implements Learner, Serializable { // required by the agent as well as initialize any values which need to be reset at the beginning of every trial/run. // ... value function // ... values across episodes - public void agent_init() { + public void agentInit() { // model = new Tree(); egreedy = new EGreedyPolicy(); run++; @@ -47,7 +47,7 @@ public class SarsaLearner implements Learner, Serializable { // At the end of a trial agent_cleanup will be called to allow for the deallocation of any resources created in agent_init. // Calls to agent_init and agent_cleanup should always be in a one to one ratio. - public void agent_cleanup() { + public void agentCleanup() { // TODO: delete qtree; } @@ -64,7 +64,6 @@ public class SarsaLearner implements Learner, Serializable { ex.printStackTrace(); } } - public void saveAgent(String filename) { FileOutputStream fos = null; @@ -82,7 +81,7 @@ public class SarsaLearner implements Learner, Serializable { // If the environment is episodic, the agent_end method will be called to allow for the experimenter // to complete the last step of his/her learning algorithm. - public void agent_end(double reward) { + public void agentEnd(double reward) { //Update last step of current episode double error = reward - sarsa_gamma * oldQValue.qvalue; oldQValue.qvalue += alpha * error; @@ -102,20 +101,20 @@ public class SarsaLearner implements Learner, Serializable { // and is behaving consistently. One easy mistake is to forget any randomness in a policy. // If an agent is implementing an epsilon greedy policy as above, the agent will have to remove the epsilon // randomness after agent_freeze is called. - public void agent_freeze() { + public void agentFreeze() { alpha = 0f; epsilon = 0f; } // Almost anything that isn't covered by these functions can be attempted using the agent_message function. - public String agent_message(String arg0) { + public String agentMessage(String arg0) { return "SarsaAgent does not implement agent_message currently"; } // chooses first action // state -> action - public rrl.Action agent_start(State state) { + public rrl.Action agentStart(State state) { //Choose and return the agent's first action rrl.Action action = egreedy.chooseAction(model, epsilon, state); @@ -133,7 +132,7 @@ public class SarsaLearner implements Learner, Serializable { // chooses next action // state -> new action - public rrl.Action agent_step(double reward, State state) { + public rrl.Action agentStep(double reward, State state) { // Update one step, return agent's action rrl.Action newAction = egreedy.chooseAction(model, epsilon, state); @@ -192,4 +191,28 @@ public class SarsaLearner implements Learner, Serializable { ex.printStackTrace(); } } + /* (non-Javadoc) + * @see rrl.Learner#agentFreezeExploring() + */ + @Override + public void agentFreezeExploring() { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreeze() + */ + @Override + public void agentUnfreeze() { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) + * @see rrl.Learner#agentUnfreezeExploring() + */ + @Override + public void agentUnfreezeExploring() { + // TODO Auto-generated method stub + + } } src/learners/SarsaLearner.java @@ -9,11 +9,14 @@ package rrl; * */ public interface Learner { - public void agent_init(); - public void agent_cleanup(); - public void agent_end(double reward); - public void agent_freeze(); - public String agent_message(String arg0); - public rrl.Action agent_start(State state); - public rrl.Action agent_step(double reward, State state); + public void agentInit(); + public void agentCleanup(); + public void agentEnd(double reward); + public void agentFreeze(); + public void agentUnfreeze(); + public void agentFreezeExploring(); + public void agentUnfreezeExploring(); + public String agentMessage(String arg0); + public rrl.Action agentStart(State state); + public rrl.Action agentStep(double reward, State state); } src/rrl/Learner.java eb3df449e53c6de8e872c2b56a51c1efdcc01791 Stanislav Slusny slusnys@gmail.com http://github.com/slusnys/learny/commit/eac432d46f5623898537535b532bbc0badf1226d eac432d46f5623898537535b532bbc0badf1226d 2009-02-18T01:39:00-08:00 2009-02-18T01:39:00-08:00 Finished adaptation to RL-glue 3.0 28278af16724089002083aed911c4936bdf80989 Stanislav Slusny slusnys@gmail.com