avoid_obstacles/controllers/ModelLearner/Makefile avoid_obstacles/controllers/ModelLearner/ModelLearner.java avoid_obstacles/controllers/ModelLearner/modelActions.csv avoid_obstacles/controllers/ModelLearner/modelActions.txt avoid_obstacles/worlds/model_learner.wbt avoid_obstacles/worlds/model_learner1.wbt avoid_obstacles/worlds/textures/floor.png avoid_obstacles/worlds/textures/lightwood.png avoid_obstacles/worlds/textures/stone-2.png @@ -4,3 +4,4 @@ *.svn* .project .classpath +*~ .gitignore @@ -37,10 +37,12 @@ public class AvoidSupervisor extends Supervisor { static int TIMESTEP = 32; // number of episodes - static int EPISODES = 2; + static int EPISODES = 5; // duration of episode - static int MAXSTEPS = 100000; + static int MAXSTEPS = 10; + + static int nosteps = 0; // robot IDs Node robot[] = new Node[ROBOTS]; @@ -61,9 +63,6 @@ public class AvoidSupervisor extends Supervisor { initialPosition = robotTranslation[i].getSFVec3f(); } - /* run an empty step in order to update the world with the imported node */ - step(0); - emitter = getEmitter("emitter"); receiver = getReceiver("receiver"); receiver.enable(TIMESTEP); @@ -78,37 +77,48 @@ public class AvoidSupervisor extends Supervisor { return false; } - public void run() { + public void STEP() { + // System.out.println("SUPER: Calling step() function" + nosteps++); + step(TIMESTEP); + // System.out.println("SUPER: Out of step() function" + nosteps); + } + + public void run() { + System.out.println("SUPERVISOR-RUN"); + STEP(); + Message msg = new MessageAgentInit(""); try { msg.send(emitter); - step(TIMESTEP); - step(TIMESTEP); + STEP(); for (int episode = 0; episode < EPISODES; episode++) { + System.out.println(episode + "Episode \n"); + for (int i = 0; i < ROBOTS; i++) robotTranslation[i].setSFVec3f(initialPosition); - System.out.println(episode + "SENDING START\n"); + + System.out.println("Sending AGENT-START"); msg = new MessageAgentStart(); msg.send(emitter); - step(TIMESTEP); - step(TIMESTEP); + STEP(); for (int step = 0; step < MAXSTEPS; step++) { - // System.out.println(episode + "/STEP " + step); + System.out.println(episode + "/STEP " + step); msg = new MessageAgentStep(0); msg.send(emitter); - step(TIMESTEP); - step(TIMESTEP); + STEP(); } - + + System.out.println("Sending AGENT-END"); msg = new MessageAgentEnd(MAXSTEPS); msg.send(emitter); - step(TIMESTEP); - step(TIMESTEP); + STEP(); + // STEP(); + } } catch (Exception e) { System.out.println("Error!!!"); avoid_obstacles/controllers/AvoidSupervisor/AvoidSupervisor.java @@ -16,14 +16,16 @@ public class Learner extends Controller { System.out.println("Starting webot agent."); AvoiderLearner robot = new AvoiderLearner(); - for (;;) { + for (int i = 0; i >= 0; i++) { try { robot.senseAct(); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } + // System.out.println("CLIENT: Calling " + i); robot.step(AvoiderLearner.TIMESTEP); + // System.out.println("CLIENT: Out of " + i); } } avoid_obstacles/controllers/Learner/Learner.java @@ -1,6 +1,7 @@ package epuck; -import learners.SarsaLearner; +import learners.EpisodicSarsaLearner; +import rrl.Learner; import rrl.State; import tableModel.DeterministicQTable; import webotsRL.Message; @@ -43,24 +44,28 @@ public class AvoiderLearner extends DifferentialWheels { Receiver receiver; Emitter emitter; - static SarsaLearner learner; + static Learner learner; static int previousLeftSpeed, previousRightSpeed; public AvoiderLearner() { + System.out.println("INITALiZATION!!!"); rrl.TaskDescription.environment = new EpuckEnvironment(); String text = new String(); - camera = getCamera("camera"); - camera.enable(5*TIMESTEP); + accelerometer = getAccelerometer("accelerometer"); + accelerometer.enable(TIMESTEP); + +// camera = getCamera("camera"); +// camera.enable(5*TIMESTEP); receiver = getReceiver("receiver"); - touchSensor = getTouchSensor("touch_sensor"); - touchSensor.enable(TIMESTEP); +// touchSensor = getTouchSensor("touch_sensor"); +// touchSensor.enable(TIMESTEP); receiver = getReceiver("receiver"); receiver.enable(TIMESTEP); - emitter = getEmitter("emitter)"); + emitter = getEmitter("emitter"); for (int i = 0; i < NB_DIST_SENS; i++) { text = String.format("ps%d", i); @@ -83,17 +88,17 @@ public class AvoiderLearner extends DifferentialWheels { } - - public static void agent_init() { + public void agent_init() { + System.out.println("Agent INIT Received"); //System.out.println("Size of qtable " + getStateSpaceSize() + "x" + getActionSpaceSize()); // learner = new EpisodicSarsaLearner(new DeterministicQTable(getStateSpaceSize(), getActionSpaceSize())); - // learner = new EpisodicSarsaLearner(new DeterministicQTable(343, 14)); + learner = new EpisodicSarsaLearner(new DeterministicQTable(400, 100)); - learner = new SarsaLearner(new DeterministicQTable(343, 14)); + // learner = new SarsaLearner(new DeterministicQTable(400, 100)); //learner = new SarsaLearner(new TabularModel()); // learner = new SarsaLearner(new TabularModel()); - learner.agent_init(); + learner.agentInit(); } public void executeAction(EpuckAction action) { @@ -108,8 +113,8 @@ public class AvoiderLearner extends DifferentialWheels { double[] av = accelerometer.getValues(); double acceleration = getAcceleration(av[0], av[1], av[2]); -// System.out.println("Acceleration " + acceleration); - + System.out.println("Acceleration " + acceleration); +// return acceleration > 10; } @@ -127,17 +132,20 @@ public class AvoiderLearner extends DifferentialWheels { } public void agent_start() { + System.out.println("AGENT-START received"); //Choose and return the agent's first action State state = new EpuckState(this); previousLeftSpeed = previousRightSpeed = 0; - rrl.Action action = learner.agent_start(state); + rrl.Action action = learner.agentStart(state); EpuckAction ca = (EpuckAction) action; executeAction(ca); } public void agent_step(double r) { + System.out.println("AGENT-STEP received"); + if (collisionOccured()) { System.out.println("COLLISION"); r -= 100; @@ -148,23 +156,27 @@ public class AvoiderLearner extends DifferentialWheels { // sense-act cycle State state = new EpuckState(this); - rrl.Action action = learner.agent_step(r, state); + rrl.Action action = learner.agentStep(r, state); EpuckAction ca = (EpuckAction) action; executeAction(ca); } public void agent_end(double r) { + System.out.println("AGENT-END received"); + System.out.flush(); + step(0); + r += getReward(); - learner.agent_end(r); + learner.agentEnd(r); } public void agent_cleanup() { - learner.agent_cleanup(); + learner.agentCleanup(); } public void agent_freeze() { - learner.agent_freeze(); + learner.agentFreeze(); } public String agent_message(String theMessage) { @@ -174,6 +186,7 @@ public class AvoiderLearner extends DifferentialWheels { public void senseAct() throws Exception { if (receiver.getQueueLength() > 0) { Message msg = new MessageParser(receiver.getData()).getBody(); + receiver.nextPacket(); switch (msg.header) { case Messages.MESSAGE_AGENT_CLEANUP: @@ -206,7 +219,8 @@ public class AvoiderLearner extends DifferentialWheels { break; } } - - step(TIMESTEP); + else + System.out.println("NO DATA RECEIVED!"); } + } avoid_obstacles/controllers/Learner/epuck/AvoiderLearner.java @@ -26,7 +26,7 @@ public class EpuckEnvironment extends tableModel.TableEnvironment implements Ser public static EpuckAction[] actions; public EpuckEnvironment() { - int RESOLUTION = 50; + int RESOLUTION = 100; int START = 0; int END = 1000; @@ -45,6 +45,8 @@ public class EpuckEnvironment extends tableModel.TableEnvironment implements Ser rSpeed += RESOLUTION; } + + System.out.println("Number of actions " + k); } public Action chooseRandomAction(State state) { avoid_obstacles/controllers/Learner/epuck/EpuckEnvironment.java @@ -1,10 +1,4 @@ -#VRML_SIM V5.0 utf8 -#000000 -#!mainWindow: 0 0 0.5 0.750501 -#!sceneTreeWindow: 0.472857 0.0911824 0.435714 0.37976 0.1 -#!logWindow: 0.523571 0.352705 0.472143 0.608216 -#!robotWindow: "e-puck" 0.551429 0.109218 0.29 0.259519 1 - +#VRML_SIM V6.0 utf8 WorldInfo { info [ "Author: fabien.rohrer@epfl.ch" @@ -175,17 +169,6 @@ DEF WALL Solid { } locked TRUE } -Supervisor { - children [ - Receiver { - name "receiver" - } - Emitter { - name "emitter" - } - ] - controller "Supervisor" -} epuck_obstacle { translation 0.0361483 0.025 0.0822249 } @@ -218,3 +201,14 @@ DEF E_PUCK epuck_emitter_receiver { rotation -1.93909e-05 1 0.000200911 -1.23578 controller "Learner" } +Supervisor { + children [ + Receiver { + name "receiver" + } + Emitter { + name "emitter" + } + ] + controller "AvoidSupervisor" +} avoid_obstacles/worlds/avoid_obstacles.wbt 88385ef7739d0eabc5c860f0e7911e36133ef071 Stanislav Slusny slusnys@gmail.com http://github.com/slusnys/webotsprojects/commit/9954eb4508e3b35aefca7df0dad91253f8d83c96 9954eb4508e3b35aefca7df0dad91253f8d83c96 2009-02-19T05:41:19-08:00 2009-02-19T05:41:19-08:00 Last chenges 148d3f8ed586466b986ced484978c05c91a0df69 Stanislav Slusny slusnys@gmail.com