reinforce/sarsaview.cpp
reinforce/sarsaview.h
@@ -86,6 +86,85 @@ void Agent::initQ()
}
}
+void Agent::initE()
+{
+ m_e.clear();
+ int i,j;
+
+ m_e.insert(MyQ(Status(0,0), Action(1,0)), 0);
+ m_e.insert(MyQ(Status(0,0), Action(1,1)), 0);
+ m_e.insert(MyQ(Status(0,0), Action(0,1)), 0);
+
+ m_e.insert(MyQ(Status(19,0), Action(0,1)), 0);
+ m_e.insert(MyQ(Status(19,0), Action(-1,1)), 0);
+ m_e.insert(MyQ(Status(19,0), Action(-1,0)), 0);
+
+ m_e.insert(MyQ(Status(0,9), Action(1,0)), 0);
+ m_e.insert(MyQ(Status(0,9), Action(1,-1)), 0);
+ m_e.insert(MyQ(Status(0,9), Action(0,-1)), 0);
+
+ m_e.insert(MyQ(Status(19,9), Action(-1,0)), 0);
+ m_e.insert(MyQ(Status(19,9), Action(-1,-1)), 0);
+ m_e.insert(MyQ(Status(19,9), Action(0,-1)), 0);
+
+ for(i=1;i<19;i++){
+ m_e.insert(MyQ(Status(i,0), Action(1,0)), 0);
+ m_e.insert(MyQ(Status(i,0), Action(-1,0)), 0);
+ m_e.insert(MyQ(Status(i,0), Action(1,1)), 0);
+ m_e.insert(MyQ(Status(i,0), Action(-1,1)), 0);
+ m_e.insert(MyQ(Status(i,0), Action(0,1)), 0);
+
+ m_e.insert(MyQ(Status(i,9), Action(1,0)), 0);
+ m_e.insert(MyQ(Status(i,9), Action(-1,0)), 0);
+ m_e.insert(MyQ(Status(i,9), Action(1,-1)), 0);
+ m_e.insert(MyQ(Status(i,9), Action(-1,-1)), 0);
+ m_e.insert(MyQ(Status(i,9), Action(0,-1)), 0);
+ }
+ for(j=1;j<9;j++){
+ m_e.insert(MyQ(Status(0, i), Action(0,1)), 0);
+ m_e.insert(MyQ(Status(0, i), Action(0,-1)), 0);
+ m_e.insert(MyQ(Status(0, i), Action(1,1)), 0);
+ m_e.insert(MyQ(Status(0, i), Action(1,-1)), 0);
+ m_e.insert(MyQ(Status(0, i), Action(1,0)), 0);
+
+ m_e.insert(MyQ(Status(19,i), Action(0,1)), 0);
+ m_e.insert(MyQ(Status(19,i), Action(0,-1)), 0);
+ m_e.insert(MyQ(Status(19,i), Action(-1,1)), 0);
+ m_e.insert(MyQ(Status(19,i), Action(-1,-1)), 0);
+ m_e.insert(MyQ(Status(19,i), Action(-1,0)), 0);
+ }
+ for(i=0;i<20;i++){
+ for(j=0;j<10;j++){
+ m_e.insert(MyQ(Status(i,j), Action(0,0)), 0);
+ }
+ }
+ for(i=1;i<19;i++){
+ for(j=1;j<9;j++){
+ m_e.insert(MyQ(Status(i,j), Action(1,0)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(-1,0)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(0,1)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(0,-1)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(1,1)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(1,-1)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(-1,1)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(-1,-1)), 0);
+ }
+ }
+ for(i=2;i<18;i++){
+ for(j=2;j<8;j++){
+ m_e.insert(MyQ(Status(i,j), Action(2,0)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(-2,0)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(0,2)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(0,-2)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(2,2)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(-2,2)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(2,-2)), 0);
+ m_e.insert(MyQ(Status(i,j), Action(-2,-2)), 0);
+ }
+ }
+}
+
+
void Agent::changeAlpha(int i)
{
setAlpha(i/10);
@@ -262,7 +341,30 @@ int Agent::doEpisode()
Reward old_Q_sa = m_Q.find(MyQ(s, a)).value();
Reward old_Q_sa_dash = m_Q.find(MyQ(s_dash, a_dash)).value();
- m_Q[MyQ(s, a)] = old_Q_sa + m_alpha*(r + m_gamma*old_Q_sa_dash - old_Q_sa);
+ Reward delta = r + m_gamma*old_Q_sa_dash - old_Q_sa;
+ m_e[MyQ(s, a)] = m_e[MyQ(s, a)] + 1;
+
+
+ //Normal Sarsa
+// Reward newval = old_Q_sa + m_alpha*(delta);
+// m_Q[MyQ(s, a)] = newval;
+// Reward debugval = m_Q[MyQ(s, a)];
+
+ //ă
+ //‚·‚ׂĂÌs,a‚ɑ΂µ:
+ //Q(s,a) *© Q(s,a)*{ƒ¿ƒÂe(s,a)
+ //e(s,a)*© ƒÁƒÉe(s,a)
+ // ˆê——
+ QMap< MyQ, Reward >::iterator p;
+ for(p=m_Q.begin(); p!=m_Q.end(); p++)
+ {
+ MyQ q(p.key());
+ m_Q[p.key()] = m_Q[p.key()] + m_alpha * delta * m_e[p.key()];
+ m_e[p.key()] = m_gamma * 0.9 * m_e[p.key()];
+ }
+
+
+ emit updateQ();
s = s_dash;
a = a_dash;
ret++;
reinforce/agent.cpp
@@ -17,6 +17,7 @@ public:
void setEps(double eps){ m_eps = eps; }
int doEpisode();
void initQ();
+ void initE();
bool isFin(Status st);
void stop();
Action selectAction(Status s);
@@ -24,13 +25,17 @@ public slots:
void changeAlpha(int i);
void changeGamma(int i);
void changeEps(int i);
+signals:
+ void updateQ();
private:
Environment *m_env;
double m_alpha;
double m_gamma;
double m_eps;
bool m_stop;
+public:
QMap< MyQ, Reward > m_Q;
+ QMap< MyQ, double > m_e;
};
#endif // AGENT_H
reinforce/agent.h
@@ -9,9 +9,12 @@ TEMPLATE = app
SOURCES += main.cpp \
mainwindow.cpp \
environment.cpp \
- agent.cpp
+ agent.cpp \
+ sarsaview.cpp \
+ ../Acrobot/utils.cpp
HEADERS += mainwindow.h \
environment.h \
agent.h \
- defs.h
+ defs.h \
+ sarsaview.h
FORMS += mainwindow.ui
reinforce/demo1.pro
@@ -26,6 +26,10 @@ MainWindow::MainWindow(QWidget *parent)
agent, SLOT(changeGamma(int)));
connect(ui->epsilonCombo, SIGNAL(currentIndexChanged(int)),
agent, SLOT(changeEps(int)));
+
+ ui->sView->setQ(&(agent->m_Q));
+ connect(agent, SIGNAL(updateQ()),
+ ui->sView, SLOT(update()));
}
MainWindow::~MainWindow()
@@ -68,6 +72,7 @@ void MainWindow::on_startButton_clicked()
}
env->initStatus();
agent->initQ();
+ agent->initE();
for(int i=0;i<1000;i++){
if(stop) break;
ui->textEdit->append("try");
reinforce/mainwindow.cpp
@@ -1414,6 +1414,9 @@
</item>
</widget>
</item>
+ <item>
+ <widget class="SarsaView" name="sView" native="true"/>
+ </item>
</layout>
</widget>
</item>
@@ -1462,6 +1465,14 @@
</action>
</widget>
<layoutdefault spacing="6" margin="11"/>
+ <customwidgets>
+ <customwidget>
+ <class>SarsaView</class>
+ <extends>QWidget</extends>
+ <header>sarsaview.h</header>
+ <container>1</container>
+ </customwidget>
+ </customwidgets>
<resources/>
<connections>
<connection>
reinforce/mainwindow.ui
b52adbaf108272cd5b1467351ac78a97542c8a49
NOSE Takafumi
ahya365@gmail.com
http://github.com/plus7/plus7pub/commit/907e79d2ab92715084e154e80523b25730416d95
907e79d2ab92715084e154e80523b25730416d95
2009-05-14T22:22:40-07:00
2009-05-14T22:22:40-07:00
Sarsa(lambda)
0df7dc2a48d7b255ba6c5991987535f575213af6
NOSE Takafumi
ahya365@gmail.com