switch back to adam

convoys/regression.py

@@ -19,12 +19,11 @@ def _get_placeholders(n, k):
 
 def _optimize(sess, target, feed_dict, variables):
     learning_rate_input = tf.placeholder(tf.float32, [])
-    optimizer = tf.train.GradientDescentOptimizer(learning_rate_input).minimize(-target)
+    optimizer = tf.train.AdamOptimizer(learning_rate_input).minimize(-target)
 
-    for variable in variables:
-        sess.run(tf.assign(variable, tf.zeros(tf.shape(variable))))
+    sess.run(tf.global_variables_initializer())
 
-    step = 0
+    step, best_step = 0, 0
     learning_rate = 1.0
     best_state = sess.run(variables)
     best_cost = sess.run(target, feed_dict=feed_dict)
@@ -39,15 +38,16 @@ def _optimize(sess, target, feed_dict, variables):
         if cost > best_cost:
             best_cost = cost
             best_state = sess.run(variables)
-        else:
-            learning_rate *= 10**-0.5
+        elif step - best_step > 20:
+            best_step = step
+            learning_rate *= 10 ** -0.5
             for variable, value in zip(variables, best_state):
                 sess.run(tf.assign(variable, value))
-        if learning_rate < 1e-9:
+        if learning_rate < 1e-6:
             sys.stdout.write('\n')
             break
         step += 1
-        sys.stdout.write('step %6d (lr %9.9f): %14.3f' % (step, learning_rate, cost))
+        sys.stdout.write('step %6d (lr %6.6f): %14.3f%30s' % (step, learning_rate, cost, ''))
         sys.stdout.write('\n' if step % 100 == 0 else '\r')
         sys.stdout.flush()