1
私はcartpole problemを実装していたためtensorflowを使用して私の最初の強化深い学習モデルを実装しています。openai cartpoleにtensorflowモデルを訓練
iが閾値を上回るスコアを有するランダムに生成されたデータセットでトレーニング6つの層を使用して深いニューラルネットワークに頼ってきました。問題は、モデルが収束しておらず、最終スコアが平均で約10ポイント残っていることです。特定の記事を読んだ後に提案されているよう
私はオーバーフィッティング運を発生しませんが、まだ可能性のを減らすために正則やドロップアウトを適用します。私も学習率を下げようとしました。
精度もちょうど損失が、私はそれもこれらの後で記憶考える反復ごとに減少しているものの1つのバッチを訓練した後、0.60の周りに残っています。 ですが、この種のモデルは簡単な深い学習タスクで動作します。ここ
が私のコードです：
import numpy as np
import tensorflow as tf
import gym
import os
import random
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
model_path = "C:/Users/sanka/codes/cart pole problem/tf_save3"
env = gym.make("CartPole-v0")
env.reset()
def train_set(): #training set generation function
try:
tx = np.load("final_trainx.npy")
ty = np.load("final_trainy.npy")
return tx,ty
except:
tx = []
ty = []
for _ in range(10000):
env.reset()
score = 0
moves = []
obs = []
p = []
for _ in range(500):
action = np.random.randint(0, 2)
observation, reward, done, info = env.step(action)
if (len(p)==0):
p = observation
else:
moves += [action]
obs += [observation]
p = observation
score += reward
if done:
break
if (score > 50):
tx+=obs
for i in range(len(moves)):
ac = moves[i]
if (ac == 1):
ty.append([0, 1])
else:
ty.append([1, 0])
tx=np.array(tx)
ty=np.array(ty)
np.save("final_trainx.npy",tx)
np.save("final_trainy.npy",ty)
return tx, ty
weights = {
1: tf.Variable(tf.truncated_normal([4, 128]), dtype=tf.float32),
2: tf.Variable(tf.truncated_normal([128, 256]), dtype=tf.float32),
3: tf.Variable(tf.truncated_normal([256, 512]), dtype=tf.float32),
4: tf.Variable(tf.truncated_normal([512, 256]), dtype=tf.float32),
5: tf.Variable(tf.truncated_normal([256, 128]), dtype=tf.float32),
6: tf.Variable(tf.truncated_normal([128, 2]), dtype=tf.float32)
}
biases = {
1: tf.Variable(tf.truncated_normal([128]), dtype=tf.float32),
2: tf.Variable(tf.truncated_normal([256]), dtype=tf.float32),
3: tf.Variable(tf.truncated_normal([512]), dtype=tf.float32),
4: tf.Variable(tf.truncated_normal([256]), dtype=tf.float32),
5: tf.Variable(tf.truncated_normal([128]), dtype=tf.float32),
6: tf.Variable(tf.truncated_normal([2]), dtype=tf.float32)
}
def neural_network(x):
x = tf.nn.relu(tf.add(tf.matmul(x, weights[1]), biases[1]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[2]), biases[2]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[3]), biases[3]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[4]), biases[4]))
x = tf.nn.dropout(x, 0.8)
x = tf.nn.relu(tf.add(tf.matmul(x, weights[5]), biases[5]))
x = tf.nn.dropout(x, 0.8)
x = tf.add(tf.matmul(x, weights[6]), biases[6])
return x
def test_nn(x):
x = tf.nn.relu(tf.add(tf.matmul(x, weights[1]), biases[1]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[2]), biases[2]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[3]), biases[3]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[4]), biases[4]))
x = tf.nn.relu(tf.add(tf.matmul(x, weights[5]), biases[5]))
x = tf.nn.softmax(tf.add(tf.matmul(x, weights[6]), biases[6]))
return x
def train_nn():
prediction = neural_network(x)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
lo=tf.nn.l2_loss(weights[1])+tf.nn.l2_loss(weights[2])+tf.nn.l2_loss(weights[3])+tf.nn.l2_loss(weights[4])+tf.nn.l2_loss(weights[5])+tf.nn.l2_loss(weights[6])
loss=tf.reduce_mean(loss+0.01*lo)
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss)
test_pred = test_nn(x)
correct = tf.equal(tf.argmax(test_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, dtype=tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
epoches = 5
batch_size = 100
for j in range(epoches):
ep_loss=0
for i in range(0,len(train_x),batch_size):
epoch_x=train_x[i:min(i+batch_size,len(train_x))]
epoch_y = train_y[i:min(i + batch_size, len(train_y))]
_,c=sess.run([optimizer,loss],feed_dict={x:epoch_x,y:epoch_y})
ep_loss+=c
#print("Accuracy is {0}".format(sess.run(accuracy, feed_dict={x: epoch_x, y: epoch_y})))
print("epoch {0} completed out of {1} with loss {2}".format(j,epoches,ep_loss))
print("Accuracy is {0}".format(sess.run(accuracy,feed_dict={x:train_x,y:train_y})))
scores = []
choices = []
for each_game in range(10):
print("game ", each_game)
score = 0
game_memory = []
prev_obs = []
env.reset()
for _ in range(500):
env.render()
if (len(prev_obs) == 0):
action = random.randrange(0, 2)
else:
x1 = np.array([prev_obs]).reshape(-1,4)
a = tf.argmax(test_pred, 1)
action = sess.run(a, feed_dict={x: x1})
action=action[0]
choices.append(action)
new_observation, reward, done, info = env.step(action)
prev_obs = new_observation
game_memory.append([new_observation, action])
score += reward
if done:
break
scores.append(score)
print('Average Score:', sum(scores)/len(scores))
print('choice 1:{} choice 0:{}'.format(choices.count(1)/len(choices), choices.count(0)/len(choices)))
train_x,train_y=train_set()
print(train_x.shape)
print(train_y.shape)
x=tf.placeholder(tf.float32,[None,4])
y=tf.placeholder(tf.int32,[None,2])
train_nn()