def build(self):
model = self.net.model
mu_model = self.net.mu_model
log_std_model = self.net.log_std_model
q_model = self.net.q_model
target_model = self.net.target_model
target_mu_model = self.net.target_mu_model
target_log_std_model = self.net.target_log_std_model
target_q_model = self.net.target_q_model
self.states = tf.placeholder(tf.float32, shape=(None, self.in_dim), name='states')
self.actions = tf.placeholder(tf.float32, shape=[None, self.action_dim], name='actions')
self.rewards = tf.placeholder(tf.float32, shape=[None], name='rewards')
self.next_states = tf.placeholder(tf.float32, shape=[None, self.in_dim], name='next_states')
self.ys = tf.placeholder(tf.float32, shape=[None])
# There are other implementations about how can we take aciton.
# Taking next action version or using only mu version or searching action which maximize Q.
target_mu = target_mu_model(self.states)
target_log_std = target_log_std_model(self.states)
target_action = target_mu + K.random_normal(K.shape(target_mu), dtype=tf.float32) * K.exp(target_log_std)
self.target_q = K.sum(target_q_model(Concatenate()([target_model(self.states), target_action])), axis=-1)
self.q = K.sum(q_model(Concatenate()([model(self.states), self.actions])), axis=-1)
self.q_loss = K.mean(K.square(self.ys-self.q))
self.mu = mu_model(self.states)
self.log_std = log_std_model(self.states)
self.eta = (self.actions - self.mu) / K.exp(self.log_std)
inferred_action = self.mu + K.stop_gradient(self.eta) * K.exp(self.log_std)
self.pi_loss = - K.mean(q_model(Concatenate()([model(self.states), inferred_action])))
self.q_updater = self.q_optimizer.minimize(self.q_loss, var_list=self.net.var_q)
self.pi_updater = self.pi_opimizer.minimize(self.pi_loss, var_list=self.net.var_pi)
self.soft_updater = [K.update(t_p, t_p*(1-self.tau)+p*self.tau) for p, t_p in zip(self.net.var_all, self.net.var_target_all)]
self.sync = [K.update(t_p, p) for p, t_p in zip(self.net.var_all, self.net.var_target_all)]
self.sess.run(tf.global_variables_initializer())
self.built = True
评论列表
文章目录
