Я создал пользовательскую реализацию GRU в TensorFlow для экспериментов. При обучении он ничем не отличается от родного TensorFlow GRUcell. Я пытаюсь понять, почему, и просто не могу. Любой вклад будет оценен.
Нативная реализация:
encoder = tf.nn.rnn_cell.GRUCell(feature_no,activation = tf.nn.tanh)
encoding = tf.nn.dynamic_rnn(encoder,inputs,dtype=tf.float32)[1]
Моя реализация:
class GRU:
def __init__(self, hid_dim, vec_dim,
name = None):
self.hid_dim = hid_dim #Dimension of the hidden layer
self.vec_dim = vec_dim #Dimension of the input layer
## Declare the variables
## Update gate
self.W_ux = tf.get_variable(name = name + ".GRU.W_ux",
shape = [vec_dim, hid_dim],
initializer = tf.random_normal_initializer())
self.W_uh = tf.get_variable(name = name + ".GRU.W_uh",
shape = [hid_dim, hid_dim],
initializer = tf.random_normal_initializer())
self.b_u = tf.get_variable(name = name + ".GRU.g_u",
shape = [hid_dim],
initializer = tf.random_normal_initializer())
## Forget gate
self.W_rx = tf.get_variable(name = name + ".GRUt.W_rx",
shape = [vec_dim, hid_dim],
initializer = tf.random_normal_initializer())
self.W_rh = tf.get_variable(name = name + ".GRU.W_rh",
shape = [hid_dim, hid_dim],
initializer = tf.random_normal_initializer())
self.b_r = tf.get_variable(name = name + ".GRU.b_r",
shape = [hid_dim],
initializer = tf.random_normal_initializer())
## Update function
self.W_hx = tf.get_variable(name = name + ".GRU.W_hx",
shape = [vec_dim, hid_dim],
initializer = tf.random_normal_initializer())
self.W_hh = tf.get_variable(name = name + ".GRU.W_hh",
shape = [hid_dim, hid_dim],
initializer = tf.random_normal_initializer())
self.b_h = tf.get_variable(name = name + ".GRU.b_h",
shape = [hid_dim],
initializer = tf.random_normal_initializer())
def update_state(self, x, h):
u = tf.sigmoid(tf.matmul(x, self.W_ux) + tf.matmul(h, self.W_uh) +
self.b_u)
## 'Forget' gate
r = tf.sigmoid(tf.matmul(x, self.W_rx) + tf.matmul(h, self.W_rh) + self.b_r)
## Hidden state
hp = tf.tanh(tf.matmul(x, self.W_hx) + r*tf.matmul(h, self.W_hh)
+ self.b_h)
return (1 - u) * hp + u * h
def get_states(self, x):
init = tf.reshape(tf.tile(tf.zeros_like(x[:,0,0]),[self.hid_dim]),
shape = [-1,self.hid_dim])
x_t = tf.transpose(x,perm=[1,0,2])
self.h_set = tf.transpose(tf.scan(lambda h, x: self.update_state(x, h),
x_t, init), perm = [1,0,2])
self.h = self.h_set[:,-1]
def __call__(self, x):
self.get_states(x)
return self.h
encoder = GRU(feature_no,vec_dim,name='encoder')
encoding = encoder(sent)
