diff --git a/vanishing_gradient.py b/vanishing_gradient.py index 91b6ffd..5573246 100644 --- a/vanishing_gradient.py +++ b/vanishing_gradient.py @@ -3,6 +3,7 @@ base_path = "C:\\Users\\Andy\\PycharmProjects\\Tensorboard\\" + class Model(object): def __init__(self, input_size, label_size, activation, num_layers=6, hidden_size=10): @@ -16,8 +17,10 @@ def __init__(self, input_size, label_size, activation, num_layers=6, def _model_def(self): # create placeholder variables - self.input_images = tf.placeholder(tf.float32, shape=[None, self._input_size]) - self.labels = tf.placeholder(tf.float32, shape=[None, self._label_size]) + self.input_images = tf.placeholder( + tf.float32, shape=[None, self._input_size]) + self.labels = tf.placeholder( + tf.float32, shape=[None, self._label_size]) # create self._num_layers dense layers as the model input = self.input_images for i in range(self._num_layers - 1): @@ -25,11 +28,12 @@ def _model_def(self): name='layer{}'.format(i+1)) # don't supply an activation for the final layer - the loss definition will # supply softmax activation. This defaults to a linear activation i.e. f(x) = x - logits = tf.layers.dense(input, 10, name='layer{}'.format(self._num_layers)) + logits = tf.layers.dense( + input, 10, name='layer{}'.format(self._num_layers)) # use softmax cross entropy with logits - no need to apply softmax activation to # logits self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, - labels=self.labels)) + labels=self.labels)) # add the loss to the summary tf.summary.scalar('loss', self.loss) self._log_gradients(self._num_layers) @@ -53,7 +57,8 @@ def _log_gradients(self, num_layers): mean = tf.reduce_mean(tf.abs(grad)) tf.summary.scalar('mean_{}'.format(i + 1), mean) tf.summary.histogram('histogram_{}'.format(i + 1), grad) - tf.summary.histogram('hist_weights_{}'.format(i + 1), grad) + tf.summary.histogram('hist_weights_{}'.format(i + 1), weight) + def run_training(model, mnist, sub_folder, iterations=2500, batch_size=30): with tf.Session() as sess: @@ -71,6 +76,7 @@ def run_training(model, mnist, sub_folder, iterations=2500, batch_size=30): print("Iteration {} of {}, loss: {:.3f}, train accuracy: " "{:.2f}%".format(i, iterations, l, acc * 100)) + if __name__ == "__main__": scenarios = ["sigmoid", "relu", "leaky_relu"] act_funcs = [tf.sigmoid, tf.nn.relu, tf.nn.leaky_relu] @@ -81,4 +87,4 @@ def run_training(model, mnist, sub_folder, iterations=2500, batch_size=30): tf.reset_default_graph() print("Running scenario: {}".format(scenarios[i])) model = Model(784, 10, act_funcs[i], 6, 10) - run_training(model, mnist, scenarios[i]) \ No newline at end of file + run_training(model, mnist, scenarios[i])