Skip to content

Fixed typo, where instead of 'weight', 'grad' was being logged in fun… #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ankur-rc wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fixed typo, where instead of 'weight', 'grad' was being logged in fun… #11

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions vanishing_gradient.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

base_path = "C:\\Users\\Andy\\PycharmProjects\\Tensorboard\\"


class Model(object):
def __init__(self, input_size, label_size, activation, num_layers=6,
hidden_size=10):
Expand All @@ -16,20 +17,23 @@ def __init__(self, input_size, label_size, activation, num_layers=6,

def _model_def(self):
# create placeholder variables
self.input_images = tf.placeholder(tf.float32, shape=[None, self._input_size])
self.labels = tf.placeholder(tf.float32, shape=[None, self._label_size])
self.input_images = tf.placeholder(
tf.float32, shape=[None, self._input_size])
self.labels = tf.placeholder(
tf.float32, shape=[None, self._label_size])
# create self._num_layers dense layers as the model
input = self.input_images
for i in range(self._num_layers - 1):
input = tf.layers.dense(input, self._hidden_size, activation=self._activation,
name='layer{}'.format(i+1))
# don't supply an activation for the final layer - the loss definition will
# supply softmax activation. This defaults to a linear activation i.e. f(x) = x
logits = tf.layers.dense(input, 10, name='layer{}'.format(self._num_layers))
logits = tf.layers.dense(
input, 10, name='layer{}'.format(self._num_layers))
# use softmax cross entropy with logits - no need to apply softmax activation to
# logits
self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits,
labels=self.labels))
labels=self.labels))
# add the loss to the summary
tf.summary.scalar('loss', self.loss)
self._log_gradients(self._num_layers)
Expand All @@ -53,7 +57,8 @@ def _log_gradients(self, num_layers):
mean = tf.reduce_mean(tf.abs(grad))
tf.summary.scalar('mean_{}'.format(i + 1), mean)
tf.summary.histogram('histogram_{}'.format(i + 1), grad)
tf.summary.histogram('hist_weights_{}'.format(i + 1), grad)
tf.summary.histogram('hist_weights_{}'.format(i + 1), weight)


def run_training(model, mnist, sub_folder, iterations=2500, batch_size=30):
with tf.Session() as sess:
Expand All @@ -71,6 +76,7 @@ def run_training(model, mnist, sub_folder, iterations=2500, batch_size=30):
print("Iteration {} of {}, loss: {:.3f}, train accuracy: "
"{:.2f}%".format(i, iterations, l, acc * 100))


if __name__ == "__main__":
scenarios = ["sigmoid", "relu", "leaky_relu"]
act_funcs = [tf.sigmoid, tf.nn.relu, tf.nn.leaky_relu]
Expand All @@ -81,4 +87,4 @@ def run_training(model, mnist, sub_folder, iterations=2500, batch_size=30):
tf.reset_default_graph()
print("Running scenario: {}".format(scenarios[i]))
model = Model(784, 10, act_funcs[i], 6, 10)
run_training(model, mnist, scenarios[i])
run_training(model, mnist, scenarios[i])