TensorFlow - An Introduction

# This may take a few seconds if you are running it for the first time
%matplotlib notebook
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import tutorial  # see tutorial.py
from PIL import Image
from os.path import abspath

tf.reset_default_graph()

hello_world = tf.constant("Hello World!")
with tf.Session() as sess:
    print(sess.run(hello_world))

x = tf.constant(8)
y = tf.constant(34)
result = tf.add(x, y)

print(x)
print(y)
print(result)

with tf.Session() as sess:
    print(sess.run(result))

with tf.Session() as sess:
    print(sess.run(x))  # returns a scalar
    print(sess.run([x, y, result]))  # returns a list of scalars
    print(sess.run({"x": x, "my_result": result}))  # returns a dictionary of scalars

x = tf.placeholder(name="x", dtype=tf.float32)  # almost anything in TensorFlow can be named.
y = tf.placeholder(name="y", dtype=tf.float32)
result = x + y  # the common operations for tensors are available as operators

print(x)
print(y)
print(result)

with tf.Session() as sess:
    # Add up 8 and 34
    result_value = sess.run(result, feed_dict={x: 8, y: 34})
    print("8 + 34 = %d" % result_value)
        
    # We didn't specify the shapes of our placeholders above, so we can substitute in anything
    # that can be added up!
    feed_dict = {
        x: np.arange(12).reshape(3, 4),
        y: np.ones(12).reshape(3, 4) * 10
    }
    result_value = sess.run(result, feed_dict)
    print("Sum of two 3x4 matrices:\n" + str(result_value))

a = tf.placeholder(name="a", shape=(2, 2), dtype=tf.float32)  # our input are 1x2 matrices

# create the variable and specify that it should be initialized with zeros
accumulator = tf.get_variable(name="acc",
                              shape=(2, 2),
                              dtype=tf.float32,
                              initializer=tf.zeros_initializer())
    
# compute the new value of the accumulator and assign it to the variable
new_accumulator = accumulator + a
update_accumulator = tf.assign(accumulator, new_accumulator)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())  # variables need to be initialized before using them!
    
    # let's look at the initial value of the accumulator:
    print('Initial value of `acc`: \n' + str(sess.run(accumulator)))
    
    # we can query the value of `new_accumulator` without updating the variable `acc`
    summand = np.random.randn(2, 2)
    feed_dict = {a: summand}
    results = sess.run([new_accumulator, accumulator], feed_dict)
    print('Evaluating `new_accumulator` does not update `acc`: \n' + str(results[1]))
    
    # but evaluating the assignment does update the accumulator
    for i in range(10):
        summand = np.random.randn(2, 2)
        sess.run(update_accumulator, feed_dict={a: summand})
    print('Evaluating `update_accumulator` updates `acc`: \n' + str(sess.run(accumulator)))

q = tf.placeholder(name="q", shape=(), dtype=tf.float32)
sqrt_q = tf.get_variable(name="sqrt_q",
                         shape=(),
                         dtype=tf.float32,
                         initializer=tf.ones_initializer())

# Name scopes can be used to group operations together, making it easier to avoid name-clashes
# and making the graph structure easier to understand
with tf.name_scope("error"):
    delta = (q - (sqrt_q * sqrt_q))
    loss = tf.multiply(delta, delta)

# build the optimizer and get a tensor that performs the optimization when evaluated
optimizer = tf.train.GradientDescentOptimizer(name="SGD", learning_rate=0.001)
optimization_step = optimizer.minimize(loss)

def compute_sqrt(x):
    with tf.Session() as sess:
        # Initialize all variables, in our case this sets sqrt_q to 1
        sess.run(tf.global_variables_initializer())
    
        feed_dict = {
            q: x
        }
    
        step = 0
        loss_value = 1
        while loss_value > 1e-10:
            # we repeatedly evaluate the optimization step, get the loss, and optimized value
            _, loss_value, sqrt = sess.run((optimization_step, loss, sqrt_q), feed_dict=feed_dict)
            
            if step % 5 == 0:
                print('Current loss: %f with value %f' % (loss_value, sqrt))
            step += 1

        print('Final loss: %f after %d steps' % (loss_value, step))
        sqrt = sess.run(sqrt_q)
        print('Sqrt of %f is %f' % (x, sqrt))
        return sqrt

compute_sqrt(49)

tf.reset_default_graph()

def make_convolution(input, name, kernel_size, num_filters, stride, padding='SAME'):
    
    # We are using a variable scope here to ensure that the `kernel` variable name
    # is taken relative to the name of this convolution.
    with tf.name_scope(name), tf.variable_scope(name):
        # get the number of input channels, assuming NHWC format
        in_channels = input.get_shape()[-1]
        # create the kernel of the convolution
        kernel = tf.get_variable(name="kernel",
                                 shape=[kernel_size, kernel_size, in_channels, num_filters])
        output = tf.nn.conv2d(input,
                              kernel,
                              strides=[1, stride, stride, 1],
                              padding=padding)
        # create bias, choose correct initializers etc.
        return tf.nn.relu(output)

def model(input):
    """
        This function creates a CNN with two convolution/pooling pairs, followed by two dense layers.
        These are then used to predict one of ten digit classes.
        
        Arguments:
            input: a tensor with shape `[batch_size, 28, 28, 1]` representing MNIST data.
        
        Returns:
            A dictionary with the keys `probabilities`, `class`, and `logits`, containing the 
            corresponding tensors.
    """
    
    # we assume the input has shape [batch_size, 28, 28, 1]
    net = input
    
    # create two convolution/pooling layers
    for i in range(1, 3):
        net = tf.layers.conv2d(inputs=net,
                               filters=32 * i,
                               kernel_size=[5, 5],
                               padding="same",
                               activation=tf.nn.relu,
                               name="conv2d_%d" % i)
        net = tf.layers.max_pooling2d(inputs=net,
                                      pool_size=[2, 2],
                                      strides=2,
                                      name="maxpool_%d" % i)
    # flatten the input to [batch_size, 7 * 7 * 64]
    net = tf.reshape(net, [-1, 7 * 7 * 64])
    net = tf.layers.dense(net, units=128, name="dense_1")
    logits = tf.layers.dense(net, units=10, name="dense_2")
    # logits has shape [batch_size, 10]
    
    # The predictions of our model; we return the logits to formulate a numerically
    # stable loss in our optimization routine
    tensors = {
        "probabilities": tf.nn.softmax(logits),
        "class": tf.argmax(logits, axis=-1),
        "logits": logits
    }
    return tensors

def classify_image(img):
    input_image = tf.placeholder(...)
    outputs = model(input_image)
    with tf.Session() as sess:
        # initialize model etc. ...
        return sess.run(outputs["class"], feed_dict: {input_image: img})

def training_model(input, label):
    tensors = model(input)
    logits = tensors["logits"]
    
    # one-hot encode the labels, compute cross-entropy loss, average
    # over the number of elements in the mini-batch
    with tf.name_scope("loss"):
        label = tf.one_hot(label, 10)
        loss = tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=logits)
        loss = tf.reduce_mean(loss)
    tensors["loss"] = loss
        
    # create the optimizer and register the global step counter with it,
    # it is increased every time the optimization step is executed
    optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
    global_step = tf.train.get_or_create_global_step()
    tensors["opt_step"] = optimizer.minimize(loss, global_step=global_step)
    return tensors

def load_data_set(is_training=True):
    """
        Loads either the training or evaluation dataset by performing the following steps:
        1. Read the file containing the file paths of the images.
        2. Construct the correct labels from the file paths.
        3. Shuffle paths jointly with the labels.
        4. Turn them into a dataset of tuples.
        
        Arguments:
            is_training: whether to load the training dataset or the evaluation dataset
        
        Returns:
            The requested dataset as a dataset containing tuples `(relative_path, label)`.
    """
    substring = "training" if is_training else "testing"
    file = "mnist_%s.dataset" % substring
    
    # this converts the file names into (file_name, class) tuples
    offset = len('mnist_data/%s/c' % substring)
    data_points = []
    with open(file, 'r') as f:
        for line in f:
            cls = int(line[offset])
            path = abspath(line.strip())
            data_points.append((path, cls))
            
    # now shuffle all data points around and create a dataset from it
    from random import shuffle
    shuffle(data_points)
    
    # Neat fact: zip can be used to build its own inverse:
    data_points = zip(*data_points)
    
    # note how this is a tensor of strings! Tensors are not inherently numeric!
    images = tf.constant(data_points[0])
    labels = tf.constant(data_points[1])
    
    # `from_tensor_slices` takes a tuple of tensors slices them along the first
    # dimension. Thus we get a dataset of tuples.
    return tf.contrib.data.Dataset.from_tensor_slices((images, labels))

def read_image(img_path, label):
    """
        Constructs operations to turn the path to an image to the actual image data.
        
        Arguments:
            img_path: The path to the image.
            label: The label for the image.
        
        Returns:
            A tuple `(img, label)`, where `img` is the loaded image.
    """
    img_content = tf.read_file(img_path)
    img = tf.image.decode_png(img_content, channels=1)
    img = tf.image.convert_image_dtype(img, tf.float32)
    return img, label

def make_training_dataset(batch_size=128):
    """
        Creates a training dataset with the given batch size.
    """
    dataset = load_data_set(is_training=True)
    dataset = dataset.repeat(-1)
    dataset = dataset.shuffle(10000)
    dataset = dataset.map(read_image,
                          num_threads=4,
                          output_buffer_size=100 * batch_size)
    dataset = dataset.batch(batch_size)
    return dataset

def make_eval_dataset(batch_size=128):
    """
        Creates the evaluation dataset.
    """
    dataset = load_data_set(is_training=False)
    dataset = dataset.map(read_image,
                          num_threads=4,
                          output_buffer_size=100 * batch_size)
    dataset = dataset.batch(batch_size)
    return dataset

def perform_training(steps, batch_size):
    dataset = make_training_dataset(batch_size)
    # The one-shot iterator is just another node in the graph, and so is the
    # `get_next` operation. The tensors produced by `get_next` depend on the
    # structure of the dataset that we build: Our dataset consists of tuples,
    # so we get a tuple of tensors.
    next_image, next_label = dataset.make_one_shot_iterator().get_next()   
    model_outputs = training_model(next_image, next_label)
    
    loss = model_outputs["loss"]
    opt_step = model_outputs["opt_step"]
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        
        for i in xrange(steps):
            # Note how we no longer need to use the `feed_dict`. This avoids
            # additional overhead due to Python -> native Tensorflow
            # serialization
            _, loss_value = sess.run((opt_step, loss))
            
            if i % 5 == 0:
                print("Loss: %f" % loss_value)
        print("Final Loss: %f" % loss_value)

tutorial.perform_training_with_visualization(steps=500, batch_size=128,
                                             dataset=make_training_dataset,
                                             model=training_model)

    saver = tf.train.Saver(variables_to_save)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        saver.restore(sess, "checkpoint-file-to-restore-from")
        # your code here
        saver.save(sess, "checkpoint-file-to-save-to")

def perform_training(steps, batch_size, checkpoint_name):
    dataset = make_training_dataset(batch_size)
    next_image, next_label = dataset.make_one_shot_iterator().get_next()   
    model_outputs = training_model(next_image, next_label)
    
    loss = model_outputs["loss"]
    opt_step = model_outputs["opt_step"]
    
    # We exclude some variables created by the optimizer and data loading process
    # when saving the model, since we assume that we do not want to continue training
    # from the checkpoint.
    variables_to_save = tutorial.get_variables_for_saver(exclude=[".*Adam", ".*beta", ".*Iterator"])
    saver = tf.train.Saver(variables_to_save)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        
        for i in xrange(steps):
            _, loss_value = sess.run((opt_step, loss))
        
        # In the end, we  will save the model
        saver.save(sess, checkpoint_name)
        print("Final Loss: %f" % loss_value)


tf.reset_default_graph()
perform_training(steps=100, batch_size=128,
                 checkpoint_name="model")

def image_classifier(checkpoint_name, *images):
    # note that we are deciding to feed the images one-by-one for simplicity
    input = tf.placeholder(name="input", shape=(1, 28, 28, 1), dtype="float32")
    tensors = model(input)
    
    saver = tf.train.Saver()
    with tf.Session() as sess:
        # Initialize all variables, since there may be variables in our graph that aren't
        # loaded form the checkpoint
        sess.run(tf.global_variables_initializer())
        saver.restore(sess, checkpoint_name)
        
        classes = []
        for i in images:
            cls = sess.run(tensors["class"], feed_dict={input: np.expand_dims(i, 0)})
            classes.append(cls[0])
        return classes

def load_image_as_array(filepath):
    im = Image.open(abspath(filepath)).convert('L')
    (width, height) = im.size
    greyscale_map = list(im.getdata())
    greyscale_map = np.array(greyscale_map)
    greyscale_map = greyscale_map.reshape((height, width, 1))
    return greyscale_map


tf.reset_default_graph()
images = [
    "mnist_data/testing/c1/994.png",
    "mnist_data/testing/c8/606.png",
    "mnist_data/testing/c4/42.png"
]
classification = image_classifier("model", *map(load_image_as_array, images))
print("Classification: " + str(classification))

def evaluation_model(input, label):
    tensors = model(input)
    # Let's track some metrics like accuracy, false negatives, and false positives.
    # Each of them returns a tuple `(value, update_op)`. The value is, well, the value
    # of the metric, the update_op is the tensor that needs to be evaluated to update
    # the metric.
    accuracy, update_acc = tf.metrics.accuracy(label, tensors["class"])
    false_negatives, update_fn = tf.metrics.false_negatives(label, tensors["class"])
    false_positives, update_fp = tf.metrics.false_positives(label, tensors["class"])
    tensors["accuracy"] = accuracy
    tensors["false_negatives"] = false_negatives
    tensors["false_positives"] = false_positives
    # We can group the three metric updates into a single operation and run that instead.
    tensors["update_metrics"] = tf.group(update_acc, update_fn, update_fp)
    return tensors

def perform_evaluation(checkpoint_name):
    # this is the same as in the training case, except that we are using the
    # evaluation dataset and model
    dataset = make_eval_dataset()
    next_image, next_label = dataset.make_one_shot_iterator().get_next()   
    model_outputs = evaluation_model(next_image, next_label)
    
    saver = tf.train.Saver()
    with tf.Session() as sess:
        # Initialize variables. This is really necessary here since the metrics
        # need to be initialized to sensible values. They are local variables,
        # meaning that they are not saved by default, which is why we need to run
        # `local_variables_initializer`.
        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())
        # restore the weights
        saver.restore(sess, checkpoint_name)
        
        update_metrics = model_outputs["update_metrics"]
        # feed the inputs until we run out of images
        while True:
            try:
                _ = sess.run(update_metrics)
            except tf.errors.OutOfRangeError: 
                # this happens when the iterator runs out of samples
                break
        
        # Get the final values of the metrics. Note that this call does not access the
        # `get_next` node of the dataset iterator, since these tensors can be evaluated
        # on their own and therefore don't cause the exception seen above to be triggered
        # again.
        accuracy = model_outputs["accuracy"]
        false_negatives = model_outputs["false_negatives"]
        false_positives = model_outputs["false_positives"]
        acc, fp, fn = sess.run((accuracy, false_negatives, false_positives))
        print("Accuracy: %f" % acc)
        print("False positives: %d" % fp)
        print("False negatives: %d" % fn)


tf.reset_default_graph()
perform_evaluation(checkpoint_name="model")

def perform_training(steps, batch_size, checkpoint_name, logdir):
    dataset = make_training_dataset(batch_size)
    next_image, next_label = dataset.make_one_shot_iterator().get_next()   
    model_outputs = training_model(next_image, next_label)
    
    loss = model_outputs["loss"]
    opt_step = model_outputs["opt_step"]
    
    # generate summary tensors, but don't store them -- there's a better way
    frequent_summary = tf.summary.scalar("loss", loss)
    tf.summary.histogram("logits", model_outputs["logits"])
    tf.summary.histogram("probabilities", model_outputs["probabilities"])
    # only log one of the `batch_size` many images
    tf.summary.image("input_image", next_image, max_outputs=1)
    
    # merge all summary ops into a single operation
    infrequent_summary = tf.summary.merge_all()
    
    variables_to_save = tutorial.get_variables_for_saver(exclude=[".*Adam", ".*beta", ".*Iterator"])
    saver = tf.train.Saver(variables_to_save)
    
    # the summary writer will write the summaries to the specified log directory
    summary_writer = tf.summary.FileWriter(logdir=logdir, graph=tf.get_default_graph())
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        
        for i in xrange(steps):
            # note that we explicitly ask for the evaluation of the summary ops
            if i % 10 == 0:
                _, summary = sess.run((opt_step, infrequent_summary))
            else:
                _, summary = sess.run((opt_step, frequent_summary))
            # ...and we also need to explicitly add them to the summary writer
            summary_writer.add_summary(summary, global_step=i)
        
        saver.save(sess, checkpoint_name)
        summary_writer.close()


tf.reset_default_graph()
# It's a good idea to keep the checkpoint and the event files in the same directory
perform_training(steps=5000,
                 batch_size=128,
                 checkpoint_name="training/model",
                 logdir="training")

tf.reset_default_graph()
perform_evaluation(checkpoint_name="training/model")