Feed Forward Neural Networks

%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

Note: The code is just for demonstration purpose. Some python function are just for producing plots for slides (ipython reveal).

from IPython.display import Image
Image(filename='./pics/Deeplearning 2.png') 

Eine Schicht von Neuronen:

Input: $\vec x^T = (x_1, x_2, \dots x_n)$

Output des j-ten Neurons: $$ h_j = \sigma(\sum_{i=1}^n w_{ij} x_i + b_j) $$

Mit der Gewichtsmatrix $W$:

$$ \vec h = \sigma(\vec x \cdot W + \vec b) $$

• Elementweise Anwendung der Funktion $\sigma$ auf die Vektorelemente.

# Generate training data 
# Polar coordinates: r, phi
def get_x(nb, c):
    assert (c==0 or c==1)
    r = c + np.random.rand(nb)
    phi = np.random.rand(nb) * 2 * np.pi
    return np.concatenate(((r * np.sin(phi)).reshape(-1,1), (r * np.cos(phi)).reshape(-1,1)), axis=1)

def get_data(nb_1, nb_2):
    x_0 = get_x(nb_1, 0)
    x_1 = get_x(nb_2, 1)
    X = np.concatenate((x_0, x_1), axis=0)
    t = np.zeros(len(X))
    t[len(x_0):] = 1
    p = np.random.permutation(range(len(X)))
    # permute the data
    X = X[p]
    t = t[p]
    return X, t

X_train, t_train = get_data(30, 35)
X_test, t_test = get_data(20, 20)

%matplotlib inline
import matplotlib.pyplot as plt
def plot_train_data(x_0, x_1):
    plt.figure(figsize=(7,7))
    plt.scatter(x_0[:,0], x_0[:,1], label='Class 0', color='b') 
    plt.scatter(x_1[:,0], x_1[:,1], label='Class 1', color='r') 
    plt.xlabel("$x_1$")
    plt.ylabel("$x_2$")
    ra = np.arange(-1,1.01,0.01, dtype=np.float64)
    ci = np.ndarray(len(ra))
    ci[:-1] = (np.sqrt(1. - ra[:-1]**2))
    ci[-1] = 0. # hack 
    plt.plot(ra, ci,'g-')
    plt.plot(ra, -ci,'g-')
    plt.xlim(-2.,2.)
    plt.ylim(-2.,2.)

Trainingsdaten - nicht "linear separabel"

plot_train_data(X_train[t_train==0], X_train[t_train==1])

Händisch konstruierte Merkmalstransformation

$$ \phi_1(\vec x) = x_1^2 $$$$ \phi_2(\vec x) = x_2^2 $$

def plot_train_transformed(x_0, x_1):
    plt.figure(figsize=(7,7))
    plt.scatter(x_0[:,0], x_0[:,1], label='Class 0', color='b') 
    plt.scatter(x_1[:,0], x_1[:,1], label='Class 1', color='r') 
    plt.xlabel("$\phi_1$")
    plt.ylabel("$\phi_2$")
    ra = np.arange(-0.,1.,0.01)
    plt.plot(ra, 1. - ra,'g-')
    plt.xlim(-0.1,4)
    plt.ylim(-0.1,4)

Im konstruierten Merkmalsraum sind die Daten linear-separabel.

phi_train = X_train**2
plot_train_transformed(phi_train[t_train==0], phi_train[t_train==1])

Lernen der Transformation

Mit Neuronalen Netzen können Feature-Transformationen gelernt werden.

• Der Aktivitätsvektor der "Hidden-Layers" kann als transformierter Input-Vektor interpretiert werden.

import tensorflow as tf
sess = tf.InteractiveSession()

nb_input = 2
nb_out = 1
x = tf.placeholder("float", shape=[None, nb_input])
y_ = tf.placeholder("float", shape=[None])

• Zufalls-Initialisierung der Matrix $W^{(1)}$
• Zweck: "Symmetry Breaking"

def weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape) # for ReLu neurons positive initial bias
  #initial = tf.constant(0., shape=shape) 
  return tf.Variable(initial)

nb_h = 5
W_h = weight_variable([nb_input, nb_h])
b_h = bias_variable([nb_h])
W_o = weight_variable([nb_h, nb_out])
b_o = bias_variable([nb_out])

# Variables must be initialized by running an `init` Op after having
# launched the graph.  We first have to add the `init` Op to the graph.
init_op = tf.initialize_all_variables()
sess.run(init_op)

Feed Forward Neural Network mit einer verdeckten Schicht

Aktivität der (ersten) Hidded-Layer:

$$ \vec h^{(1)} = \sigma_1 \left(\vec x \cdot W^{(1)} + \vec b^{(1)} \right) $$

Aktivität des Output-Neurons $o$:

$$ o = h^{(2)}= \sigma_2 \left( \vec h^{(1)} \cdot W^{(2)} + b^{(2)} \right) $$

def logistic_function(x):
    return 1./(1. + tf.exp(-x))

# (first) hidden layer
a = tf.matmul(x, W_h) + b_h

# activity function "rectified linear units"
h = tf.nn.relu(a)

# output neuron:
y = logistic_function(tf.matmul(h, W_o) + b_o)

Kostenfunktion und l2-Regulierung

cross_entropy = -tf.reduce_sum(y_*tf.log(y[:,0]) + (1.-y_)*tf.log(1.-y[:,0]))

l2_reg = tf.reduce_sum(tf.square(W_h)) + tf.reduce_sum(tf.square(W_o))

lambda_ = 0.002
cost = cross_entropy + lambda_ * l2_reg

# Decaying the learning rate:
# http://www.tensorflow.org/api_docs/python/train.html#exponential_decay
global_step = tf.Variable(0, trainable=False)
starter_alpha = 0.001
alpha = tf.train.exponential_decay(starter_alpha, global_step,
                                           100, 0.96, staircase=True)
optimizer = tf.train.GradientDescentOptimizer(alpha)

# Passing global_step to minimize() will increment it at each step.
train_step = optimizer.minimize(cost, global_step=global_step)

sess.run(tf.initialize_all_variables())

epochs = 10000
cost_ = np.ndarray(epochs)
for j in range(epochs):
    #full online learning
    cost_[j] = cross_entropy.eval(feed_dict={x: X_train, y_: t_train})
    train_step.run(feed_dict={x: X_train, y_: t_train})
  

# Note: there is also a visualization tool called TensorBoard, not used here.
plt.plot(range(epochs), cost_, '-b')
plt.xlabel('Iterations')
plt.ylabel('Cost')
cost_

array([ 44.88797379,  44.88481522,  44.88169098, ...,   6.09101915,
         6.09098625,   6.09095526])

import matplotlib
matplotlib.rcParams['xtick.direction'] = 'out'
matplotlib.rcParams['ytick.direction'] = 'out'
def plot_contour(x_0, x_1, data_type):
    plt.figure(figsize=(8,8))
    CS = plt.contour(A, B, Z, (0.2,0.5,0.8), colors=('b', 'green', 'r'))
    plt.clabel(CS, inline=1, fontsize=10)
    plt.title('Prediction surface plot with %s'%data_type)
    plt.scatter(x_0[:,0], x_0[:,1], label='Class 0', color='b') 
    plt.scatter(x_1[:,0], x_1[:,1], label='Class 1', color='r') 
    plt.xlabel("$x_1$")
    plt.ylabel("$x_2$")
    plt.xlim(-2.,2.)
    plt.ylim(-2.,2.)

Entscheidungsfläche (Decision Boundary)

delta = 0.025
a = np.arange(-2.5, 2.5, delta)
b = np.arange(-2.5, 2.6, delta)
A, B = np.meshgrid(a, b)
Z = np.ndarray(A.shape)

x_ = np.dstack((A,B)).reshape(-1, 2)
Z = y.eval(feed_dict={x: x_}).reshape(A.shape)
# hidden neuron activities
H = h.eval(feed_dict={x: x_})

plot_contour(X_train[t_train==0], X_train[t_train==1], 'train data')

plot_contour(X_test[t_test==0], X_test[t_test==1], 'test data')