PL03-Topic02, TensorFlow_version1

import tensorflow as tf

print(tf.__version__)

# XOR Implementation with Hidden Layers That Have Sigmoid Activation Functions
#XOR  implementation in Tensorflow with hidden layers being sigmoid to # introduce Non-Linearity 
import tensorflow as tf 

# Create placeholders for training input and output labels 
x_ = tf.placeholder(tf.float32, shape=[4,2], name="x-input") 
y_ = tf.placeholder(tf.float32, shape=[4,1], name="y-input") 

#Define the weights to the hidden and output layer respectively.
w1 = tf.Variable(tf.random_uniform([2,2], -1, 1), name="Weights1")
w2 = tf.Variable(tf.random_uniform([2,1], -1, 1), name="Weights2") 

# Define the bias to the hidden and output layers respectively 
b1 = tf.Variable(tf.zeros([2]), name="Bias1") 
b2 = tf.Variable(tf.zeros([1]), name="Bias2")

# Define the final output through forward pass
z2 = tf.sigmoid(tf.matmul(x_, w1) + b1)
pred = tf.sigmoid(tf.matmul(z2,w2) + b2)

# Define the Cross-entropy/Log-loss Cost function based on the output label y and 
# the predicted probability by the forward pass 
cost = tf.reduce_mean(( (y_ * tf.log(pred)) + ((1 - y_) * tf.log(1.0 - pred)) ) * -1) 
learning_rate = 0.01 
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) 

#Now that we have all that we need set up we will start the training 
XOR_X = [[0,0],[0,1],[1,0],[1,1]] 
XOR_Y = [[0],[1],[1],[0]]

# Initialize the variables 
init = tf.initialize_all_variables()
sess = tf.Session() 
writer = tf.summary.FileWriter("./Downloads/XOR_logs", sess.graph_def)
sess.run(init) 
for i in range(100000):     
    sess.run(train_step, feed_dict={x_: XOR_X, y_: XOR_Y})
print('Final Prediction', sess.run(pred, feed_dict={x_: XOR_X, y_: XOR_Y})) 

# Linear Regression Implementation in TensorFlow
# Importing TensorFlow, Numpy, and the Boston Housing price dataset
import tensorflow as tf
import numpy as np
from sklearn.datasets import load_boston

# Function to load the Boston data set
def read_infile():
    data = load_boston()
    features = np.array(data.data)
    target = np.array(data.target)
    return features,target

# Normalize the features by Z scaling; i.e., subtract from each feature value its mean and then divide by its
#standard deviation. Accelerates gradient descent.
def feature_normalize(data):
    mu = np.mean(data,axis=0)
    std = np.std(data,axis=0)
    return (data - mu)/std

# Append the feature for the bias term.
def append_bias(features,target):
    n_samples = features.shape[0]
    n_features = features.shape[1]
    intercept_feature  = np.ones((n_samples,1))
    X = np.concatenate((features,intercept_feature),axis=1)
    X = np.reshape(X,[n_samples,n_features +1])
    Y = np.reshape(target,[n_samples,1])
    return X,Y

#  Execute the functions to read, normalize, and add append bias term to the data
features,target = read_infile()
z_features = feature_normalize(features)
X_input,Y_input = append_bias(z_features,target)
num_features = X_input.shape[1]

# Create TensorFlow ops for placeholders, weights, and weight initialization
X = tf.placeholder(tf.float32,[None,num_features])
Y = tf.placeholder(tf.float32,[None,1])
w = tf.Variable(tf.random_normal((num_features,1)),name='weights')
init = tf.global_variables_initializer()

# Define the different TensorFlow ops and input parameters for Cost and Optimization.
learning_rate = 0.01
num_epochs = 1000
cost_trace = []
pred = tf.matmul(X,w)
error = pred - Y
cost = tf.reduce_mean(tf.square(error))
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)

# Execute the gradient-descent learning
with tf.Session() as sess:
    sess.run(init)
    for i in range(num_epochs):
        sess.run(train_op,feed_dict={X:X_input,Y:Y_input})
        cost_trace.append(sess.run(cost,feed_dict={X:X_input,Y:Y_input}))
        error_ = sess.run(error,{X:X_input,Y:Y_input})
        pred_ = sess.run(pred,{X:X_input})
print('MSE in training:',cost_trace[-1])

# Multi-class Classification with Softmax Function Using Full-Batch Gradient Descent
# Import the required libraries 
import tensorflow as tf
import numpy as np
from sklearn import datasets 
from tensorflow.examples.tutorials.mnist import input_data

# Function to read the MNIST dataset along with the labels
def read_infile():  
    mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)  
    train_X, train_Y,test_X, test_Y = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels   
    return train_X, train_Y,test_X, test_Y

#  Define the weights and biases for the neural network 
def weights_biases_placeholder(n_dim,n_classes):  
    X = tf.placeholder(tf.float32,[None,n_dim])    
    Y = tf.placeholder(tf.float32,[None,n_classes])  
    w = tf.Variable(tf.random_normal([n_dim,n_classes],stddev=0.01),name='weights') 
    b = tf.Variable(tf.random_normal([n_classes]),name='weights')  
    return X,Y,w,b

# Define the forward pass
def forward_pass(w,b,X):  
    out = tf.matmul(X,w) + b
    return out

# Define the cost function for the SoftMax unit
def multiclass_cost(out,Y):  
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=out,labels=Y)) 
    return cost

# Define the initialization op
def init():
    return tf.global_variables_initializer()

# Define the training op
def train_op(learning_rate,cost): 
    op_train = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)   
    return op_train
train_X, train_Y,test_X, test_Y = read_infile()
X,Y,w,b = weights_biases_placeholder(train_X.shape[1],train_Y.shape[1])
out = forward_pass(w,b,X)
cost = multiclass_cost(out,Y)
learning_rate,epochs = 0.01,1000 
op_train = train_op(learning_rate,cost) 
init = init() 
loss_trace = [] 
accuracy_trace = []

# Activate the TensorFlow session and execute the stochastic gradient descent 
with tf.Session() as sess:   
    sess.run(init)
    for i in range(epochs):    
        sess.run(op_train,feed_dict={X:train_X,Y:train_Y})  
        loss_ = sess.run(cost,feed_dict={X:train_X,Y:train_Y})  
        accuracy_ = np.mean(np.argmax(sess.run(out,feed_dict={X:train_X,Y:train_Y}),axis=1) == np.argmax(train_Y,axis=1))       
        loss_trace.append(loss_)      
        accuracy_trace.append(accuracy_)   
        if (((i+1) >= 100) and ((i+1) % 100 == 0 )) :   
            print('Epoch:',(i+1),'loss:',loss_,'accuracy:',accuracy_)
            
    print('Final training result:','loss:',loss_,'accuracy:',accuracy_)   
    loss_test = sess.run(cost,feed_dict={X:test_X,Y:test_Y})  
    test_pred = np.argmax(sess.run(out,feed_dict={X:test_X,Y:test_Y}),axis=1)  
    accuracy_test = np.mean(test_pred == np.argmax(test_Y,axis=1))  
    print('Results on test dataset:','loss:',loss_test,'accuracy:',accuracy_test)

import tensorflow as tf
import numpy as np
from sklearn import datasets 
from tensorflow.examples.tutorials.mnist import input_data

# Multi-class Classification with Softmax Function Using Stochastic Gradient Descent
def read_infile():   
    mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)  
    train_X, train_Y,test_X, test_Y = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels   
    return train_X, train_Y,test_X, test_Y

def weights_biases_placeholder(n_dim,n_classes): 
    X = tf.placeholder(tf.float32,[None,n_dim])   
    Y = tf.placeholder(tf.float32,[None,n_classes])   
    w = tf.Variable(tf.random_normal([n_dim,n_classes],stddev=0.01),name='weights') 
    b = tf.Variable(tf.random_normal([n_classes]),name='weights')   
    return X,Y,w,b

def forward_pass(w,b,X):  
    out = tf.matmul(X,w) + b  
    return out

def multiclass_cost(out,Y):   
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=out,labels=Y))  
    return cost

def init(): 
    return tf.global_variables_initializer()

def train_op(learning_rate,cost): 
    op_train = tf.train.AdamOptimizer(learning_rate).minimize(cost) 
    return op_train

train_X, train_Y,test_X, test_Y = read_infile()
X,Y,w,b = weights_biases_placeholder(train_X.shape[1],train_Y.shape[1]) 
out = forward_pass(w,b,X) 
cost = multiclass_cost(out,Y) 
learning_rate,epochs,batch_size = 0.01,1000,1000 
num_batches = int(train_X.shape[0]/batch_size)
op_train = train_op(learning_rate,cost) 
init = init() 
epoch_cost_trace = []
epoch_accuracy_trace = []

with tf.Session() as sess:  
    sess.run(init)
    for i in range(epochs):  
        epoch_cost, epoch_accuracy = 0, 0
        for j in range(num_batches):         
            sess.run(op_train,feed_dict={X:train_X[j*batch_size:(j+1)*batch_size],Y:train_Y[j*batch_size:(j+1)*batch_size]})      
            actual_batch_size = train_X[j*batch_size:(j+1)*batch_size].shape[0]         
            epoch_cost += actual_batch_size*sess.run(cost,feed_dict={X:train_X[j*batch_size:(j+1)*batch_size],Y:train_Y[j*batch_size:(j+1)*batch_size]})
            
        epoch_cost = epoch_cost/float(train_X.shape[0])     
        epoch_accuracy = np.mean(np.argmax(sess.run(out,feed_dict={X:train_X,Y:train_Y}), axis=1) == np.argmax(train_Y,axis=1))    
        epoch_cost_trace.append(epoch_cost)     
        epoch_accuracy_trace.append(epoch_accuracy)
        if (((i +1) >= 100) and ((i+1) % 100 == 0 )) :  
            print('Epoch:',(i+1),'Average loss:',epoch_cost,'accuracy:',epoch_accuracy)
    print('Final epoch training results:','Average loss:',epoch_cost,'accuracy:',epoch_accuracy)   
    loss_test = sess.run(cost,feed_dict={X:test_X,Y:test_Y})    
    test_pred = np.argmax(sess.run(out,feed_dict={X:test_X,Y:test_Y}),axis=1) 
    accuracy_test = np.mean(test_pred == np.argmax(test_Y,axis=1)) 
    print('Results on test dataset:','Average loss:',loss_test,'accuracy:',accuracy_test)