게으른 우루루

from sklearn.datasets import load_iris

import numpy as np

import tensorflow as tf

from tensorflow.python.framework import ops

ops.reset_default_graph()

sess = tf.Session()

iris = load_iris()

# load the data

x_vals = iris.data[:, [0,3]]

print(x_vals[:6])

# [[5.1 0.2]

#  [4.9 0.2]

#  [4.7 0.2]

#  [4.6 0.2]

#  [5.  0.2]

#  [5.4 0.4]]

y_vals = np.array([1 if y==0 else -1 for y in iris.target])

print(y_vals)

# [ 1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1

#   1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1

#   1  1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1

#  -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1

#  -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1

#  -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1

#  -1 -1 -1 -1 -1 -1]

# split data

from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(x_vals, y_vals,

                                                    random_state=0, stratify=y_vals)

# placeholder

x_data = tf.placeholder(shape=[None, 2], dtype=tf.float32)

y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

# create Variable

A = tf.Variable(tf.random_normal(shape=[2, 1]))

b = tf.Variable(tf.random_normal(shape=[1, 1]))

# fomula (y = ax - b)

fomula = tf.subtract(tf.matmul(x_data, A), b)

# declare loss function

# Loss = summation(max(0, 1 - pred*actual)) + alpha * L2_norm(A)^2

alpha = tf.constant([0.01])

# Margin term in loss

left_term = tf.reduce_mean(tf.maximum(0., tf.subtract(1., tf.multiply(fomula, y_target))))

# Put terms together

loss = tf.add(left_term, tf.multiply(alpha, L2_norm))

# prediction function

prediction = tf.sign(fomula)

accuracy = tf.reduce_mean(tf.cast(tf.equal(prediction, y_target), tf.float32))

train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)

# initialize variable

init = tf.global_variables_initializer()

sess.run(init)

test_acc = []

train_acc = []

loss_vec = []

batch_size = 112

for i in np.arange(1000):

    rand_idx = np.random.choice(len(x_train), size=batch_size)

    rand_x = x_train[rand_idx]

    rand_y = y_train[rand_idx].reshape(-1, 1)

    sess.run(train_step, feed_dict={x_data:rand_x, y_target:rand_y})

    temp_loss = sess.run(loss, feed_dict={x_data:rand_x, y_target:rand_y})

    loss_vec.append(temp_loss)

    temp_train_acc = sess.run(accuracy, feed_dict={x_data:x_train, y_target:y_train.reshape(-1, 1)})

    train_acc.append(temp_train_acc)

    temp_test_acc = sess.run(accuracy, feed_dict={x_data:x_test, y_target:y_test.reshape(-1, 1)})

    test_acc.append(temp_test_acc)

    if (i+1) % 150 == 0:

        print('step {}:\n  A={}, b={}\nloss={}\n'.format(i+1, sess.run(A), sess.run(b), temp_loss))

        # step 150:

        # A=[[-0.1508689 ]

        #    [-0.02083003]], b=[[0.03109207]]

        # loss=[0.6171418]

        # step 300:

        # A=[[-0.06546283]

        #    [-0.4349473 ]], b=[[-0.04024722]]

        # loss=[0.39113477]

        # step 450:

        # A=[[ 0.01723341]

        #    [-0.8050836 ]], b=[[-0.12105083]]

        # loss=[0.41022006]

        # step 600:

        # A=[[ 0.07058643]

        #    [-1.1637722 ]], b=[[-0.19953299]]

        # loss=[0.2737451]

        # step 750:

        # A=[[ 0.14719042]

        #    [-1.503774  ]], b=[[-0.28408656]]

        # loss=[0.2209843]

        # step 900:

        # A=[[ 0.19714044]

        #    [-1.8002963 ]], b=[[-0.35890797]]

        # loss=[0.17502114]

# visualization

import matplotlib.pyplot as plt

[[a1], [a2]] = sess.run(A)

[[b]] = sess.run(b)

slope = -a2/a1

icept = b

best_fit = []

for i in x_vals[:, 1]:

    temp = i * slope + icept

    best_fit.append(temp)

setosa_x = [setosa[1] for i, setosa in enumerate(x_vals) if y_vals[i] == 1]

setosa_y = [setosa[0] for i, setosa in enumerate(x_vals) if y_vals[i] == 1]

not_setosa_x = [setosa[1] for i, setosa in enumerate(x_vals) if y_vals[i] == -1]

not_setosa_y = [setosa[0] for i, setosa in enumerate(x_vals) if y_vals[i] == -1]

_, axe = plt.subplots(1, 3)

axe[0].scatter(setosa_x, setosa_y, marker='o', label='setosa')

axe[0].scatter(not_setosa_x, not_setosa_y, marker='o', label='non-setosa')

axe[0].plot(x_vals[:, 1], best_fit, c='red', label='Linear Separator')

axe[0].set_ylim([0, 10])

axe[0].legend(loc='best')

axe[0].set_title('Sepal Length vs Petal Width')

axe[0].set_xlabel('Petal Width')

axe[0].set_ylabel('Sepal Length')

axe[1].plot(train_acc, c='k', linestyle='-', label='Training Accuracy')

axe[1].plot(test_acc, c='r', linestyle='--', label='Test Accuracy')

axe[1].set_title('Train and Test Accuracy')

axe[1].set_xlabel('Genearation')

axe[1].set_ylabel('Accuracy')

axe[1].legend(loc='best')

axe[2].plot(loss_vec, c='k')

axe[2].set_title('Loss per Generation')

axe[2].set_xlabel('Generation')

axe[2].set_ylabel('loss')

plt.show()

학습 진행중인 모델의 최종 결과 그래프와 테스트셋과 학습셋의 정확도, 두 분류가 선형적으로 구분 가능하기 때문에 정확도가 100%에 도달

import requests

import os.path

birth_weight_file = 'birth_weight.csv'

if not os.path.exists(birth_weight_file):

    url='https://raw.githubusercontent.com/nfmcclure/tensorflow_cookbook/master/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'

    load_file = requests.get(url)

    data = load_file.text.split('\r\n')

    header = data[0].split('\t')

    raw_birth_data = [[float(args) for args in row.split('\t') if len(args) >= 1] for row in data[1:] if len(row)>=1]

    length = len(raw_birth_data[0])

    birth_data = np.empty([0, length])

    for birth in raw_birth_data:

        birth_data = np.vstack([raw_birth_data, birth])

print(header)

# ['LOW', 'AGE', 'LWT', 'RACE', 'SMOKE', 'PTL', 'HT', 'UI', 'BWT']

print(birth_data)

# [[1.000e+00 2.800e+01 1.130e+02 ... 0.000e+00 1.000e+00 7.090e+02]

#  [1.000e+00 2.900e+01 1.300e+02 ... 0.000e+00 1.000e+00 1.021e+03]

#  [1.000e+00 3.400e+01 1.870e+02 ... 1.000e+00 0.000e+00 1.135e+03]

#  ...

#  [0.000e+00 2.400e+01 2.160e+02 ... 0.000e+00 0.000e+00 4.593e+03]

#  [0.000e+00 4.500e+01 1.230e+02 ... 0.000e+00 0.000e+00 4.990e+03]

#  [0.000e+00 4.500e+01 1.230e+02 ... 0.000e+00 0.000e+00 4.990e+03]]

import csv

with open(birth_weight_file, 'w', newline='') as f:

    writer = csv.writer(f)

    writer.writerow(header)

    writer.writerows(birth_data)

    f.close()

import numpy as np

birth_data = []

with open(birth_weight_file, newline='') as csvfile:

    csv_reader = csv.reader(csvfile)

    birth_header = next(csv_reader)

    birth_data = [[float(args) for args in row] for row in csv_reader]

    birth_data = np.array(birth_data)

print(birth_data)

# [[1.000e+00 2.800e+01 1.130e+02 ... 0.000e+00 1.000e+00 7.090e+02]

#  [1.000e+00 2.900e+01 1.300e+02 ... 0.000e+00 1.000e+00 1.021e+03]

#  [1.000e+00 3.400e+01 1.870e+02 ... 1.000e+00 0.000e+00 1.135e+03]

#  ...

#  [0.000e+00 2.400e+01 2.160e+02 ... 0.000e+00 0.000e+00 4.593e+03]

#  [0.000e+00 4.500e+01 1.230e+02 ... 0.000e+00 0.000e+00 4.990e+03]

#  [0.000e+00 4.500e+01 1.230e+02 ... 0.000e+00 0.000e+00 4.990e+03]]

import pandas as pd

birth_data = []

birth_data_pd = pd.read_csv(birth_weight_file)

birth_data = birth_data_pd.values

print(birth_data)

# [[1.000e+00 2.800e+01 1.130e+02 ... 0.000e+00 1.000e+00 7.090e+02]

#  [1.000e+00 2.900e+01 1.300e+02 ... 0.000e+00 1.000e+00 1.021e+03]

#  [1.000e+00 3.400e+01 1.870e+02 ... 1.000e+00 0.000e+00 1.135e+03]

#  ...

#  [0.000e+00 2.400e+01 2.160e+02 ... 0.000e+00 0.000e+00 4.593e+03]

#  [0.000e+00 4.500e+01 1.230e+02 ... 0.000e+00 0.000e+00 4.990e+03]

#  [0.000e+00 4.500e+01 1.230e+02 ... 0.000e+00 0.000e+00 4.990e+03]]

birth_m_data = birth_data[:,1:8]

print(birth_m_data)

# [[ 28. 113.   1. ...   1.   0.   1.]

#  [ 29. 130.   0. ...   0.   0.   1.]

#  [ 34. 187.   1. ...   0.   1.   0.]

#  ...

#  [ 24. 216.   0. ...   0.   0.   0.]

#  [ 45. 123.   0. ...   1.   0.   0.]

#  [ 45. 123.   0. ...   1.   0.   0.]]

birth_target = birth_data[:, 0]

print(birth_target)

# [1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.

#  1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.

#  1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.

#  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.

#  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.

#  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.

#  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.

#  0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]

# Split data into train/test = 80%/20%

data_size = len(birth_m_data)

train_idx = np.random.choice(data_size, round(data_size*0.8), replace=False)

temp_test_idx = set(np.arange(data_size)) - set(train_idx)

test_idx = np.array(list(temp_test_idx))

x_train = birth_m_data[train_idx]

x_test = birth_m_data[train_idx]

y_train = birth_target[test_idx]

y_test = birth_target[test_idx]

print(x_train)

# [[ 20. 105.   1. ...   0.   0.   0.]

#  [ 31. 100.   0. ...   0.   0.   1.]

#  [ 32. 132.   0. ...   0.   0.   0.]

#  ...

#  [ 20. 120.   1. ...   0.   0.   0.]

#  [ 20. 150.   0. ...   0.   0.   0.]

#  [ 16. 110.   1. ...   0.   0.   0.]]

print(x_test)

# [[ 20. 105.   1. ...   0.   0.   0.]

#  [ 31. 100.   0. ...   0.   0.   1.]

#  [ 32. 132.   0. ...   0.   0.   0.]

#  ...

#  [ 20. 120.   1. ...   0.   0.   0.]

#  [ 20. 150.   0. ...   0.   0.   0.]

#  [ 16. 110.   1. ...   0.   0.   0.]]

# user define function

def zero_to_one(m):

    col_max = np.max(x_train, axis=0)

    col_min = np.min(x_train, axis=0)

    return (m-col_min)/(col_max-col_min)

x_train_scaled_def = np.nan_to_num(zero_to_one(x_train))

print(x_train_scaled_def)

# [[0.19354839 0.14705882 1.         ... 0.         0.         0.        ]

#  [0.5483871  0.11764706 0.         ... 0.         0.         1.        ]

#  [0.58064516 0.30588235 0.         ... 0.         0.         0.        ]

#  ...

#  [0.19354839 0.23529412 1.         ... 0.         0.         0.        ]

#  [0.19354839 0.41176471 0.         ... 0.         0.         0.        ]

#  [0.06451613 0.17647059 1.         ... 0.         0.         0.        ]]

x_test_scaled_def = np.nan_to_num(zero_to_one(x_test))

print(x_test_scaled_def)

# [[0.19354839 0.14705882 1.         ... 0.         0.         0.        ]

#  [0.5483871  0.11764706 0.         ... 0.         0.         1.        ]

#  [0.58064516 0.30588235 0.         ... 0.         0.         0.        ]

#  ...

#  [0.19354839 0.23529412 1.         ... 0.         0.         0.        ]

#  [0.19354839 0.41176471 0.         ... 0.         0.         0.        ]

#  [0.06451613 0.17647059 1.         ... 0.         0.         0.        ]]

# scikit-learn

from sklearn.preprocessing import MinMaxScaler

scaler = MinMaxScaler()

scaler.fit(x_train)

x_train_scaled = scaler.transform(x_train)

print(x_train_scaled)

# [[0.19354839 0.14705882 1.         ... 0.         0.         0.        ]

#  [0.5483871  0.11764706 0.         ... 0.         0.         1.        ]

#  [0.58064516 0.30588235 0.         ... 0.         0.         0.        ]

#  ...

#  [0.19354839 0.23529412 1.         ... 0.         0.         0.        ]

#  [0.19354839 0.41176471 0.         ... 0.         0.         0.        ]

#  [0.06451613 0.17647059 1.         ... 0.         0.         0.        ]]

x_test_scaled = scaler.transform(x_test)

print(x_test_scaled)

# [[0.19354839 0.14705882 1.         ... 0.         0.         0.        ]

#  [0.5483871  0.11764706 0.         ... 0.         0.         1.        ]

#  [0.58064516 0.30588235 0.         ... 0.         0.         0.        ]

#  ...

#  [0.19354839 0.23529412 1.         ... 0.         0.         0.        ]

#  [0.19354839 0.41176471 0.         ... 0.         0.         0.        ]

#  [0.06451613 0.17647059 1.         ... 0.         0.         0.        ]]

import tensorflow as tf

from tensorflow.python.framework import ops

import matplotlib.pyplot as plt

ops.reset_default_graph()

# Create graph

sess = tf.Session()

# Set for reproducible results

np.random.seed(seed=99)

tf.set_random_seed(seed=99)

# Initialize placeholders

x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32)

y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

# Create variables for linear regression

A = tf.Variable(tf.random_normal(shape=[7,1]))

b = tf.Variable(tf.random_normal(shape=[1,1]))

# Declare model operations

fomula = tf.add(tf.matmul(x_data, A), b)

# Declare loss function (Cross Entropy loss)

loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=fomula, labels=y_target))

# Declare optimizeraon

opt = tf.train.GradientDescentOptimizer(0.01)

train_step = opt.minimize(loss)

# Initialize variables

init = tf.global_variables_initializer()

sess.run(init)

# Actual Prediction

prediction = tf.round(tf.sigmoid(fomula))

predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32) # tf.cast: 자료형태를 변환합니다. tf.cast(input tensor, dtype)

accuracy = tf.reduce_mean(predictions_correct)

# Training loop

batch_size = 25

loss_vec = []

train_acc = []

test_acc = []

for i in range(1500):

    rand_idx = np.random.choice(len(x_train_scaled), size=batch_size)

    rand_x = x_train_scaled[rand_idx]

    rand_y = y_train[rand_idx].reshape(-1, 1) # reshape에 주의

    sess.run(train_step, feed_dict={x_data:rand_x, y_target:rand_y})

    temp_loss = sess.run(loss, feed_dict={x_data:rand_x, y_target:rand_y}) # train step을 실행

    loss_vec.append(temp_loss)

    temp_train = sess.run(accuracy, feed_dict={x_data:x_train_scaled, y_target:y_train.reshape(-1, 1)}) # accuracy를 실행, reshape 주의

    train_acc.append(temp_train)

    temp_test = sess.run(accuracy, feed_dict={x_data:x_test_scaled, y_target:y_test.reshape(-1, 1)}) # accuracy를 실행, reshape 주의

    test_acc.append(temp_test)

    if (i+1)%300==0:

        print('Loss = {}'.format(temp_loss))

        # Loss: 0.8982473015785217

        # Loss: 0.641773521900177

        # Loss: 0.630251944065094

        # Loss: 0.5026944279670715

        # Loss: 0.5662710070610046

        # Loss: 0.6175627708435059

        # Loss: 0.5372310876846313

sess.close()

_, axe = plt.subplots(1, 2)

# Plot loss over time

axe[0].plot(loss_vec, 'k-')

axe[0].set_title('Cross Entropy Loss per Generation')

axe[0].set_xlabel('Generation')

axe[0].set_ylabel('Cross Entropy Loss')

# Plot train and test accuracy

axe[1].plot(train_acc, 'k-', label='Train Set Accuracy')

axe[1].plot(test_acc, 'r--', label='Test Set Accuracy')

axe[1].set_title('Train and Test Accuracy')

axe[1].set_xlabel('Generation')

axe[1].set_ylabel('Accuracy')

axe[1].legend(loc='lower right')

plt.show()

import matplotlib.pyplot as plt

import numpy as np

import tensorflow as tf

from sklearn import datasets

from tensorflow.python.framework import ops

def elastic_net_func(idx, batch_size):

    ops.reset_default_graph()

    # iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)]

    iris = datasets.load_iris()

    x_vals = iris.data[:, idx]

    y_vals = iris.data[:, 0]

    # Initialize placeholders

    x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)

    y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

    # Create variables for linear regression

    A = tf.Variable(tf.random_normal(shape=[1,1]))

    b = tf.Variable(tf.random_normal(shape=[1,1]))

    with tf.Session() as sess:

        fomula = tf.add(tf.matmul(x_data, A), b)

        np.random.seed(seed=42)

        tf.set_random_seed(seed=42)

        l1_a_loss = tf.reduce_mean(tf.abs(A))

        l2_a_loss = tf.reduce_mean(tf.square(A))

        y_loss = tf.reduce_mean(tf.square(y_target - fomula)) 

        loss = tf.add(tf.add(l1_a_loss, l2_a_loss), y_loss) # cost function

        opt = tf.train.GradientDescentOptimizer(0.001)

        train_step = opt.minimize(loss)

        # Initialize variables

        init = tf.global_variables_initializer()

        init.run()

        loss_vec = []

        for i in range(1000):

            rand_idx = np.random.choice(len(x_vals), size=batch_size)

            rand_x = x_vals[rand_idx].reshape(-1, 1)

            rand_y = y_vals[rand_idx].reshape(-1, 1)

            my_dict = {x_data:rand_x, y_target:rand_y}

            sess.run(train_step, feed_dict=my_dict)

            temp_loss = sess.run(loss, feed_dict=my_dict)

            loss_vec.append(temp_loss)

            if (i+1)%200==0:

                print('step {}: A={}, b={}, Loss={}'.format(i+1, A.eval()[0], b.eval()[0], temp_loss))

        [coef] = A.eval()[0]

        [intercept] = b.eval()[0]

        best_params = []

        for i in x_vals:

            poly = i*coef + intercept

            best_params.append(poly)

    _, axe = plt.subplots(1, 2)

    axe[0].scatter(x_vals, y_vals, edgecolors='k', label='data points')

    axe[0].plot(x_vals, best_params, c='red', label='best pit line')

    axe[0].set_title('index={}\nslope = {:.5f}, intercept = {:.5f}'.format(idx ,coef, intercept))

    axe[0].set_xlabel('{}'.format(iris.feature_names[idx]))

    axe[0].set_ylabel('{}'.format(iris.feature_names[0]))

    axe[0].legend(loc=2)

    axe[1].plot(loss_vec, c='k')

    axe[1].set_xlabel('Generation')

    axe[1].set_ylabel('Loss')

    axe[1].set_title('Loss per Generation')

    plt.show()

elastic_net_func(idx=1, batch_size=50)

sepal width vs sepal length

# step 200: A=[1.70271], b=[-0.14161193], Loss=6.42409610748291

# step 400: A=[1.6244563], b=[0.16109753], Loss=6.042064189910889

# step 600: A=[1.5415831], b=[0.44974053], Loss=5.765136241912842

# step 800: A=[1.4513652], b=[0.72334766], Loss=4.744622707366943

# step 1000: A=[1.3864329], b=[0.99185705], Loss=5.004234790802002

elastic_net_func(idx=2, batch_size=50)

petal length vs sepal length

# step 200: A=[1.3025422], b=[-0.19726731], Loss=7.4931416511535645

# step 400: A=[1.2043393], b=[0.24439183], Loss=5.619389057159424

# step 600: A=[1.1216723], b=[0.6478416], Loss=4.893489837646484

# step 800: A=[1.0472624], b=[1.0189508], Loss=5.274661540985107

# step 1000: A=[0.9723399], b=[1.3565185], Loss=2.696936845779419

elastic_net_func(idx=3, batch_size=50)

petal width VS sepal length

# step 200: A=[1.3511531], b=[1.0177041], Loss=14.15509033203125

# step 400: A=[1.5765594], b=[2.0053844], Loss=8.151086807250977

# step 600: A=[1.4302077], b=[2.6739304], Loss=5.979291915893555

# step 800: A=[1.2245288], b=[3.1957018], Loss=4.475618362426758

# step 1000: A=[1.0373899], b=[3.6267276], Loss=3.252098560333252

이전 포스팅 보기

[1]Ridge, [2]Lasso

import tensorflow as tf

from tensorflow.python.framework import ops

import numpy as np

from sklearn.datasets import load_iris

ops.reset_default_graph()

iris = load_iris()

# iris.keys(): dict_keys(['data', 'target', 'target_names', 'DESCR', 'feature_names'])

# iris.feature_names: ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']

regression_type = 'LASSO' # LASSO, Ridge

x_vals = iris.data[:, 3] # petal width

y_vals = iris.data[:, 0] # sepal length

x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)

y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

A = tf.Variable(tf.random_normal(shape=[1, 1]))

b = tf.Variable(tf.random_normal(shape=[1, 1]))

deltas = [0.1, 1, 5, 50]

_, axes = plt.subplots(2, 2)

for ax, delta in zip(axes.ravel(), deltas):

    line = np.linspace(-5, 5)

    theta = 0.9

    step = 1

    y = 1/(np.exp((line-theta)*(-delta)) + 1) * step

    ax.plot(line, y)

    ax.set_title('$\delta$={}'.format(delta))

plt.show()

$\delta$값에 따른 heavyside step function

with tf.Session() as sess:

    fomula = tf.add(tf.matmul(x_data, A), b)

    if regression_type == 'LASSO':

        lasso_params = tf.constant(0.9, dtype=tf.float32) # limit slope

        heavyside_step = tf.truediv(1., tf.add(tf.exp(tf.multiply(tf.subtract(A, lasso_params), -50)), 1))

        regularization = tf.multiply(99., heavyside_step)

        loss = tf.add(tf.reduce_mean(tf.square(y_target-fomula)), regularization)

    elif regression_type == 'Ridge':

        ridge_params = tf.constant(1., dtype=tf.float32)

        ridge_loss = tf.reduce_mean(tf.square(A))

loss = tf.expand_dims(tf.add(tf.reduce_mean(tf.square(y_target-fomula)),

                                     tf.multiply(ridge_params, ridge_loss)), 0) 

    opt = tf.train.GradientDescentOptimizer(learning_rate=0.001)

    train_step = opt.minimize(loss)

    init = tf.global_variables_initializer()

    init.run()

    loss_vec = []

    for i in range(1500):

        rnd_idx = np.random.choice(len(x_vals), size = batch_size)

        rnd_x = x_vals[rnd_idx].reshape(-1, 1)

        rnd_y = y_vals[rnd_idx].reshape(-1, 1)

        my_dict = {x_data:rnd_x, y_target:rnd_y}

        sess.run(train_step, feed_dict=my_dict)

        temp_loss = sess.run(loss, feed_dict=my_dict)

        loss_vec.append(temp_loss[0])

        if (i+1)%300 == 0:

            print('step={}: A={}, b={}, Loss={}'.format(i+1, A.eval(), b.eval(), temp_loss))

     # step=300: A=[[0.7634046]], b=[[2.8413053]], Loss=[[5.382721]]

     # step=600: A=[[0.75245243]], b=[[3.7877374]], Loss=[[1.4977918]]

     # step=900: A=[[0.7403701]], b=[[4.3101645]], Loss=[[0.6288415]]

     # step=1200: A=[[0.72969586]], b=[[4.60605]], Loss=[[0.40883213]]

     # step=1500: A=[[0.72272784]], b=[[4.7734385]], Loss=[[0.3297159]]

    slope = A.eval()[0]

    cept = b.eval()[0]

best_fit = []    

for i in x_vals.ravel():

    poly = i*slope + cept

    best_fit.append(poly)

_, ax = plt.subplots(1, 2)

ax[0].scatter(x_vals, y_vals, edgecolors='k', label='Data Points')

ax[0].plot(x_vals, best_fit, c='red', label='Best fit line', linewidth=3)

ax[0].legend(loc=2)

ax[0].set_title('Petal Width vs Sepal Length')

ax[0].set_xlabel('Petal Width')

ax[0].set_ylabel('Sepal Length')

ax[1].plot(loss_vec, c='k')

ax[1].set_title(regression_type + ' Loss per Generation')

ax[1].set_xlabel('Generation')

ax[1].set_ylabel('Loss')

plt.show()

LASSO Regression과 Loss함수

my_regression_type(regression_type='LASSO', batch_size=50)

# step=300: A=[[0.764575]], b=[[2.5382898]], Loss=[[5.8621655]]

# step=600: A=[[0.7535565]], b=[[3.618642]], Loss=[[1.8615394]]

# step=900: A=[[0.7427662]], b=[[4.2173624]], Loss=[[0.7045775]]

# step=1200: A=[[0.7338138]], b=[[4.5521526]], Loss=[[0.44136143]]

# step=1500: A=[[0.72335386]], b=[[4.739038]], Loss=[[0.23825285]]

LASSO Linear Regression과 Loss

my_regression_type(regression_type='Ridge', batch_size=50)

# step=300: A=[[1.9796406]], b=[[1.416961]], Loss=[8.540436]

# step=600: A=[[1.7114378]], b=[[2.419876]], Loss=[5.517313]

# step=900: A=[[1.4033114]], b=[[3.132286]], Loss=[3.4445624]

# step=1200: A=[[1.1520133]], b=[[3.6732914]], Loss=[2.298217]

# step=1500: A=[[0.95343864]], b=[[4.086554]], Loss=[1.4839184]

Ridge Linear Regression과 Loss 

import tensorflow as tf

from tensorflow.python.framework import ops

import numpy as np

from sklearn.datasets import load_iris

ops.reset_default_graph()

iris = load_iris()

print(iris.keys())

# dict_keys(['data', 'target', 'target_names', 'DESCR', 'feature_names'])

print(iris.feature_names)

# ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']

# load the data

x_val = iris.data[:,3] # petal width

y_val = iris.data[:,0] # sepal length

# initialize placeholders

x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)

y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

# create variables for linear regression

A = tf.Variable(tf.random_normal(shape=[1, 1]))

b = tf.Variable(tf.random_uniform(shape=[1, 1]))

with tf.Session() as sess:

    fomula = tf.add(tf.matmul(x_data, A) ,b)

    demm_numer = tf.abs(tf.subtract(fomula, y_target)) # numerator

    demm_denom = tf.sqrt(tf.add(tf.square(A), 1)) # denominator

    loss = tf.reduce_mean(tf.truediv(demm_numer, demm_denom)) # 점과 직선사이의 거리

    opt = tf.train.GradientDescentOptimizer(learning_rate=0.15)

    train_step = opt.minimize(loss)

    init = tf.global_variables_initializer()

    init.run()

    loss_vec = []

    batch_size = 125

    for i in range(1000):

        rand_idx = np.random.choice(len(x_val), size=batch_size)

        rand_x = x_val[rand_idx].reshape(-1, 1)

        rand_y = y_val[rand_idx].reshape(-1, 1)

        my_dict = {x_data:rand_x, y_target:rand_y}

        sess.run(train_step, feed_dict=my_dict)

        temp_loss = sess.run(loss, feed_dict=my_dict)

        loss_vec.append(temp_loss)

        if (i+1)%100==0:

            print('step {}: A={}, b={}, Loss={}'.format(i+1, A.eval(), b.eval(), temp_loss))

            # step 100: A=[[2.8481812]], b=[[2.1150784]], Loss=0.39886653423309326

            # step 200: A=[[2.4716957]], b=[[2.581221]], Loss=0.4149680733680725

            # step 300: A=[[2.0858126]], b=[[3.1767926]], Loss=0.37009572982788086

            # step 400: A=[[1.5102198]], b=[[3.989578]], Loss=0.30516621470451355

            # step 500: A=[[1.0213077]], b=[[4.55735]], Loss=0.25061553716659546

            # step 600: A=[[1.0353084]], b=[[4.609328]], Loss=0.2725234925746918

            # step 700: A=[[1.0107175]], b=[[4.6160936]], Loss=0.3082656264305115

            # step 800: A=[[1.0400845]], b=[[4.612001]], Loss=0.27881959080696106

            # step 900: A=[[1.0318567]], b=[[4.6159105]], Loss=0.27347463369369507

            # step 1000: A=[[0.9662517]], b=[[4.5973287]], Loss=0.2258552461862564

    [slope] = A.eval()

    [cept] = b.eval()

import matplotlib.pyplot as plt

best_fit = []

for i in x_val.ravel():

    poly = i*slope[0] + cept[0]

    best_fit.append(poly)

_, axes = plt.subplots(1, 2)

axes[0].scatter(x_val, y_val, edgecolors='k', label='Data Points')

axes[0].plot(x_val, best_fit, c='red', label='Best fit line')

axes[0].set_title('Petal Width vs Sepal Length', size=12)

axes[0].set_xlabel('Petal Width')

axes[0].set_ylabel('Sepal Length')

axes[0].legend(loc=2)

axes[1].plot(loss_vec, c='k')

axes[1].set_title('Demming Loss per Generation', size=12)

axes[1].set_xlabel('Iteration')

axes[1].set_ylabel('Demming Loss')

plt.show()

Demming Regression과 Loss Function

from sklearn.datasets import load_iris

import tensorflow as tf

import numpy as np

from tensorflow.python.framework import ops

ops.reset_default_graph()

iris = load_iris()

print(iris.keys())

# dict_keys(['data', 'target', 'target_names', 'DESCR', 'feature_names'])

print(iris.feature_names)

# ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']

x_vals = iris.data[:, 3] # petal width

y_vals = iris.data[:, 0] # sepal length

batch_size = 25

lrn_rate = 0.1 # divergenc at 0.4

iterations = 100

x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)

y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

A = tf.Variable(tf.random_normal(shape=[1, 1]))

b = tf.Variable(tf.random_normal(shape=[1, 1]))

with tf.Session() as sess:

    model_output = tf.add(tf.matmul(x_data, A), b)

    loss_l1 = tf.reduce_mean(tf.abs(y_target - model_output))

    my_opt_l1 = tf.train.GradientDescentOptimizer(learning_rate=lrn_rate)

    train_step_l1 = my_opt_l1.minimize(loss_l1)

    init = tf.global_variables_initializer()

    init.run()

    loss_vec_l1 = []

    for i in range(iterations):

        rnd_idx = np.random.choice(len(x_vals), size=batch_size)

        rnd_x = x_vals[rnd_idx].reshape(-1, 1)

        rnd_y = y_vals[rnd_idx].reshape(-1, 1)

        feed = {x_data:rnd_x, y_target:rnd_y}

        sess.run(train_step_l1, feed_dict=feed)

        temp_loss_l1 = sess.run(loss_l1, feed_dict=feed)

        loss_vec_l1.append(temp_loss_l1)

        if (i+1)%10 == 0:

            print('Step {}: A={}, b={}'.format(i+1, A.eval(), b.eval()))

            # Step 10: A=[[-0.4101382]], b=[[-0.2588229]]

            # Step 20: A=[[0.7642619]], b=[[0.74117714]]

            # Step 30: A=[[1.9774618]], b=[[1.7411773]]

            # Step 40: A=[[2.3398612]], b=[[2.3091774]]

            # Step 50: A=[[2.252261]], b=[[2.6771777]]

ops.reset_default_graph()

iris = load_iris()

print(iris.keys())

# dict_keys(['data', 'target', 'target_names', 'DESCR', 'feature_names'])

print(iris.feature_names)

# ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']

x_vals = iris.data[:, 3] # petal width

y_vals = iris.data[:, 0] # sepal length

x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)

y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

A = tf.Variable(tf.random_normal(shape=[1, 1]))

b = tf.Variable(tf.random_normal(shape=[1, 1]))

with tf.Session() as sess:

    fomula = tf.add(tf.matmul(x_data, A), b)

    loss_l2 = tf.reduce_mean(tf.square(y_target - fomula))

    opt = tf.train.GradientDescentOptimizer(lrn_rate)

    train_step_l2 = opt.minimize(loss_l2)

    init = tf.global_variables_initializer()

    init.run()

    loss_vec_l2 = []

    for i in range(iterations):

        rand_idx = np.random.choice(len(x_vals), size=batch_size)

        rand_x = x_vals[rand_idx].reshape(-1, 1)

        rand_y = y_vals[rand_idx].reshape(-1, 1)

        my_dict = {x_data:rand_x, y_target:rand_y}

        sess.run(train_step_l2, feed_dict=my_dict)

        temp_loss_l2 = sess.run(loss_l2, feed_dict=my_dict)

        loss_vec_l2.append(temp_loss_l2)

        if (i+1)%20 == 0:

            print('step {}: A={}, b={}'.format(i+1, A.eval()[0], b.eval()[0]))

            # step 20: A=[1.876194], b=[3.1437955]

            # step 40: A=[1.3978924], b=[4.1212244]

            # step 60: A=[1.1288545], b=[4.4799623]

            # step 80: A=[0.9617775], b=[4.64575]

            # step 100: A=[0.88910973], b=[4.7133427]

import matplotlib.pyplot as plt

plt.plot(loss_vec_l1, c='k', label='L1 Loss')

plt.plot(loss_vec_l2, c='red', ls='--', label='L2 Loss')

plt.title('L1 and L2 Loss per Generation, learning rate={}'.format(lrn_rate))

plt.xlabel('Generation')

plt.ylabel('Loss')

plt.legend(loc=1)

plt.show()