Artificial Neural Networks: Delta Learning Rule
Table of Contents
Problem
The aim is to calculate the weight constants of an artificial neuron using the Delta Leaning Rule/Algorithm. Inputs are the x and y coordinates of sample points. This system should be able to classify sample points, if they can be discriminated by a single linear function.
The first step was to write a program which involves only single neuron. I used MATLAB for this task. Later, to implement a multiclass function to the program I had to add more neurons. For this step I prefered to use another language, Python. One bird, two stones; introduction for both a new programming language and a new subject for me. Current versions of codes can be followed in Github for Python and MATLAB.
Note: All the GUI elements are in Turkish.
MATLAB Code
% Delta Learning Algorithm with single neuron/layer
clear;close all;clc;
% Parameters:
screen=[-10 10 -10 10];
error_limit=0.001;
counter_limit=4000;
LearningRate=0.05;
w0=0; % Initial weight vector, 0 for random
PixelDensity=2.5; % For visualizing the classes of each pixels
figure
title("Mavi sınıf için seç, ENTER")
hold on
axis(screen)
[xi,yi] = getpts;
X=[xi yi];
Y=ones(size(xi,1),1);
plot(xi,yi,'b*')
title("Kırmızı sınıf için seç, ENTER")
[xi,yi] = getpts;
X=[X; xi yi];
Y=[Y ; zeros(size(xi,1),1)];
plot(xi,yi,'r*')
[X,C,S] = normalize(X);
plot(X((Y==1),1),X((Y==1),2),'bo')
plot(X((Y==0),1),X((Y==0),2),'ro')
[n, m] = size(X);
% Initial weight vector creation
if (w0==0); w = 0.01 * randn(m+1,1); else; w = w0; end
% Finding the w vector with least error
error=1;counter=0;
while ((error>error_limit)&&(counter<counter_limit))
error=0;counter=counter+1;
for sample = 1:n
output=sigmoid(w(1)+X(sample,:)*w(2:end));
w=w+LearningRate*(Y(sample)-output)*(output)*(1-output)*[1 X(sample,:)]';
error=error+1/2*(Y(sample)-output)^2;
end
end
% Calculating the accuracy
Z = sigmoid(w(1) + X * w(2:end));
accuracy=mean(Y == (-min(Y)*(Z>0.5)+(Z>0.5)+1*min(Y)));
disp("Tek nöron ve Delta Öğrenme Algoritması için sonuçlar:");
disp(n+" örnek noktasının sınıflandırılmasında %"+100*accuracy+" başarı edildi!")
disp("Eğitim için "+counter+" döngü kullanıldı ve hata "+error+" değerine kadar düşürüldü.")
% Visualition
draw=zeros(((screen(2)-screen(1))/(1/PixelDensity)+1)*((screen(4)-screen(3))/(1/PixelDensity)+1),3);counter=0;
for y=screen(1):1/PixelDensity: screen(2)
for x=screen(3):1/PixelDensity:screen(4)
normalized_data=normalize([x y],'center',C,'scale',S);
xn=normalized_data(1); yn=normalized_data(2);
net=transpose(w)*[1;xn;yn];
output=sigmoid(net);
counter=counter+1;
if output>0.5
draw(counter,:)=[x y 1];
else
draw(counter,:)=[x y 0];
end
end
end
draw_class_1=draw(draw(:,3)==1,:);
draw_class_2=draw(draw(:,3)==0,:);
k=boundary(draw_class_1(:,1),draw_class_1(:,2));
n=boundary(draw_class_2(:,1),draw_class_2(:,2));
title("%"+100*accuracy+ " başarı!")
fill(draw_class_1(k,1),draw_class_1(k,2),'blue','FaceAlpha',0.3,'LineStyle','none');
fill(draw_class_2(n,1),draw_class_2(n,2),'red','FaceAlpha',0.3,'LineStyle','none');
% clearvars -except accuracy error w X Y Z
% figure
% plot(sigmoid(w'*[ones(size(X,1),1) X(:,1) X(:,2)]'))
function Output = sigmoid(Input) % Y is the array of desired values
Output = 1 ./ (1 + exp(-Input)); % binary activation function
end
Python Code
import tkinter as tk # for GUI
import numpy as np # for matrix multiplication
# Parameters
gui_height = 400
gui_width = 600
error_limit = 0.001
counter_limit = 100000
LearningRate = 10
gui_point_size = 3
visualize_density = 8 #lower~more dense
number_of_outputs=3
number_of_layer=1
root = tk.Tk()
inputs = np.empty((2,0), int)
classes = np.empty((number_of_outputs,0), int)
def sigmoid(x):
return 1 / (1 + np.exp(-x))
# GUI Elements
label = tk.Label(root)
label.pack()
label.config(text = 'Merhaba! Başlamak için üstteki menüden sınıf seçin...')
canvas = tk.Canvas(root, height = gui_height, width = gui_width, bg = 'white')
canvas.create_line(0, gui_height/2, gui_width, gui_height/2, dash = (5,2)) # x-axis
canvas.create_line(gui_width/2, 0, gui_width/2, gui_height, dash = (5,2)) # y-axis
canvas.pack()
# Functions
def class_pick(color):
global fill_color
fill_color = color
label.config(text = 'Örnek noktaları koordinat düzlemi üzerine tıklayarak seçiniz',fg = color)
def set_parameters():
label.config(text = 'Bu özellik henüz geliştirilmedi. Parametleri kod üzerinden ayarlayınız.', fg = 'black')
def button(event):
global inputs, classes
x, y = event.x, event.y
try:
canvas.create_oval(x-gui_point_size, y-gui_point_size, x+gui_point_size, y+gui_point_size, width = 0, fill = fill_color)
x_coor = (x-int(gui_width/2))/10
y_coor = -(y-int(gui_height/2))/10
inputs=np.append(inputs, [[x_coor],[y_coor]],axis=1)
classes=np.append(classes,([[0],[0],[0]] if fill_color == 'red' else
[[0],[0],[1]] if fill_color == 'blue' else
[[0],[1],[0]] if fill_color == 'orange' else
[[0],[1],[1]] if fill_color == 'green' else
[[1],[0],[0]] if fill_color == 'purple' else
0),axis=1)
except:
label.config(text = 'Lütfen sınıf seçtikten sonra noktaları oluşturunuz!',fg = 'indigo')
canvas.bind('<Button-1>', button)
def classify():
try:
global w
inputs_norm = inputs/np.linalg.norm(inputs)
w = np.random.rand(number_of_outputs,len(inputs_norm)+1)
#w = np.zeros((number_of_outputs,len(inputs_norm)+1))
error = 1
counter = 0
while error>error_limit and counter<counter_limit:
error = 0
counter+= 1
for x in range(inputs_norm.shape[1]):
net = np.matmul(w,np.vstack([1,inputs_norm[:,[x]]]))
try:
output = sigmoid(net)
except OverflowError: # exp(709>) = hafıza hatası
output = 0
for k in range(number_of_outputs):
w[k,:] += LearningRate*(classes[k,x]-output[k])*np.hstack([1,inputs_norm[:,x]])
error += 1/2*np.sum((classes[:,[x]]-output)**2)
print(error)
label.config(text = "Sınıflandırma {} döngü kullanılarak tamamlandı!".format(counter),fg = 'green')
except IndexError:
label.config(text = 'Henüz örnek noktalar seçilmemiş!', fg = 'indigo')
def visualize():
label.config(text = 'Görselliştirme işlemine başlandı...')
print(w)
try:
w
output1=0
output2=0
output3=0
output4=0
for x in range(int(gui_width/visualize_density)):
for y in range(int(gui_height/visualize_density)):
m = x*visualize_density
n = y*visualize_density
x_coor = ((m-int(gui_width/2))/10)/np.linalg.norm(inputs)
y_coor = -(n-int(gui_height/2))/10/np.linalg.norm(inputs)
net1 = np.matmul(w[0,:],[[1],[x_coor],[y_coor]])
try:
output1 = sigmoid(net1)
except OverflowError:
output1=0
net2 = np.matmul(w[1,:],[[1],[x_coor],[y_coor]])
try:
output2 = sigmoid(net2)
except OverflowError:
output2=0
net3 = np.matmul(w[2,:],[[1],[x_coor],[y_coor]])
try:
output3 = sigmoid(net3)
except OverflowError:
output3=0
if output1<0.5 and output2<0.5 and output3<0.5:
canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline = 'red')
elif output1<0.5 and output2<0.5 and output3>0.5:
canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline = 'blue')
elif output1<0.5 and output2>0.5 and output3<0.5:
canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline = 'orange')
elif output1<0.5 and output2>0.5 and output3>0.5:
canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline = 'green')
elif output1>0.5:
canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline = 'purple')
label.config(text = 'Görselleştirme tamamlandı!', fg = 'green')
except NameError:
label.config(text = 'Henüz sınıflandırma yapılmamış!', fg = 'indigo')
# Menubar
menubar = tk.Menu(root)
fileMenu = tk.Menu(menubar)
fileMenu.add_command(label = "Kırmızı", command = lambda: class_pick('red'))
fileMenu.add_command(label = "Mavi", command = lambda: class_pick('blue'))
fileMenu.add_command(label = "Turuncu", command = lambda: class_pick('orange'))
fileMenu.add_command(label = "Yeşil", command = lambda: class_pick('green'))
fileMenu.add_command(label = "Mor", command = lambda: class_pick('purple'))
fileMenu.add_command(label = "Sarı", command = lambda: class_pick('yellow'))
fileMenu.add_command(label = "Siyah", command = lambda: class_pick('black'))
fileMenu.add_command(label = "Çıkış", command = root.quit)
menubar.add_cascade(label = "Sınıf Seçimi", menu = fileMenu)
functionsMenu = tk.Menu(menubar)
functionsMenu.add_command(label = "Sınıfları Ayır", command = classify)
functionsMenu.add_command(label = "Sınıfları Görselleştir", command = visualize)
menubar.add_cascade(label = "İşlemler", menu = functionsMenu)
settingsMenu = tk.Menu(menubar)
settingsMenu.add_command(label = "Parametreleri Ayarla", command = set_parameters)
menubar.add_cascade(label = "Ayarlar", menu = settingsMenu)
root.config(menu = menubar)
root.mainloop()