faydin.com | Blog

Artificial Neural Networks: Delta Learning Rule

Table of Contents

Problem

The aim is to calculate the weight constants of an artificial neuron using the Delta Leaning Rule/Algorithm. Inputs are the x and y coordinates of sample points. This system should be able to classify sample points, if they can be discriminated by a single linear function.

The first step was to write a program which involves only single neuron. I used MATLAB for this task. Later, to implement a multiclass function to the program I had to add more neurons. For this step I prefered to use another language, Python. One bird, two stones; introduction for both a new programming language and a new subject for me. Current versions of codes can be followed in Github for Python and MATLAB.

Note: All the GUI elements are in Turkish.

MATLAB Code

% Delta Learning Algorithm with single neuron/layer
clear;close all;clc;

% Parameters:
screen=[-10 10 -10 10];
error_limit=0.001;
counter_limit=4000;
LearningRate=0.05;
w0=0;               % Initial weight vector, 0 for random
PixelDensity=2.5;   % For visualizing the classes of each pixels

figure
title("Mavi sınıf için seç, ENTER")
hold on
axis(screen)
[xi,yi] = getpts;
X=[xi yi];
Y=ones(size(xi,1),1);
plot(xi,yi,'b*')
title("Kırmızı sınıf için seç, ENTER")
[xi,yi] = getpts;
X=[X; xi yi];
Y=[Y ; zeros(size(xi,1),1)];
plot(xi,yi,'r*')

[X,C,S] = normalize(X);

plot(X((Y==1),1),X((Y==1),2),'bo')
plot(X((Y==0),1),X((Y==0),2),'ro')
[n, m] = size(X);

% Initial weight vector creation
if (w0==0); w = 0.01 * randn(m+1,1); else; w = w0; end

% Finding the w vector with least error
error=1;counter=0;
while ((error>error_limit)&&(counter<counter_limit))
    error=0;counter=counter+1;
    for sample = 1:n
	output=sigmoid(w(1)+X(sample,:)*w(2:end));
	w=w+LearningRate*(Y(sample)-output)*(output)*(1-output)*[1 X(sample,:)]';
	error=error+1/2*(Y(sample)-output)^2;
    end
end

% Calculating the accuracy
Z = sigmoid(w(1) + X * w(2:end));
accuracy=mean(Y == (-min(Y)*(Z>0.5)+(Z>0.5)+1*min(Y)));
disp("Tek nöron ve Delta Öğrenme Algoritması için sonuçlar:");
disp(n+" örnek noktasının sınıflandırılmasında %"+100*accuracy+" başarı edildi!")
disp("Eğitim için "+counter+" döngü kullanıldı ve hata "+error+" değerine kadar düşürüldü.")

% Visualition
draw=zeros(((screen(2)-screen(1))/(1/PixelDensity)+1)*((screen(4)-screen(3))/(1/PixelDensity)+1),3);counter=0;
for y=screen(1):1/PixelDensity: screen(2)
    for x=screen(3):1/PixelDensity:screen(4)

	normalized_data=normalize([x y],'center',C,'scale',S);
	xn=normalized_data(1); yn=normalized_data(2);

	net=transpose(w)*[1;xn;yn];
	output=sigmoid(net);
	counter=counter+1;
	if output>0.5
	    draw(counter,:)=[x y 1];
	else
	    draw(counter,:)=[x y 0];
	end
    end
end
draw_class_1=draw(draw(:,3)==1,:);
draw_class_2=draw(draw(:,3)==0,:);
k=boundary(draw_class_1(:,1),draw_class_1(:,2));
n=boundary(draw_class_2(:,1),draw_class_2(:,2));
title("%"+100*accuracy+ " başarı!")
fill(draw_class_1(k,1),draw_class_1(k,2),'blue','FaceAlpha',0.3,'LineStyle','none');
fill(draw_class_2(n,1),draw_class_2(n,2),'red','FaceAlpha',0.3,'LineStyle','none');
% clearvars -except accuracy error w X Y Z

% figure
% plot(sigmoid(w'*[ones(size(X,1),1) X(:,1) X(:,2)]'))
function Output = sigmoid(Input)      % Y is the array of desired values
Output = 1 ./ (1 + exp(-Input));    % binary activation function
end

Python Code

import tkinter as tk # for GUI
import numpy as np   # for matrix multiplication


# Parameters
gui_height = 400
gui_width = 600
error_limit = 0.001
counter_limit = 100000
LearningRate = 10
gui_point_size = 3
visualize_density = 8 #lower~more dense

number_of_outputs=3
number_of_layer=1



root = tk.Tk()
inputs = np.empty((2,0), int)
classes = np.empty((number_of_outputs,0), int)

def sigmoid(x):
    return 1 / (1 + np.exp(-x))


# GUI Elements
label = tk.Label(root)
label.pack()
label.config(text = 'Merhaba! Başlamak için üstteki menüden sınıf seçin...')
canvas = tk.Canvas(root, height = gui_height, width = gui_width, bg = 'white')
canvas.create_line(0, gui_height/2, gui_width, gui_height/2, dash = (5,2))  # x-axis
canvas.create_line(gui_width/2, 0, gui_width/2, gui_height, dash = (5,2))  # y-axis
canvas.pack()

# Functions
def class_pick(color):
    global fill_color
    fill_color = color
    label.config(text = 'Örnek noktaları koordinat düzlemi üzerine tıklayarak seçiniz',fg = color)

def set_parameters():
    label.config(text = 'Bu özellik henüz geliştirilmedi. Parametleri kod üzerinden ayarlayınız.', fg = 'black')

def button(event):
    global inputs, classes
    x, y = event.x, event.y
    try:
	canvas.create_oval(x-gui_point_size, y-gui_point_size, x+gui_point_size, y+gui_point_size, width = 0, fill = fill_color)
	x_coor = (x-int(gui_width/2))/10
	y_coor = -(y-int(gui_height/2))/10
	inputs=np.append(inputs, [[x_coor],[y_coor]],axis=1)
	classes=np.append(classes,([[0],[0],[0]] if fill_color == 'red' else
				   [[0],[0],[1]] if fill_color == 'blue' else
				   [[0],[1],[0]] if fill_color == 'orange' else
				   [[0],[1],[1]] if fill_color == 'green' else 
				   [[1],[0],[0]] if fill_color == 'purple' else
				   0),axis=1)
    except:
	label.config(text = 'Lütfen sınıf seçtikten sonra noktaları oluşturunuz!',fg = 'indigo')
canvas.bind('<Button-1>', button)

def classify():
    try:
	global w
	inputs_norm = inputs/np.linalg.norm(inputs)
	w = np.random.rand(number_of_outputs,len(inputs_norm)+1)
	#w = np.zeros((number_of_outputs,len(inputs_norm)+1))
	error = 1
	counter = 0

	while error>error_limit and counter<counter_limit:
	    error = 0
	    counter+= 1
	    for x in range(inputs_norm.shape[1]):
		net = np.matmul(w,np.vstack([1,inputs_norm[:,[x]]]))
		try:
		    output = sigmoid(net)
		except OverflowError: # exp(709>) = hafıza hatası
		    output = 0
		for k in range(number_of_outputs):
		    w[k,:] += LearningRate*(classes[k,x]-output[k])*np.hstack([1,inputs_norm[:,x]])
		error += 1/2*np.sum((classes[:,[x]]-output)**2)
	    print(error) 
	label.config(text = "Sınıflandırma {} döngü kullanılarak tamamlandı!".format(counter),fg = 'green')
    except IndexError:
	label.config(text = 'Henüz örnek noktalar seçilmemiş!', fg = 'indigo')
def visualize():
    label.config(text = 'Görselliştirme işlemine başlandı...')
    print(w)
    try:
	w
	output1=0
	output2=0
	output3=0
	output4=0

	for x in range(int(gui_width/visualize_density)):
	    for y in range(int(gui_height/visualize_density)):

		m = x*visualize_density
		n = y*visualize_density
		x_coor = ((m-int(gui_width/2))/10)/np.linalg.norm(inputs)
		y_coor = -(n-int(gui_height/2))/10/np.linalg.norm(inputs)

		net1 = np.matmul(w[0,:],[[1],[x_coor],[y_coor]])
		try:
		    output1 = sigmoid(net1)
		except OverflowError:
		    output1=0               
		net2 = np.matmul(w[1,:],[[1],[x_coor],[y_coor]])
		try:
		    output2 = sigmoid(net2)
		except OverflowError:
		    output2=0               
		net3 = np.matmul(w[2,:],[[1],[x_coor],[y_coor]])
		try:
		    output3 = sigmoid(net3)
		except OverflowError:
		    output3=0


		if output1<0.5 and output2<0.5 and output3<0.5:     
		    canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline  = 'red')
		elif output1<0.5 and output2<0.5 and output3>0.5:     
		    canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline  = 'blue')
		elif output1<0.5 and output2>0.5 and output3<0.5:     
		    canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline  = 'orange')
		elif output1<0.5 and output2>0.5 and output3>0.5:     
		    canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline  = 'green')
		elif output1>0.5:     
		    canvas.create_oval(m-2, n-2, m+2, n+2, width = 1, outline  = 'purple')




	label.config(text = 'Görselleştirme tamamlandı!', fg = 'green')
    except NameError:
	label.config(text = 'Henüz sınıflandırma yapılmamış!', fg = 'indigo')

# Menubar
menubar = tk.Menu(root)

fileMenu = tk.Menu(menubar)
fileMenu.add_command(label = "Kırmızı", command = lambda: class_pick('red'))
fileMenu.add_command(label = "Mavi", command = lambda: class_pick('blue'))
fileMenu.add_command(label = "Turuncu", command = lambda: class_pick('orange'))
fileMenu.add_command(label = "Yeşil", command = lambda: class_pick('green'))
fileMenu.add_command(label = "Mor", command = lambda: class_pick('purple'))
fileMenu.add_command(label = "Sarı", command = lambda: class_pick('yellow'))
fileMenu.add_command(label = "Siyah", command = lambda: class_pick('black'))
fileMenu.add_command(label = "Çıkış", command = root.quit)
menubar.add_cascade(label = "Sınıf Seçimi", menu = fileMenu)

functionsMenu = tk.Menu(menubar)
functionsMenu.add_command(label = "Sınıfları Ayır", command = classify)
functionsMenu.add_command(label = "Sınıfları Görselleştir", command = visualize)
menubar.add_cascade(label = "İşlemler", menu = functionsMenu)

settingsMenu = tk.Menu(menubar)
settingsMenu.add_command(label = "Parametreleri Ayarla", command = set_parameters)
menubar.add_cascade(label = "Ayarlar", menu = settingsMenu)

root.config(menu = menubar)

root.mainloop()