%
% Using Gradient Descent to solve 
% the (over- and under-determined) least squares
%
% For over-determined least squares, if A has full column rank, 
%    then the solution is unique 
%
% For under-determined least squares, if A has full row rank, 
%    then the solution is not-unique, Matlab's solution gives 
%    a sparse solution. On the other hand, the GD/normal equation
%    gives a minimum norm solution.  
%
%-----------------------------------------------------------------

% generate test data
%
%  random data for LS problem ||A*x - b|| 
n = 10;
m = 3; 
A = randn(n,m);
b = randn(n,1); 

% set up parameters

tau = 0.01; % learning rate
tol = 1e-3;    
kmax = 200; 

x = zeros(m,1);
for k = 1:kmax
    r = A'*(A*x - b); 
    xp = x - tau*r;
    res(k) = norm(r); 
    if res(k) <= tol,
       disp('GD converged at')
          k 
       break
    end
    x = xp; 

end
FinaRes = norm(A*x - b) 

%-----------

figure(1)  
semilogy((1:k),res(1:k),'r+')
title('convergence history') 
xlabel('iter k') 
ylabel('residul norm') 

%------------

xls = A\b; 
ExactRes = norm(A*xls - b) 

Solution = [x xls]
