\input{template}
\input{macros}

\begin{document}
\lecture{6}{Linear Algebra}{Alekh Agarwal}

\section{Row Rank of a matrix}

The space $\{x~:~Ax = \mathbf{0}\}$ is called the null space of the matrix $A$. Having already seen that the dimension of the null space of $A$ is equal to the $n$ - number of linearly independent columns in the matrix $A$, we now examine how this relates to the number of linearly independent rows of $A$. We will first prove that Gaussian Elimination does not change the number of linearly independent rows. Note that we have already proved that Gaussian Elimination does not change the set $\{x~:~Ax = \mathbf{0}\}$. Later we will relate the row rank to the dimension of $\{x~:~Ax = \mathbf{0}\}$.

We first prove the following lemma:

\begin{Lem}
Gaussian elimination does not change the number of linearly independent rows in a matrix.
\end{Lem}

\begin{proof}
Gaussian elimination consists of two elementary operations. Exchanging two rows and multiplying a row with a scalar and adding it to another row. It is clear that exchanging two rows does not change the row rank of a matrix Below we will show that multiplying a row with a scalar and adding it to another row does not change the row rank of a matrix. 

Consider an $n\times m$ matrix $A$ of rank $r$. Without loss of generality, let the basis of the row space be $A_1,\dots,A_r$, where $A_i$ denotes the $i_{th}$ row of $A$. 

Now look at the space spanned by the rows $A_1, A_2, \dots, A_r$. Suppose $A_i$ is replaced by $A_i + cA_j$. Clearly $A_i$ belongs to this new space and hence all the vectors in the old space are also in the new space. Also $A_i + cA_j$ is in the old space. So the vectors in the new space are also in the old space. Since the space has not changed, the dimension remains unchanged.
\end{proof}

Now, consider the following lemma.

\begin{Lem}
$\mathrm{dim}(\{x~:~Ax = {\bf 0}\}) \geq n - k$
\end{Lem}

\begin{proof}
By the previous lemma, we can assume that we can use Gaussian Elimination to solve the equation without changing either the null space or the row rank. After the Gaussian elimination, our matrix A looks like:
\begin{align}
\begin{pmatrix}
0&\dots&1&0&\dots&0&a_{k+1}&\dots&a_m\\
0&\dots&\dots1&0&\dots&0&a_{k+1}&\dots&a_m\\
~&~&~&\vdots&~&~&~&~\\
0&\dots&\dots&\dots&1&\dots&a_{k+1}&\dots&a_m\\
0&~&~&\dots&~&~&~&0\\
~&~&~&\vdots
\end{pmatrix}
\notag
\end{align}

Here $A_t$ contains a 1 in the position $i_t$ for $i \leq t \leq r$ and $1 \leq i_1 \leq i_2 \leq \dots \leq i_k \leq k$. For simplicity, we assume that $i_t = t$.

Now $x$ be any solution to the equation. Then it can be easily seen that we can assign any arbitrary values to $x_{k+1}\dots x_n$, and then solve for $x_1\dots x_k$ to find a solution to the equation.In particular then, we look at the vectors of the form $u^{i - k} = (x_1,x_2,\dots,x_n)\quad i=k+1,\dots,n$, where $u^{i - k}_i = 1$ and $u^{i - k}_j = 0 ~\mathrm{for}~ j > k,j \ne i$. Then the $u^i$'s are of the form:

\begin{displaymath}
\begin{matrix}
\begin{pmatrix}
u^1_1\\
\vdots\\
u^1_k\\
1\\
0\\
\vdots\\
0
\notag
\end{pmatrix}
&
\begin{pmatrix}
u^2_1\\
\vdots\\
u^2_k\\
0\\
1\\
\vdots\\
0
\notag
\end{pmatrix}
&
\cdots
&
\begin{pmatrix}
u^{n-k}_1\\
\vdots\\
u^{n-k}_k\\
0\\
0\\
\vdots\\
1
\notag
\end{pmatrix}
\end{matrix}
\end{displaymath}

where $u^{i}_j$ are obtained as outlined above for $1 \leq j \leq k$.
Clearly these $n - k$ vectors $u^1,\dots,u^{n-k}$ are linearly independent, as there last $(n - k)$ rows are linearly independent.
Also each $u^i$ is a solution to $Ax = \mathbf{0}$ by construction. Thus we have $(n - k)$ linearly independent solutions to $Ax = \mathbf{0}$. 
This proves the lemma.
\end{proof}

Now, as before we will prove that the $\mathrm{dim}(\{x~:~Ax = \mathbf{0}\}) = n - k$
\begin{Thm}
$\mathrm{dim}(\{x~:~Ax = {\bf 0}\}) = n - k$
\label{lec6:rowrank}
\end{Thm}

\begin{proof}

Consider any $x'$ such that $Ax' = \mathbf{0}$. Now we construct the new vectors:
\begin{align}
\tilde{x} &= x^{'}_{k+1}u^1 + \dots + x^{'}_n u^{n - k} \notag\\
x^{''} &= x' - \tilde{x} \notag
\end{align}

Note that by construction, 
\begin{equation}
x^{''}_i = 0,~i = k+1,\dots,n.
\label{lec6:x''}
\end{equation}

Then we have 
\begin{align}
Ax^{''} &= A(x' - \tilde{x})\notag\\
&= Ax' - \sum_{i=1}^{n - k}x^{'}_{k+i} Au_i\notag\\
&= \mathbf{0} - \sum_{i=1}^{n - k}x^{'}_{k+i} \mathbf{0} \quad\mathrm{because } Au_i = \mathbf{0}\notag\\
&= \mathbf{0} \label{lec6:soln}
\end{align}

Now the matrix $Ax^{''}$ looks like
\begin{align}
\begin{pmatrix}
x^{''}_1&0&~&\dots\\
0&x^{''}_2&0&\dots\\
~&~&~&\vdots&~&~&~&~\\
0&\dots&\dots&x^{''}_k&0&\dots\\
0&~&~&\dots&~&~&~&0\\
~&~&~&\vdots
\end{pmatrix}
\label{lec6:struct}
\end{align}

because of the structure of $A$ and Equation \ref{lec6:x''}.
Thus by Equations \ref{lec6:soln} and \ref{lec6:struct}, we get that 
\begin{align}
x^{''}_i &= 0\quad \mathrm{for}~i = 1,2,\dots,k\notag\\
\Rightarrow x'_i &= \sum_{j=1}^{n-k} x'_iu^j_i \quad \mathrm{for}~i = 1,2,\dots,k\label{lec6:firstdep}\\
\intertext{Also, we have}
x'_i &= \sum_{j=1}^{n-k} x'_{k+i}u^j_i \quad \mathrm{for}~i = k+1,\dots,n\label{lec6:secdep}
\end{align}
by definition of $u^i$'s.
Hence, by Equations \ref{lec6:firstdep} and \ref{lec6:secdep}, we get that 
\begin{equation}
x' = \sum_{i = 1}^{n - k} x'_{k+i}u^i
\notag
\end{equation}

Thus any other solution to $Ax = \mathbf{0}$ lies in the span of $u_1,\dots,u_{n-k}$. Thus we get
\begin{equation}
dim(\{x~:~Ax = \mathbf{0}\}) = n - k
\notag
\end{equation}
Hence proved
\end{proof}

By Theorem \ref{lec6:rowrank}, we get that the dimension of the null space of the matrix $A$ obtained after applying Gaussian elimination to the original matrix $A'$ is equal to the row rank of $A$. Having already proved that the Gaussian Elimination does not change the row rank of a matrix, this is also equal to the row rank of the origial matrix $A'$. 

This concludes the proof for the fact that the row rank of a matrix is equal to $n$ - the dimension of its null space. As we have already proved that this is also equal to the column rank of a matrix, we get as a corollary that the row rank and column rank of a matrix are equal.
\begin{Cor}
The row rank of a matrix is equal to its column rank.
\end{Cor}

\section{Solutions to $Ax = \mathbf{b}$}

Having studied the properties of $\{x~:~Ax = \mathbf{0}\}$, we looked at the solutions of $Ax = \mathbf{b}$ for any arbitrary column vector $\mathbf{b}$. In this part, the following result was discussed.

\begin{Thm}
Let $x_0$ be a vector in $\mathbb{R}^n$ such that $Ax_0 = \mathbf{b}$. Then every solution to $Ax = \mathbf{b}$ can be written in the form $x_0 + x'$, where $Ax' = \mathbf{0}$.
\end{Thm}

\begin{proof}
Consider any $\tilde{x}$ such that $A\tilde{x} = \mathbf{b}$. 
Then, we have
\begin{align}
Ax_0 = \mathbf{b} \quad \mathrm{and} \quad A\tilde{x} = \mathbf{b} \notag\\
\Rightarrow A(\tilde{x} - x_0) = \mathbf{0} \notag
\end{align}

Also $\tilde{x} = x_0 + (\tilde{x} - x_0)$.
Hence proved
\end{proof}

The solution set to $Ax = \mathbf{b}$ looks like a subspace shifted by a vector $x_0$.

\section{Convex Sets}
We look at some of the geometric properties of sets of points in this section. Consider any two points $v_1$and $v_2$. Then the vector $v_1 + k(v_2 - v_1)$ lies on the line segment joining $v_1$ and $v_2$ for $k \in [0,1]$.Rearranging, we can write this as $(1 - k)v_1 + kv_2$, or as $\lambda_1v_1 + \lambda_2v_2$ where $\lambda_1 + \lambda_2 = 1$ and $0 \leq \lambda_1,\lambda_2 \leq 1$. What is interesting, however, is that this generalizes to larger sets as well. If we consider a set of $n$ points $S = \set{v_1,\dots,v_n}$, then any point lying in the polygon with $v_1,\dots,v_n$ as its vertices can be written as $\sum_{i=1}^n \lambda_iv_i$, where $\sum_{i=1}^n\lambda_i = 1$ and $0 \leq \lambda_i \leq 1$.

We now define the term \emph{convex combination}.
\begin{Def}
Given n vectors $v_1,\dots,v_n$, vector $v$ of the form 
\begin{equation}
v = \sum_{i=1}^n\lambda_iv_i, \quad 0 \leq \lambda_i \leq 1,\quad \sum_{i=1}^n\lambda_i = 1
\notag
\end{equation}
is called a \emph{convex combination} of $v_1,\dots,v_n$.
\end{Def}

A \emph{convex set} is defined as:
\begin{Def}
A set of points $S$ is called \emph{convex} if for any subset $S'$ of $S$ and for any point $p$ which we get by convex combination of points in $S'$, $p \in S$.
\end{Def}

As an example the set $\{x~:~Ax \leq \mathbf{b}\}$ is convex. This is because for any $x_1,\dots,x_n$ satisfying $Ax_i \leq \mathbf{b}$, $A(\sum_i\lambda_ix_i) = \sum_i\lambda_iAx_i \leq \sum_i\lambda_i\mathbf{b} = \mathbf{b} ~ \mathrm{as} ~ \sum_i\lambda_i = 1$.

\end{document}