%\documentstyle[twoside]{siamltex}
\documentclass[twoside]{siamltex}
\input{epsf}

\newcommand{\bA}{{\bf A}}
\newcommand{\bB}{{\bf B}}
\newcommand{\bC}{{\bf C}}
\newcommand{\bE}{{\bf E}}
\newcommand{\bF}{{\bf F}}
\newcommand{\bG}{{\bf G}}
\newcommand{\bH}{{\bf H}}
\newcommand{\bI}{{\bf I}}
\newcommand{\bV}{{\bf V}}
\newcommand{\bP}{{\bf P}}
\newcommand{\bM}{{\bf M}}
\newcommand{\bZ}{{\bf Z}}
\newcommand{\bU}{{\bf U}}
\newcommand{\bW}{{\bf W}}
\newcommand{\bQ}{{\bf Q}}
\newcommand{\bR}{{\bf R}}
\newcommand{\bS}{{\bf S}}
\newcommand{\bT}{{\bf T}}
\newcommand{\bX}{{\bf X}}
\newcommand{\bY}{{\bf Y}}
\newcommand{\bx}{{\bf x}}
\newcommand{\bu}{{\bf u}}
\newcommand{\bv}{{\bf v}}
\newcommand{\bw}{{\bf w}}
\newcommand{\bz}{{\bf z}}
\newcommand{\by}{{\bf y}}
\newcommand{\br}{{\bf r}}
\newcommand{\bs}{{\bf s}}
\newcommand{\bh}{{\bf h}}
\newcommand{\be}{{\bf e}}
\newcommand{\bb}{{\bf b}}
\newcommand{\bp}{{\bf p}}
\newcommand{\bff}{{\bf f}}
\newcommand{\bPi}{{\bf \Pi}}

\newcommand{\la}{\langle}
\newcommand{\ra}{\rangle}
\newcommand{\dm}{\begin{displaymath}}
\newcommand{\edm}{\end{displaymath}}
\newcommand{\eq}{\begin{equation}}
\newcommand{\eeq}{\end{equation}}

\title{Block Krylov Iterations\thanks{Sandia is a multiprogram
laboratory operated by Sandia Corporation, a Lockheed Martin
Company, for the United States Department of Energy under Contract
DE-AC04-94AL85000.}}
 \author{T.~L.~Barth\thanks{Sandia National Laboratories, Applied Mathematics and Numerics Department,
 PO Box 5800, MS 1110,
 Albuquerque, NM 87185, {\tt tlbarth@sandia.gov}.}
 \and
M.~A.~Heroux\thanks{Sandia National Laboratories, Applied
Mathematics and Numerics Department, PO Box 5800, MS 1110,
Albuquerque, NM 87185, {\tt mheroux@cs.sandia.gov}.}
 \and
 R.~B.~Lehoucq\thanks{Sandia National Laboratories, Applied Mathematics
and Numerics Department,
 PO Box 5800, MS 1110,
Albuquerque, NM 87185, {\tt rblehou@sandia.gov}.
}
}

\pagestyle{myheadings} \thispagestyle{plain} \markboth{T. L.
BARTH, M. B. HEROUX, AND R. B. LEHOUCQ}{Block Krylov
iterations--DRAFT}

\begin{document}

\maketitle

\begin{abstract}
We present block Krylov iterations for the solution of a linear
system of equations with an arbitrary number of right-hand sides.
Our interest is threefold.  First, we seek to improve the data
locality of standard Krylov subspace iterations, which applies the
coefficient matrix to only one vector per iteration. By improving
the data locality, our block Krylov algorithms better utilizes the
deep memory hierachy present on microprocessor-based distributed
memory computers.  Secondly, we want a robust algorithm for the
solution of difficult linear systems (such as those arising from
multi-physics simulations) where the number of standard Krylov
iterations needed proves to be excessive.  Thirdly, independent of
performance issues, we want to reduce the total amount of serial
work required to solve a linear system with multiple right hand
sides. Numerical experiments substantiate our claims.

\end{abstract}

\begin{keywords}
GMRES, CG, Block Krylov methods, Arnoldi method, Linear systems
\end{keywords}
\begin{AMS}

\end{AMS}

\section{Introduction}
\label{intro}

We are interested in block Krylov iterations for solving
 \eq
 \bA \bX = \bB,
 \label{eq:blocklinsys}
 \eeq
where, $\bA \in {\cal C}^{n \times n}$, and $\bB = [\bb_1, \ldots,
\bb_p] \in {\cal C}^{n \times p}$ is a matrix of $p$ right-hand
sides. For us, the {\em block} size $s$ (the number of right-hand
sides to be solved for simultaneously inside the block Krylov
solver) is independent of the number of right-hand sides $p$. In
particular, we are interested in formulating block Krylov
iterations that can solve linear systems with any number of
right-hand sides, using any {\em block} size. For example, this
would allow a system with one ($p=1$) right-hand side to be solved
using a block implementation.

Like their single right-hand side counterparts, block Krylov
methods produce a sequence of block iterates,
  \dm
 \bX^{(i+1)} = \left[ \bx_1^{(i+1)}, \ldots, \bx_s^{(i+1)}
 \right] \in {\cal C}^{n \times s},
 \edm
 of the form
  \dm
  \bX^{(i+1)} = \bX^{(0)} + \bZ^{(i)},
  \edm
 where $\bZ^{(i)}$ is chosen from the block Krylov subspace of
 dimension $i+1$,
  \dm
 {\cal K}_{i+1}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA
 \bR^{(0)},\ldots,\bA^{i}\bR^{(0)}\},
 \edm
and, $\bR^{(0)} = \left[ \br_1^{(0)}, \ldots, \br_s^{(0)}
\right]$, denotes the matrix of $s$ initial residuals. These
methods generally choose $\bZ^{(i)} \in {\cal
K}_{i+1}(\bR^{(0)},\bA)$ so that some orthogonality or block
minimization condition is satisfied.

Block generalizations of Krylov methods are implemented by
constructing a block-wise orthogonal basis for the underlying
block Krylov subspaces. At step {\small $k = ceil(n/s)$}, the
block Krylov subspace ${\cal K}_{k}(\bR^{(0)}, \bA)$ spans the
entire $n$-dimensional space, and the orthogonalization process
will terminate. At each step, this process involves $s$ (the block
size used by the solver) matrix vector multiplications. Here, the
matrix $\bA$ can be applied to $s$ vectors all at once. For users
who have multiple right-hand systems to solve, a block
implementation may provide significant savings in the work
required. For example, to solve for $s$ right-hand sides
individually using a single right-hand side Krylov subspace
implementation would require at most (in exact arithmetic) $s
\times n$ steps, and $s \times n$ matrix vector multiplications.
To solve them using a block implementation with a block size of
$s$ would require at most $ceil(n/s)$ steps, and roughly $n$
matrix vector multiplications (applied $s$ at a time). Although
real world application problems utilize preconditioning techniques
and generally (to be useful) converge to within some acceptable
tolerance level in much fewer iterations than our exact arithmetic
analysis bounds yield, the goal is to produce block
implementations for solving (\ref{eq:blocklinsys}) that converge
much faster that it would take to solve the $s$ linear systems
consecutively using a single right-hand side implementation. Much
work is still needed to determine the true savings that block
implementations might provide. Any such study should also factor
in the details of the particular block implementation, as these
can make a significant impact on the actual convergence behavior
in finite precision arithmetic.

We are interested in impacting three user groups of iterative
linear solver technologies. The first group has coefficient
matrices $\bA$ that are expensive to apply. This expense might
arise from the cost of communication when computing a matrix
vector product with $\bA$ on a distributed memory parallel
computer, or the memory latency that exists on floating point rich
code that often exhibits poor data locality, or both. Because, in
a block implementation, the coefficient matrix $\bA$ is applied to
several vectors at a time, a significant benefit results for this
user group. The second group consists of users with extremely
difficult linear systems (poorly conditioned) that typically arise
from high-fidelity multi-physics simulations. It was pointed out
in \cite{olea:80} that a block conjugate gradient (CG)
implementation was more robust that a single CG implementation for
symmetric positive definite (SPD) matrices containing several
extreme eigenvalues widely separated from the others. The third
group is users who have multiple simultaneous right-hand sides to
solve. The above discussion using exact arithmetic bounds suggests
that a block Krylov implementation may be able to find the
solution to all right-hand sides in a fraction of the total number
of iterations required to solve for each right-hand side
separately.

Block generalizations of Krylov subspace methods are not new,
however, good robust implementations are needed in order to make
them a viable methodology for solving the large linear systems of
equations with one or more right-hand sides that arise from
today's challenging application problems. In this article, we
study block implementations of the well known conjugate gradient
(CG) \cite{HS52} and generalized minimal residual
(GMRES)\cite{SS86} algorithms for solving (\ref{eq:blocklinsys}).
The article is organized as follows.  In Section's 2 and 3, we
outline the details of our block CG and GMRES implementations. In
the remaining sections, we perform a series of numerical
experiments to assess the differences between the blocked and
unblocked variants.

\subsection{Notation and Definitions}
\label{notation_and_terminology}

This section establishes the basic notation to be used in this
article. We employ Householder notational conventions. Capital and
lower-case letters denote matrices and vectors, respectively,
while lower-case Greek letters denote scalars. In addition,
bracketed lower-case Greek letters denote coefficient matrices.
The transpose of a vector ${\bf x}$ is denoted by ${\bf x}^T$, and
the complex conjugate of ${\bf x}^T$ is denoted by ${\bf x}^H.$
Similarly, the matrix $\bV^H = {\bar \bV}^T$. The norm used is the
Euclidean one denoted by $\| \cdot \|_2.$

The order of $\bA$ will always be denoted by $n.$ The identity
matrix of order $m$ is denoted by ${\bf I}_{(m,m)}.$ The $m \times
t$ zero matrix is given by ${\bf 0}_{(m,t)}$. The block size used
by the solvers is $s$. The $j$'th canonical basis vector is
denoted by ${\bf e}_j$, the $j$th column of the identity matrix,
and
 $\bE_j \equiv \left[\begin{array}{ccc}
                    \be_{(j-1)s+1} & \cdots & \be_{js}
                   \end{array}\right] ,$
where $s$ (the block size) is a positive integer.

Superscripts on matrices correspond to iteration numbers. For
example, $\bX^{(i)}$ and $\bR^{(i)}$, denote the $i$'th blocks of
iterates and residuals (each containing $n$ rows and $s$ columns),
respectively, produced by the iteration. The matrix $\bP^{(i)}$
denotes the $i$'th block of direction vectors associated with the
block CG iteration, and the matrices $\bF^{(i)}$ and $\bU^{(i)}$
correspond to the $i$'th blocks of basis vectors associated with
the block GMRES iteration.

A matrix of lower bandwidth $s$ will be called a banded upper
Hessenberg matrix.  We drop ``upper'' when the context is clear.
Omission of the word {\em band} implies that the block size is
one. We say that a band Hessenberg matrix is unreduced if all the
elements on the $s$'th subdiagonal are nonzero.

We now define several matrices used in our presentation of block
GMRES. $\bH_j$ denotes a band Hessenberg matrix of order $sj$ of
lower bandwidth $s$. $\bV_j$ denotes a matrix with $n$ rows and
$sj$ columns. $\bU^{(j)}$ denotes the $j$'th block of $s$ vectors
of $\bV_m$, and $\bG_{i,j}$ denotes the square matrix of order $s$
located in the $i,j$'th block of order $s$ of $\bH_m.$ Note that
$\bG_{j+1,j}$ is an upper triangular matrix. These matrices will
define the dimensions of other matrices used in this article.

The remaining notation will be defined as needed, or will be clear
from the context.


\section{Block CG}
\label{sec:blkcg}

Block generalizations of the well known CG algorithm \cite{HS52}
for solving linear systems involving an SPD coefficient matrix
$\bA$ are not new. For example, they have been studied in
\cite{olea:80}, and more recently in (\cite{FOP95}, \cite{NY95},
and \cite{DUB01}).

Analogous to the common single CG method, the block CG method for
solving the block linear system of equations
(\ref{eq:blocklinsys}) (with an SPD system matrix $\bA$) is
implemented via the construction of a block $\bA$-orthogonal basis
$\{ \bP^{(0)}, \bP^{(1)}, \ldots, \bP^{(i)} \}$ for the underlying
block Krylov subspaces
 \dm
 {\cal K}_{i+1}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA
 \bR^{(0)},\ldots,\bA^{i}\bR^{(0)}\}= {\rm sp}\{ \bP^{(0)},
 \bP^{(1)}, \ldots, \bP^{(i)} \},
 \edm
 where,
 \dm
 \bP^{(j)T} \bA \bP^{(k)} = {\bf 0}_{(s,s)}, \quad {\rm
 if}~j \ne k.
 \edm
At each step $i$, the block CG iterates,
 \dm
 \bX^{(i+1)} = \bX^{(i)} + \bZ^{(i)}
 \edm
are defined by choosing $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$ such that $\bE r^{(i+1)} \perp_{\bA} {\cal
K}_{i+1}(\bR^{(0)}, \bA)$, where $\bE r^{(i+1)}$ refers to the
error at step $i+1$ and $\perp_\bA$ means orthogonality in the
$\bA$ inner product \cite{AMS90}. An implementation of the block
CG method is given in Figure \ref{fig:blkomin} below.

\begin{figure}[hbt]
\vspace{.05in} \hrule \vspace{.1in}
\begin{description}
\item[Initialization]~~

\begin{itemize}
 \item Given $\bX^{(0)}, ~\bR^{(0)} = \bB - \bA \bX^{(0)}$ ~~Initial Guess,
 Initial Residual Blocks
 \item Set~ $\bP^{(0)} = \bR^{(0)}$ ~~Initial Direction Vector Block
 \end{itemize}

\item[For $i = 0,1,...,$ until convergence, compute:]~~

\begin{enumerate}

\item $[\alpha_i] = ( {\bP}^{(i)T} \bA {\bP}^{(i)})^{-1} ( {\bP}^{(i)T}
\bR^{(i)})$
\item $\bX^{(i+1)} = \bX^{(i)} + {\bP}^{(i)} [\alpha_i]$
\item $\bR^{(i+1)} = \bR^{(i)} - \bA {\bP}^{(i)} [\alpha_i]$
\item $[\beta_i] = -( {\bP}^{(i)T} \bA {\bP}^{(i)})^{-1} ( {\bP}^{(i)T} \bA \bR^{(i+1)})$
\item $\bP^{(i+1)} = \bR^{(i+1)} + {\bP}^{(i)} [\beta_i]$


\end{enumerate}

\end{description}

\vspace{.1in} \hrule
 \caption{Block CG implementation}
 \label{fig:blkomin}
\end{figure}

The blocks of direction and residual vectors produced by the block
CG algorithm satisfy the following block orthogonality properties
(see for example \cite{olea:80}):

 \eq
 \begin{array}{rcl}
  \bP^{(j)T} \bA \bP^{(k)} = {\bf 0}_{(s,s)}, \qquad j &\ne& k, \\
  \bR^{(j)T} \bR^{(k)} = {\bf 0}_{(s,s)}, \qquad j &\ne& k, \\
  \bP^{(j)T} \bR^{(k)} = {\bf 0}_{(s,s)}, \qquad j &<& k.
  \end{array}
  \label{eq:cgorth}
  \eeq

The orthogonality properties of the method imply that at most
$ceil(n/s)$ blocks of linearly independent direction and residual
vectors exist. Thus, the $s \times s$ coefficient matrices
$[\alpha_i]$ and $[\beta_i]$ may become singular at iteration
counts beyond $n/s$. For now, we assume that the $\bP^{(j)}$'s and
$\bR^{(j)}$'s have full rank. The coefficient matrices,
$[\alpha_i]$ and $[\beta_i]$ given in Figure \ref{fig:blkomin} can
be written using different formulas. For example,
 \dm
 \begin{array}{rcl}
  \left[ \alpha_i \right] &=& ( \bP^{(i)T} \bA \bP^{(i)} )^{-1} ( \bR^{(i)T}
  \bR^{(i)} ), \\
  \left[ \beta_i \right] &=& ( \bR^{(i)T} \bR^{(i)} )^{-1} ( \bR^{(i+1)T}
  \bR^{(i+1)} ).
  \end{array}
 \edm
With these formulas, we obtain the more common implementation of
the block CG method studied in \cite{olea:80}. When $s=1$, this
yields the well known single vector CG implementation. That the
coefficients can be written using these different formulas was
proven in \cite{HS52} for the single right-hand side CG
implementation. The formulas for the block implementation can be
derived using the orthogonality properties of the CG method given
in (\ref{eq:cgorth}) above.

Like the common block CG algorithm, this implementation of block
CG can breakdown during the computation of the coefficient
matrices $[\alpha_i]$ and $[\beta_i]$ if $(\bP^{(i)T} \bA
\bP^{(i)})$ becomes singular at some step during the iteration.
This happens if the columns of the matrix $\bP^{(i)}$ become
linearly dependent at some step in the iteration. We note that
with this formulation of the block CG algorithm, the quantity
$(\bR^{(i)T} \bR^{(i)})^{-1}$ does not appear explicitly in the
computation of the coefficient matrix $[\beta_i]$. Thus, this
formulation may be advantageous numerically since we only need to
be concerned explicitly with the rank of one quantity.

In \cite{olea:80} it was shown that the ranks of the direction
vector blocks $\bP^{(j)}$ and the residual blocks $\bR^{(j)}$ are
the same for every $j$. It follows that the linear independence
(dependence) of these blocks can be monitored via an
orthogonalization procedure involving just the direction vector
blocks. Once a new block of direction vectors are computed,
dependent columns can be detected via a QR factorization
procedure,
  \dm
  \bP^{(i+1)} = {\hat \bP}^{(i+1)} \bG_{i+1}, \qquad {\hat \bP}^{(i+1)T}{\hat
  \bP}^{(i+1)}= I_{(s,s)}.
  \edm
In an analogous manner to that mentioned in \cite{olea:80}, a
deflation procedure could be incorporated into the algorithm given
in Figure \ref{fig:blkomin} to prevent breakdowns.

Since the columns of ${\hat \bP}^{(i+1)}$ are just linear
combinations of those in $\bP^{(i+1)}$, it follows that
 \dm
  {\hat \bP}^{(j)T} \bA {\hat \bP}^{(k)} = {\bf 0}_{(s,s)}, \qquad {\rm for}~ j
  \ne k,
  \edm
and the ${\hat \bP}^{(j)}$'s also form a block-wise
$\bA$-orthogonal basis for the underlying block Krylov subspaces.
Thus, the block CG implementation given in Figure
\ref{fig:blkomin} can be easily reformulated in terms of the
${\hat \bP}^{(j)}$'s. If we are already computing the ${\hat
\bP}^{(j)}$'s as a result of an orthogonalization process to
detect linear dependencies, no additional work results from this
formulation. By substituting $\bP^{(i+1)} = {\hat \bP}^{(i+1)}
\bG_{i+1}$ into the algorithm given in Figure \ref{fig:blkomin},
and some algebra, we obtain a version of the algorithm that is
written in terms of the ${\hat \bP}^{(j)}$'s, which is detailed in
Figure \ref{fig:blkomin_qr}. The ${\hat \bP}^{(j)}$'s and the
corresponding $\bR^{(j)}$'s produced by this formulation of block
CG satisfy the same block orthogonality conditions
(\ref{eq:cgorth}).


\begin{figure}[hbt]
\vspace{.05in} \hrule \vspace{.1in}
\begin{description}
\item[Initialization]~~

\begin{itemize}
 \item Given $\bX^{(0)}, ~\bR^{(0)} = \bB - \bA \bX^{(0)}$ ~~Initial Guess,
 Initial Residual Blocks
 \item Set~ $\bP^{(0)} = \bR^{(0)}$ ~~Initial Direction Vector Block
 \item Compute ~ $\bP^{(0)} = {\hat \bP}^{(0)T} \bG_{0}$ ~~QR
 Factorization
 \end{itemize}

\item[For $i = 0,1,...,$ until convergence, compute:]~~

\begin{enumerate}

\item $[\alpha_i] = ( {\hat \bP}^{(i)T} \bA {\hat \bP}^{(i)})^{-1} ( {\hat \bP}^{(i)T}
\bR^{(i)})$
\item $\bX^{(i+1)} = \bX^{(i)} + {\hat \bP}^{(i)} [\alpha_i]$
\item $\bR^{(i+1)} = \bR^{(i)} - \bA {\hat \bP}^{(i)} [\alpha_i]$
\item $[\beta_i] = -( {\hat \bP}^{(i)T} \bA {\hat \bP}^{(i)})^{-1} ( {\hat \bP}^{(i)T} \bA \bR^{(i+1)})$
\item $\bP^{(i+1)} = \bR^{(i+1)} + {\hat \bP}^{(i)} [\beta_i]$
\item $\bP^{(i+1)} = {\hat \bP}^{(i+1)T} \bG_{i+1}$ ~~ QR
factorization

\end{enumerate}

\end{description}

\vspace{.1in} \hrule
 \caption{Block CG implementation with QR factorization of
 direction vector blocks} \label{fig:blkomin_qr}
\end{figure}

As long as the $\bP^{(j)}$'s have full rank, the columns of the
${\hat \bP}^{(j)}$'s are nonzero, and orthonormal. Thus, we would
expect that the systems to be solved for the coefficient matrices
$[\alpha_j]$ and $[\beta_j]$ in the algorithm given in Figure
\ref{fig:blkomin_qr} to be better conditioned that those in Figure
\ref{fig:blkomin}. If the $\bP^{(j)}$'s lose full rank at some
step, a deflation procedure can also be incorporated into the
algorithm given in Figure \ref{fig:blkomin_qr} to prevent
numerical breakdowns.

Our block CG implementation follows the algorithm given in Figure
\ref{fig:blkomin_qr}, together with a deflation mechanism to
prevent numerical breakdowns. We describe the details of our
implementation in Section \ref{sec:belosblkcg} below.

Our implementation is similar to one presented in \cite{DUB01}. In
\cite{DUB01} they explored various modifications of the block CG
algorithm, which included an approach that used the basic
formulation given in Figure \ref{fig:blkomin_qr}. Their approach
differs from our implementation in that instead of a deflation
mechanism, they use a procedure that utilizes a change of basis to
supplement rank defects. In addition, they explored an
orthogonalization process that enforced $\bA$-orthogonality
between the individual vectors within the direction vector blocks.
In this case, a simplification occurs when computing the
coefficient matrices $[\alpha_i]$ and $[\beta_i]$. This
orthogonalization process may be more expensive, and for large
block sizes, loss of $\bA$-orthogonality could become an issue in
the implementation.

\subsection{Block CG Implementation}
\label{sec:belosblkcg}

A deflation mechanism is incorporated into our block CG
implementation to prevent breakdowns that can occur during the
iteration. This is accomplished by detecting and then dropping
both dependent direction vectors and residuals that are very small
in norm from the corresponding blocks, and then continuing the
iteration by enforcing the orthogonality properties of the block
CG method on the deflated block system. To avoid physically
changing the block sizes during the iteration, this process is
realized by an indexing mechanism. Only the direction vectors,
residual vectors, and iterates within the blocks that correspond
to certain indices are updated. The algorithm is essentially the
same as that outlined in Figure \ref{fig:blkomin_qr} except that
in the case that deflation has occurred, the computations are
actually carried out using subsets of the $s$ columns in the $n
\times s$ matrices of direction, residual, and iterates.

Three sets of indices are used in this implementation. The
independent indices, $[iidx]$ and $[piidx]$, are the indices of
the linearly independent columns of the current and previous
direction vector blocks, respectively. The quantity $numind$
refers to the number of independent vectors in the current
direction vector block. The current indices, $[ridx]$, are the
indices corresponding to the nonzero residuals (those residuals
with residual norm greater than very some small tolerance,
$rtol$), in the current residual block, and {\it numcur} refers to
the number of these indices.  As the name implies, the converged
indices, $[cidx]$, are the indices in the current residual block
that correspond to residuals that have converged according to some
user specified residual tolerance, $ctol$, and $numconv$ refers to
the number of converged residuals. For proper execution of the
block algorithm, $ctol$ must be chosen to be greater than or equal
to $rtol$. Thus, $[ridx]$ and $[cidx]$ can be different sets. This
distinction was made for numerical reasons, since residuals that
have converged according to some user specified tolerance may
still be large enough to be used to generate new linearly
independent direction vectors, and thus, further assist in the
convergence of the block linear system.

Let ${\hat \bP}^{(i)}_{[piidx]}$ denote the columns of the $n
\times s$ matrix of direction vectors ${\hat \bP}^{(i)}$ that are
linearly independent from the previous step. Let
$\bR^{(i)}_{[ridx]}$ denotes those residuals that are greater than
$rtol$ in norm, and $\bX_{[ridx]}^{(i)}$, the corresponding
iterates from the previous step. Only the iterates and residuals
corresponding to the indices $[ridx]$ will be updated at the
current step. Below, we detail the steps in the block CG
implementation given in Figure \ref{fig:blkomin_qr}.

The quantity, $\bW = \bA {\hat \bP}_{[piidx]}^{(i)}$, which is
used in the computation of $[\alpha_i]$ and $\bR_{[ridx]}^{(i+1)}$
in Steps 1 and 3, is computed by applying $\bA$ to the group of
vectors ${\hat \bP}_{[piidx]}^{(i)}$ all at once. This can be an
important feature to users with matrices $\bA$ that are expensive
to access.

The new iterates $\bX^{(i+1)}_{[ridx]}$ are constructed such that
 \dm
  \bE r_{[ridx]}^{(i+1)}  \perp_{\bA}  {\cal K}_{i+1} (\bR^{(0)}, \bA) \\
  ~~~~~{\rm or~equivalently}~~ \\
  \bR_{[ridx]}^{(i+1)}  \perp  {\cal K}_{i+1} (\bR^{(0)}, \bA).
  \edm
Since ${\hat \bP}^{(i)}_{[piidx]} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$, the expression for $[\alpha_i]$ given in Step 1 follows
from enforcing
 \dm
 [ {\bf 0} ] = {\hat \bP}^{(i)T}_{[piidx]} \bR^{(i+1)}_{[ridx]} =
   {\hat \bP}^{(i)T}_{[piidx]} \bR^{(i)}_{[ridx]}
 - ( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \alpha_i ].
 \edm
The coefficient matrix $[\alpha_i]$ is actually computed by
solving the block linear system
 \dm
 \left( {\hat \bP}^{(i)T}_{[piidx]} \bW \right)
 [ \alpha_i ] = {\hat \bP}^{(i)T}_{[piidx]} \bR^{(i)}_{[ridx]}, ~~
 {\rm where}~~\bW = \bA {\hat \bP}_{[piidx]}^{(i)}.
 \edm
Note that $[\alpha_i]$ has dimensions $numind \times numcur$.
Thus, in the case that deflation has occurred at a previous step,
$[\alpha_i]$ is no longer an $s \times s$ matrix. Since $\bA$ is
SPD, $[\alpha_i]$ will be well defined as long as ${\hat
\bP}^{(i)}_{[piidx]}$ has full column rank, which will be the case
as long as the procedure for identifying and then dropping
dependent direction vectors is working properly. The quantities,
$\left( {\hat \bP}_{[piidx]}^{(i)T} \bW \right)$ and $\left( {\hat
\bP}_{[piidx]}^{(i)T} \bR_{[ridx]}^{(i)} \right)$, are computed
using Level 3 {\small BLAS}~\cite{dddh:90} matrix-matrix
multiplication subroutine {\tt \_GEMM}. Assuming that $\bA$ is
SPD, and ${\hat \bP}^{(i)}_{[piidx]}$ has full column rank, it
follows that $( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat
\bP}^{(i)}_{[piidx]})$ is also SPD, and the above system can be
solved via a Cholesky factorization of the system matrix, and
subsequent forward and back solves for the columns of
$[\alpha_i]$. The updates of the iterates and residuals in Steps 2
and 3 are then obtained by computing
 \dm
 \begin{array}{rcl}
  \bX_{[ridx]}^{(i+1)} &=& \bX_{[ridx]}^{(i)} + {\hat \bP}_{[piidx]}^{(i)}
  [\alpha_i],
  \\ \\
  \bR_{[ridx]}^{(i+1)} &=& \bR_{[ridx]}^{(i)} - \bA {\hat \bP}_{[piidx]}^{(i)}
  [\alpha_i].
  \end{array}
 \edm

Once the new residual block $\bR^{(i+1)}_{[ridx]}$ has been
computed in Step 3, we check for convergence of the individual
residuals. The column indices of converged residuals (those that
are smaller in norm than the user specified tolerance $ctol$) are
added to the set of converged residual indices $[cidx]$, and the
$numconv$ is adjusted accordingly. The iteration will terminate if
all residuals in the block have converged.

This implementation of block CG uses the residuals $\bR^{(i+1)}$
to bring the iteration up to the next higher dimensional Krylov
subspace ${\cal K}_{i+2}(\bR^{(0)}, \bA)$. If a column of
$\bR^{(i+1)}$ is very small in norm, so will be the corresponding
column of $\bP^{(i+1)}$. So, after computing the new block of
residuals in Step 3, we can delete the indices corresponding to
residuals whose norm is less than $rtol$ from both the sets
$[ridx]$ and $[iidx]$, and adjust the quantities $numcur$ and
$numind$. This information will no longer be useful for creating
new linearly independent direction vectors, and thus, reducing the
residual norms further.

The residuals corresponding to the linearly independent columns of
direction vectors, $\bR^{(i+1)}_{[iidx]}$, are used to compute
$\bP^{(i+1)}_{[iidx]}$ in Step 5.  The coefficient matrix
$[\beta_i]$ is chosen to enforce $\bA$-orthogonality between the
new and previous direction vector blocks. Since ${\hat
\bP}^{(i)}_{[piidx]} \in {\cal K}_{i+1}(\bR^{(0)}, \bA)$, this can
be accomplished by solving
 \dm
 [ {\bf 0} ] = {\hat \bP}^{(i)T}_{[piidx]} \bA \bP^{(i+1)}_{[iidx]} =
   {\hat \bP}^{(i)T}_{[piidx]} \bA \bR^{(i+1)}_{[iidx]}
  + ( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \beta_i ]
 \edm
for $[\beta_i]$. Like the coefficient matrix $[\alpha_i]$, this is
actually computed by solving the block linear system
 \dm
  {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \beta_i ] =  -{\hat \bP}^{(i)T}_{[piidx]} \bA \bR^{(i+1)}_{[iidx]}.
 \edm
Since $\bA$ is SPD, we can write
 \dm
  -{\hat \bP}^{(i)T}_{[piidx]} \bA \bR^{(i+1)}_{[iidx]} = - \left[
  \bA {\hat \bP}_{[piidx]}^{(i)} \right]^T \bR_{[iidx]}^{(i+1)} =
  -\bW^T \bR_{[iidx]}^{(i+1)},
  \edm
which can be computed via Level 3 {\small BLAS}~\cite{dddh:90}
matrix-matrix multiplication subroutine {\tt \_GEMM}. Note that
the Cholesky factorization of this system matrix has already been
computed during the computation of $[\alpha_i]$, thus, only
forward and back solves are required to obtain the columns of
$[\beta_i]$.

The direction vector block is then updated according to
 \dm
 \bP_{[iidx]}^{(i+1)} = \bR_{[iidx]}^{(i+1)} + {\hat
 \bP}_{[piidx]}^{(i)} [\beta_i].
 \edm

In Step 6, a QR factorization is used to obtain an orthonormal set
${\hat \bP}_{[iidx]}^{(i+1)}$ from $\bP^{(i+1)}_{[iidx]}$. This
procedure is carried out via an iterated classical Gram-Schmidt
(CGS) process. A second orthogonalization step is done to ensure
orthogonality. One benefit of the CGS process is that it allows
the use of the Level 2 {\small BLAS}~\cite{ddhh:88} matrix-vector
multiplication subroutine {\tt \_GEMV}. After constructing column
$j$ of ${\hat \bP}_{[iidx]}^{(i+1)}$ by this process, the sine of
the angle between $\bp_j^{(i+1)}$ (the $j$'th column of
$\bP_{[iidx]}^{(i+1)}$) and the space spanned by the previous
$j-1$ columns of ${\hat \bP}_{[iidx]}^{(i+1)}$ is measured. If
this angle is very small, we conclude that $\bp_j^{(i+1)}$ is
dependent on the previous $j-1$ columns ${\hat
\bP}_{[iidx]}^{(i+1)}$. The column indices corresponding to
dependent vectors are dropped from the set $[iidx]$ and the
quantity $numind$ is updated. Finally, we set $[piidx] = [iidx]$,
and continue to the next step of the iteration.

Our implementation handles the case when the initial residuals
have converged or are linearly dependent. In this case, the
indices of the blocks are adjusted in a similar manner as outlined
above (Steps 1 - 6) for a general step in the iteration. It may be
desirable however, to check for these conditions before calling
the block solver routine, because, otherwise, the solver will
start the iteration effectively using a reduced block size.

\section{Block GMRES}
\label{sec:blkgmres}

The block GMRES method for solving the block linear system of
equations (\ref{eq:blocklinsys}) is implemented via the
construction of an orthonormal basis
 \dm
 \{ \bU^{(k)} \}_{k=1}^{i+1}
 \edm
for the underlying block Krylov subspaces
  \dm
 {\cal K}_{i+1}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA
 \bR^{(0)},\ldots,\bA^{i}\bR^{(0)}\}= {\rm sp}\{ \bU^{(1)},
 \ldots, \bU^{(i+1)} \},
 \edm
where,
 \dm
 \begin{array}{rcl}
 \bU^{(j)H} \bU^{(k)} &=& {\bf 0}_{(s,s)}, \quad {\rm
 if}~j \ne k, \\
 \bU^{(j)H} \bU^{(j)} &=& \bI_{(s,s)}.
 \end{array}
 \edm
At each step, the block GMRES iterates
 \dm
 \bX^{(i+1)} = \bX^{(0)} + \bZ^{(i)}
 \edm
are defined by choosing $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$ such that the 2-norm of the individual columns of the block
residual $\bR^{(i+1)}$ are minimized.

There are two main steps in our block GMRES implementation:
\begin{enumerate}
\item Constructing an orthonormal basis for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$
\item Defining the iterates $\bX^{(i+1)}$ via a block minimization
of the residual norms.
\end{enumerate}
In Section's \ref{sec:blkarn} and \ref{sec:belosblkgmres} we will
describe these processes, and give the details of our block GMRES
implementation.

Our approach is in direct contrast to the s-step
methods~\cite{chro:91,chki:92,li:97} used for solving with one
right-hand side. These methods are unstable (for increasing $p$)
and only block the construction of the Hessenberg matrix.
Moreover, as the cost of applying the matrix increases relative to
the cost of the orthogonalizations, s-step methods produce
diminishing returns.

Block GMRES methods were considered in~\cite{siga:95,siga:96}, but
the implementations required that the number of right-hand sides
$p$ in the block linear system be the same as the block size $s$.
These studies indicated that block GMRES implementations were not
efficient due to their prohibitive memory and computational
requirements. We will show that by formulating the construction of
the orthogonal basis for ${\cal K}_{i+1}(\bR^{(0)}, \bA)$ in terms
of level 3 BLAS, the computational requirements are not excessive.
Moreover, unlike s-step methods, a robust and stable algorithm is
realized where the application of $\bA$ is to a matrix. Thus, as
the cost of applying $\bA$ increases relative to the cost of the
orthogonalizations, the efficiency our block GMRES algorithm will
scale with the cost of $\bA \bR^{(0)}$.


\subsection{Consturcting an Orthonormal Basis} \label{sec:blkarn}

An orthonormal basis for the block Krylov subspaces
 \dm
 {\cal K}_{i+1}(\bR^{(0)}, \bA) = {\rm sp} \{ \bR^{(0)}, \bA
 \bR^{(0)}, \ldots, \bA^i \bR^{(0)} \}
 \edm
can be constructed via a block Arnoldi process.

 Let $\bA $ be a matrix of order $n$ and $s > 0$ be the block size.
 We say that
 \begin{equation}
 \bA \bV_m = \bV_m \bH_m + \bF^{(m)} \bE^T_m \label{eq:blk_arn_red}
 \end{equation}
is a block Arnoldi reduction of length $m$ when
$\bV_m^H\bA\bV_m=\bH_m$ is a banded upper Hessenberg matrix,
$\bV_m^H\bV_m={\bf I}_{(m s,m s)}$, and $\bV_m^H\bF^{(m)}={\bf
0}_{(ms,s)}.$ Let $\bU^{(m+1)} \bG_{m+1,m}$ denote the QR
factorization of $\bF^{(m)}.$ We have
\begin{eqnarray*}
\bA \, \bV_m
     &=& \left[\begin{array}{ccc}
         \bU^{(1)} & \cdots & \bU^{(m)} \end{array}\right]
    \left[\begin{array}{cccc}
      \bG_{1,1}&\cdots& \cdots & \bG_{1,m}\\
      \bG_{2,1}&  \ddots& \vdots & \vdots \\
       \vdots & \ddots & \vdots & \vdots \\
     {\bf 0}& \cdots &  \bG_{m,m-1} & \bG_{m,m} \end{array}\right]\\
    & & + \bU^{(m+1)} \bG_{m+1,m}\bE_m^T. \\ \label{block_not} \nonumber
\end{eqnarray*}
or equivalently,
 \dm
 \begin{array}{rcl}
 \bA \bV_m &=& \left[ \bV_m ~~ \bU^{(m+1)} \right] \left[
 \begin{array}{c}
 \bH_m \\
 \bG_{m+1,m} \end{array} \right] \\ \\
 ~&=& \bV_{m+1} {\widetilde \bH}_m,
 \end{array}
 \edm
where, the columns of $\bV_m$ form an orthonormal basis for the
block Krylov subspace ${\cal K}_m ( \bR^{(0)}, \bA)$.

If $ m > \bar{m} \equiv \mbox{ceiling}(n/s)$, then $\bF^{(m)} =
{\bf 0}_{(n,s)}$ and $\bH_{\bar{m}}$ is the orthogonal reduction
of $\bA$ into banded upper Hessenberg form.  We assume, for the
moment, that $\bF^{(m)}$ is of full rank and the diagonal elements
of $\bG_{m+1,m}$ are positive. Thus, a straightforward extension
of the implicit Q theorem \cite{govl:96} gives that $\bF^{(m)}$ is
(uniquely) specified by the starting block $\bU^{(1)}.$ Note that
if $\bA = \bA^H$, then $\bH_m$ is a block tridiagonal matrix.
Figure~\ref{fig:blk_arn_red_alg} describes the process for
extending a block Arnoldi reduction, thus obtaining an orthonormal
basis for the next higher dimensional block Krylov subspace.
Below, we detail the steps of this process.

{\small
\begin{figure}
\vspace{.05in} \hrule \vspace{.1in}
\begin{description}
\item[Start] ~~
Let $\bA \bV_m = \bV_m \bH_m + \bF^{(m)} \bE^T_m$ be a length-$m$
block Arnoldi reduction from the previous step, where $\bV_m^H
\bF^{(m)}= {\bf 0}_{(ms,s)}$,~$\bU^{(m+1)} \bG_{m+1,m} =
\bF^{(m)}$, and $ \bV_{m+1} = \left[ \bV_m ~~ \bU^{(m+1)}
\right]$.

\item[Compute:] ~~

\begin{enumerate}

\item $ \bW = \bA \bU^{(m+1)} $.

\item If $depflg = false$:
\begin{enumerate}
\item $ \begin{array}{rcl} \bF^{(m+1)} &=& \bW - \bV_{m+1} \left[
\bV_{m+1}^H
\bW
\right] \\
~&=& \bW - \left[ \bV_m ~~ \bU^{(m+1)} \right] \left[
\begin{array}{c} \bV_m^H \bW \\
                 \bG_{m+1,m+1}  \end{array} \right],
\end{array} $ \\
where, $\bG_{m+1,m+1} = \bU^{(m+1)H} \bW. $

\item Check for dependencies between $\bW$ and $\bV_{m+1}$.
\begin{itemize}

\item If no dependencies exist, compute
\begin{itemize}
\item $ \bF^{(m+1)} = \bU^{(m+2)} \bG_{m+2,m+1}$
     \begin{itemize}

      \item If the columns of $\bF^{(m+1)}$ are linearly
      independent, set \\

      $\bH_{m+1} = \left[\begin{array}{cc}
                             \bH_m & \bV_m^H \bW \\
                             \bG_{m+1,m} \bE_m^T & \bG_{m+1,m+1}
                        \end{array}\right]$, \\
                         ~~~~~~${\widetilde \bH}_{m+1} = \left[
                        \begin{array}{c} \bH_{m+1} \\
             \bG_{m+2,m+1} \end{array} \right] $, and \\
      $\bV_{m+2} = \left[ \bV_{m+1} ~~ \bU^{(m+2)} \right]$.


      \item If the columns of $\bF^{(m+1)}$ are linearly
      dependent, set $depflg = true$, and recompute $\bF^{(m+1)}$ using step 3.

      \end{itemize}
\end{itemize}

\item If dependencies exist
\begin{itemize}
\item Set $depflg = true$, recompute $\bF^{(m+1)}$
      using step 3 below.
\end{itemize}

\end{itemize}

\end{enumerate}

\item If $depflg = true$: \\
Set $\bV_{prev} = \bV_{m+1}$ ~(all previous Arnoldi vectors) \\
For $j = 1, \ldots, s$,~~Compute:
\begin{enumerate}

\item ${\bf f}_j^{(m+1)} = \bw_j - \bV_{prev} \left[ \bV_{prev}^H \bw_j \right]$.
\item Set the $(ms+j)$'th column of ${\widetilde \bH}_{m+1}$ to
$\bV_{prev}^T \bw_j^{(m+1)}$.
\item Compute $\bu_j^{(m+2)}$.
\item Set $\bV_{prev} = \left[ \bV_{prev} ~~ \bu_j^{(m+2)} \right]$.

\end{enumerate}

\end{enumerate}
\end{description}
\vspace{.1in} \hrule \caption{Extending a Block Arnoldi Reduction}
\label{fig:blk_arn_red_alg}
\end{figure}
}


Step 1 allows the application of $\bA$ to a group of $s$ vectors.
This might prove essential when accessing $\bA$ is expensive.
Clearly, the goal is to amortize the cost of applying $\bA$ over
several vectors.

Step's 2 and 3 detail two separate processes for computing a new
block of orthonormal vectors at a given step in the iteration. The
process outlined in Step 2 is more efficient and is used by our
block GMRES solver until dependencies are detected in the Arnoldi
basis vectors. This may not occur at all, or if it does, it
usually occurs towards the very end of the iteration when some of
the systems comprising the block converge before others. When a
dependency is detected, a flag is set, $depflg = true$, indicating
this is the case, and relegating the construction of the current
and remaining blocks of basis vectors to the process given in Step
3. This decision was made because preliminary testing showed the
approach in Step 3 was more stable under these circumstances.

As written, Step 2a is one step of block classical Gram-Schmidt
(bCGR). This allows the use of the Level 3 {\small
BLAS}~\cite{dddh:90} matrix-matrix multiplication subroutine {\tt
\_GEMM} for computing $\bV_{m+1}^H \bW.$ To ensure the
orthogonality of $\bF^{(m+1)}$ with $\bV_{m+1}$, a second step of
bCGR is performed except when $s = 1$. In this latter case, a
simple test in DGKS~\cite{dgks:76} is used to determine whether a
second orthogonalization (correction) step is needed to ensure
orthogonality. See (\cite{lesy:98} and \cite{Le95}) for details.
Once $\bF^{(m+1)}$ is computed, we check for rank deficiencies in
Step 2b. If any of the columns of $\bW$~ ($\bw_j,~j=1,\ldots,s$)
are dependent on previous Arnoldi vectors (columns of
$\bV_{m+1}$), then the sine of the angle between $\bw_j$ and the
subspace spanned by the columns of $\bV_{m+1}$ will be very small.
If any dependencies are detected between the vectors in $\bW$ (the
component that brings the iteration up to the next higher
dimensional block Krylov subspace) and the previous Arnoldi
vectors, $depflg$ is set to true, and $\bF^{(m+1)}$ is recomputed
according to the process outlined in Step 3 below.

If no dependencies are detected, a QR factorization is used to
obtain an orthonormal set $\bU^{(m+2)}$ from $\bF^{(m+1)}$. This
procedure is also used to check for dependencies within the blocks
$\bF^{(m+1)}$. The QR factorization in Step 2b is computed via an
iterated classical Gram-Schmidt (CGS) process. The simple test in
DGKS~\cite{dgks:76} is used to determine whether a second
orthogonalization (correction) step is needed to ensure
orthogonality. One benefit of this scheme is that it allows the
use of the Level 2 {\small BLAS}~\cite{ddhh:88} matrix-vector
multiplication subroutine {\tt \_GEMV}. After constructing column
$j$ of $\bU^{(m+2)}$ by this process, linear dependencies are
detected by measuring the sine of the angle between ${\bf
f}_j^{(m+1)}$ and the space spanned by the previous $j-1$ columns
of $\bU^{(m+2)}$. Very small values indicate ${\bf f}_j^{(m+1)}$
is linearly dependent on the previous $j-1$ columns of
$\bU^{(m+2)}$. If dependencies are detected during this process,
$depflg$ is set to true, and the block $\bF^{(m+1)}$ is recomputed
according to Step 3 below.

The block orthogonalization process outlined in Step 2 allows for
the computation of the matrix coefficients $\bV_{m+1}^H \bW$ to be
separated from the QR factorization of $\bF^{(m+1)}.$ This is
advantageous in that it reduces the cost of I/O by a factor of the
block size and increases the amount of floating-point operations
per memory reference.

In the case that a rank deficiency is produced during the block
orthogonalization process (described in Step 2) at either the
current, or a previous step in the iteration, $depflg$ will be set
to true. The new block of Arnoldi vectors will be constructed one
at a time via the process outlined in Step 3.

In Step 3, we denote the matrix whose columns correspond to all
previously generated Arnoldi vectors as $\bV_{prev}$. The
orthogonalization process described in Step 3, iterates over the
$s$ columns of $\bW$ producing the corresponding columns of
$\bF^{(m+1)}$ and ${\widetilde \bH}_{m+1}$ via an iterated CGS
process. If needed, a correction step is used to ensure
orthogonality of ${\bf f}_j^{(m+1)}$ to all previous Arnoldi
vectors. Again, dependencies are detected by measuring the sine of
the angle between $\bw_j$ and all previous Arnoldi vectors. Very
small quantities indicate that $\bw_j$ is linearly dependent on
the columns of $\bV_{prev}$. If $\bw_j$ is independent, then ${\bf
f}_j^{(m+1)}$ is normalized to produce the $j$'th column
$\bu_j^{(m+2))}$ of $\bU^{(m+2)}$, and added to the set of
previous orthonormal Arnoldi vectors, $\bV_{prev} = \left[
\bV_{prev} ~~ \bu_j^{(m+2)} \right]$. If $\bw_j$ is dependent on
previous vectors, the subdiagonal element in the $(ms+j)$'th
column of ${\widetilde \bH}_{m+1}$ is set to zero, and a random
vector is generated and subsequently orthogonalized against all
previous Arnoldi vectors. The resulting vector ${\hat
\bu}_j^{(m+2)}$ is normalized producing the $j$'th column of
$\bU^{(m+2)}$, and is then augmented to the matrix $\bV_{prev}$.
Note that after all $s$ columns have been orthogonalized,
$\bV_{prev} = \bV_{m+2}$. This approach for producing $\bV_{m+2}$
is a modification of that used by Ruhe in \cite{ruhe:79} for
symmetric matrices. However, this process is more efficient than
that given in \cite{ruhe:79} in that it allows for the matrix
vector multiplication to take place between $\bA$ and a group of
vectors $\bU^{(m+1)}$ instead of performing $s$ individual matrix
vector multiplications.


\subsection{Defining the Block GMRES iterates}
\label{sec:belosblkgmres}

This section gives the details of how we update the iterates
defined by the block GMRES method. This development is essentially
the same as that given in (\cite{Sa96}, pp. 200-201). We summarize
the process here to establish our notation which is used to detail
our block GMRES implementation given in Figure \ref{fig:bGMRES}
below.

Let $\bX^{(0)}$ be the matrix of $s$ vectors (the guesses) and let
$\bR^{(0)} = \bB - \bA \bX^{(0)}$ be a matrix of full column rank
(the initial residuals). Compute the QR factorization $\bU^{(1)}
\bG_{1,0} = \bR^{(0)}$ where $\bU^{(1)}$ has $s$ orthonormal
columns. Use the algorithm in Figure~\ref{fig:blk_arn_red_alg} to
build a block Arnoldi reduction $\bA \bV_{i+1} = \bV_{i+1}
\bH_{i+1} + \bF^{(i+1)} \bE^T_{i+1}$ of length $i+1$ and let
 \dm
 \widetilde{\bH}_{i+1} =
 \left[
 \begin{array}{c}
                            \bH_{i+1} \\
                            \bG_{i+2,i+1} \end{array} \right].
 \edm
The columns of
 \dm
 \bV_{i+1} = \left[ \bU^{(1)} \ldots \bU^{(i+1)} \right]
 \edm
form an orthonormal basis for ${\cal K}_{i+1}(\bR^{(0)}, \bA)$.
Denote $\bR^{(i+1)} = \left[ \br_1^{(i+1)}, \ldots, \br_s^{(i+1)}
\right]$ as the $i+1$'st residual block. The block GMRES iterates
 \dm
 \bX^{(i+1)} = \bX^{(0)} + \bZ^{(i)}
 \edm
are defined by choosing $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$ such that
 \eq
 \|\br_j^{(i+1)}\|_2 ~~~{\rm
 is~minimized}~, {\rm for}~ j = 1,\ldots, s.
 \label{eq:gmresmin}
 \eeq
Since the columns of $\bV_{i+1}$ form a basis for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$, we can write $\bZ^{(i)} = \bV_{i+1}
\bY$, for some coefficient matrix $\bY$ with $(i+1)s$ rows and $s$
columns.

Simple manipulation gives
 \eq
 \begin{array}{rcl}
 \bR^{(i+1)} & = & \bB - \bA( \bX^{(0)} + \bV_{i+1} \bY) \\
      & = & \bR^{(0)} -  \bA \bV_{i+1} \bY \\
      & = & \bU^{(1)} \bG_{1,0} - \bA \bV_{i+1} \bY \\
      & = & \bV_{i+2}(\bE_1 \bG_{1,0} - \widetilde{\bH}_{i+1} \bY).
 \end{array} \label{eq:gmresresids}
 \eeq
Denote the $j$'th column of $\bE_1 \bG_{1,0}$ as ${\bf g}_j$, with
length $(i+2)s$.

The individual block GMRES approximations $\bx_j^{(i+1)}, j = 1,
\ldots, s$, have the form
 \dm
 \bx_j^{(0)} + \bV_{i+1} \by_j, \quad {\rm for}~ j = 1, \ldots, s,
 \edm
where $\by_j$ is a vector of length $(i+1)s$. Thus, in block GMRES
the vectors $\by_j$ are chosen to minimize the individual columns
in (\ref{eq:gmresresids}). Since $\bV_{i+2}$ has orthonormal
columns, this is equivalent to solving the $s$ least squares
problems
\begin{equation}
{\rm min}\| {\bf g}_j - \widetilde{\bH}_{i+1} \by_j \|_2, \quad
{\rm for}~ j = 1, \ldots, s.
\end{equation}
Assuming that $\widetilde{\bH}_{i+1}$ is of full column rank,
denote by $\bY^{(i+1)}$ the matrix whose columns are the $s$
unique minimizers. If $\widetilde{\bQ} \widetilde{\bT}$ is a QR
factorization of $\widetilde{\bH}_{i+1} $ with
 \dm
 \widetilde{\bT} =\left[
 \begin{array}{l} \bT_{i+1} \\ {\bf 0}_{(s,(i+1)s)} \end{array}
 \right]
 \edm
an upper triangular matrix with $(i+2)s$ rows and $(i+1)s$
columns, then the $s$ least squares solutions are given by solving
\begin{equation}
\bT_{i+1} \bY^{(i+1)} = \bS_1 \,\,\,\mbox{with} \,\,\,
\widetilde{\bQ}^T \bE_1 \bG_{1,0} = \left[ \begin{array}{c} \bS_1
\\ \bS_2
\end{array} \right] \begin{array}{l} \} \, (i+1)s \mbox{ rows}
\\ \} \, s \mbox{ rows} \end{array}
\end{equation}

Hence, $\bE_1 \bG_{1,0} - \widetilde{\bH}_{i+1} \bY^{(i+1)} =
\widetilde{\bQ} \left[
\begin{array}{l} {\bf 0}_{((i+1) \cdot s, s)} \\ \bS_2 \end{array}
\right] $ is the projection of $\bE_1 \bG_{1,0}$ onto the
orthogonal compliment of the range of $\widetilde{\bH}_{i+1}$. And
thus, if
 \dm
 \bX^{(i+1)} = \bX^{(0)} + \bV_{i+1} \bY^{(i+1)}
 \edm
and
$$\bR^{(i+1)} = \bB - \bA \bX^{(i+1)} = \bV_{i+2}(\bE_1
\bG_{1,0} - \widetilde{\bH}_{i+1} \bY^{(i+1)}),$$ it follows that
\begin{equation}
\| \br_j^{(i+1)} \|_2 = \|{\bf g}_j - \widetilde{\bH}_{i+1}
\bY^{(i+1)} \|_2 = \|\bs_j\|_2, ~{\rm for}~ j = 1,\ldots,s,
\end{equation}
where $\bs_j$ denotes the $j$'th column of $\bS_2$. This is
significant as it allows us to easily determine the quality of the
GMRES approximation during every iteration without having to
update the iterates $\bX^{(i+1)}$, because $\bS_2$ is easily
computed during the $s$ least squares solutions.

Our block GMRES implementation contains a restart mechanism, where
at each restart iteration, $restariter$, the final iterates (if
not converged sooner) are computed from a block Krylov subspace of
dimension $maxits$ (maximum number of iterations allowed before
restarting). The iteration is then restarted using this solution
as an initial guess for the next $restartiter$. We have observed
that if the iteration is close to convergence, the new initial
residuals produced after a restart cycle can be linearly
dependent. Our implementation handles the case when this occurs
during the initial QR factorization of the residual block.

We summarize our block GMRES algorithm in Figure~\ref{fig:bGMRES}.
This implementation is a careful reorganization of the scheme due
to B. Vital~\cite{vita:90} given in the appendix
of~\cite{siga:95}. There are three major differences between our
scheme and that listed in~\cite{siga:95}. The first is that we use
a classical Gram--Schmidt scheme via the level 3 BLAS instead of a
modified Gram--Schmidt scheme. The second difference is that we do
not explicitly compute $\bR^{(i+1)}$ via an application of $\bA$.
The final distinction is that we consider using a block GMRES
iteration regardless of the number of right-hand sides.


{\small
\begin{figure}[hbt]
\vspace{.05in} \hrule \vspace{.1in}
\begin{description}
\item[Initialization]
Set $ \bX^{(cur)} = \bX^{(0)}$ to be the  matrix of current
initial guesses, and set $restartiter=0$
\item[While] ($restartiter < numrestarts$, and not converged)
\begin{itemize}
\item Set $depflg = false$
\item Set $\bR^{(0)} = \bB - \bA \bX^{(cur)}$
\item Compute the QR factorization $\bU^{(1)} \bG_{1,0} =
\bR^{(0)}$
\item Set $\bV_1 = \left[ \bU^{(1)} \right]$, and $i=0$
\end{itemize}
\begin{description}
\item [While] ($i < maxits$, and not converged)
\begin{enumerate}
\item  Extend the block Arnoldi reduction by one so that a
length $i+1$ block Arnoldi reduction now exists for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$

\item Compute the $s$ least squares minimizers,\\
${\rm min}\| {\bf g}_j - \widetilde{\bH}_{i+1} \by_j \|_2, \quad
{\rm for}~ j = 1, \ldots, s, $  via a QR factorization
$\widetilde{\bQ} \widetilde{\bT}$ of $\widetilde{\bH}_{i+1}$ and
store in the matrix $\bY^{(i+1)}$

\item For $j = 1,\ldots, s$, compute $\| \br_j^{(i+1)} \|_2 = \| \bs_j \|_2$, where
$\bs_j$ is the $j$'th column of $\bS_2$ (an order $s$ sub-matrix
in the last $s$ rows of $\widetilde{\bQ}^T \bE_1 \bG_{1,0}$)

\begin{itemize}
\item If $\| \br_j^{(i+1)} \|_2 $ is small enough, for every $j$, or $maxits$ is reached,
compute the $s$ approximations $\bX^{(i+1)} = \bX^{(0)} +
\bV_{i+1} \bY^{(i+1)}$
\item Set $i = i+1$
\end{itemize}

\end{enumerate}

\item [Set] $\bX^{(cur)} = \bX^{(i+1)}$, and $restartiter =
restartiter + 1$
\end{description}

\end{description}
\vspace{.1in} \hrule \caption{Block GMRES implementation}
\label{fig:bGMRES}
\end{figure}
}



\section{Serial and Parallel Complexity Analysis}
\subsection{Complexity as a function of block size}
\subsection{Communication costs}

\section{Implementation Issues}
\subsection{Use of BLAS kernels}
\subsection{Possibilities for ``out-of-core'' implementations}

\section{Numerical experiments}
\subsection{Systems with known spectra}
\subsection{Systems with one RHS}
\subsubsection{Systems that are ``easy'' to solve}
\subsubsection{Systems that are ``hard'' to solve}
\subsection{Systems with multiple RHS}

\section{Conclusions}

It's great!

\begin{thebibliography}{AAAA99}

\bibitem[1]{AMS90} S. F. Ashby, T. A. Manteuffel and P. E.
Saylor, {\em A Taxonomy for Conjugate Gradient Methods}, SIAM J.
Numer. Anal., 27 (1990), pp. 1542-1568.

\bibitem[2]{bcrr:98} J. Baglama, D. Calvetti, L. Reichel, and A.
Ruttan, {\em Computation of a few close eigenvalues of a large
matrix with application to liquid crystal modeling}, Journal of
Computational Physics, 146 (1998), pp. 203-226.

\bibitem[3]{chro:91} A. Chronopoulos, {\em s-step methods for
(non)'symmetric and (in)definite linear systems}, SIAM J.
Numerical Analysis, 28 (1991), pp. 1776-1789.

\bibitem[4]{chki:92} A. Chronopoulos and S. K. Kim, {\em Towards
efficient parallel implementation of Krylov subspace iterative
methods}, Supercomputer, 47 (1992), pp. 4-17.

\bibitem[5]{dgks:76} J. Daniel, W. B. Gragg, L. Kaufman, and G. W.
Stewart, {\em Reorthogonalization and stable algorithms for
updating the Gram-Schmidt QR factorization}, Mathematics of
Computation, 30 (1976), pp. 772-795.

\bibitem[6]{dddh:90} J. J. Dongarra, J. DuCroz, I. S. Duff, and S.
Hammarling, {\em A set of Level 3 Basic Linear Algebra
Subprograms}, ACM Trans. Mathematical Software, 16 (1990), pp.
1-17.

\bibitem[7]{ddhh:88} J. J. Dongarra, J. DuCroz, S. Hammarling, and
R. J. Hanson, {\em An extended set of Fortran Basic Linear Algebra
Subprograms}, ACM Trans. Mathematical Software, 14 (1988), pp.
1-17.

\bibitem[8]{DUB01} A. Dubrulle, {\em Retooling The Method Of
Block Conjugate Gradients}, ETNA, 12 (2001), pp. 216-233.

\bibitem[9]{FOP95} Y. Feng, D. Owen, and D. Peric, {\em A
block conjugate gradient method applied to linear systems with
multiple right-hand sides}, Computer Methods in Applied Mechanics
and Engineering, 127 (1995), pp. 203-215.

\bibitem[10]{govl:96} G. H. Golub and C. F. V. Loan, {\em Matrix
Computations}, Johns Hopkins University Press, Baltimore, third
ed., 1996.

\bibitem[11]{HS52} M. Hestenes, and E. Stiefel, {\em Methods of
Conjugate Gradients for Solving Linear Systems}, J. Research of
the National Bureau of Standards, Vol. 49, No. 6 (1952), pp.
409-436.

\bibitem[12]{Le95} R. B. Lehoucq, {\em Analysis and Implementation
of an Implicitly Restarted Arnoldi Iteration}, Ph.D. Thesis, Rice
University, Houston, TX, 1995.

\bibitem[13]{lesy:98} R. B. Lehoucq, D. C. Sorensen, and C. Yang,
{\em ARPACK USERS GUIDE: Solution of Large Scale Eigenvalue
Problems with Implicitly Restarted Arnoldi Methods}, SIAM,
Philidelphia, PA, 1998.

\bibitem[14]{li:97} G. Li, {\em A block variant of the GMRES
method on massively parallel processors}, Parallel Computing, 23
(1997), pp. 1005-1019.

\bibitem[15]{NY95} A. Nikishin, and Y. Yeremin, {\em Variable
Block CG Algorithms For Solving Large Sparse Symmetric Positive
Definite Linear Systems On Parallel Computers, I: General
Iterative Scheme}, SIAM J. Matrix Anal. Appl., Vol. 16, No 4,
(1995), pp. 1135-1153.

\bibitem[16]{olea:80} D. P. O'Leary, {\em The Block Conjugate Gradient
Algorithm and Related Methods}, Linear Algebra And Its
Applications, 29 (1980), pp. 293-322.

\bibitem[17]{ruhe:79} A. Ruhe, {\em Implementation aspects of band
Lanczos algorithms for computation of eigenvalues of large sparse
symmetric matrices}, Mathematics of Computation, 33 (1979), pp.
680-687.

\bibitem[18]{SS86} Y. Saad, and M. H. Schultz, {\em GMRES: A
generalized minimal residual algorithm for solving nonsymmetric
linear systems}, SIAM J. Scientific Computing, 7 (1986), pp.
856-869.

\bibitem[19]{Sa96} Y. Saad, {\em Iterative Methods for Sparse
Linear Systems}, PWS Publishing Company, Boston, 1996.

\bibitem[20]{siga:95} V. Simoncini and E. Gallopoulos, {\em An
iterative method for nonsymmetric systems with multiple right-hand
sides}, SIAM J. Scientific Computing, 16 (1995), pp. 917-933.

\bibitem[21]{siga:96} V. Simoncini and E. Gallopoulos, {\em A
hybrid block GMRES method for nonsymmetric systems with multiple
right-hand sides}, Journal of Computational and Applied
Mathematics, 66 (1996), pp. 457-469.

\bibitem[22]{vita:90} B. Vital, {\em Etude de quelques m\`{e}thodes de
r\`{e}solution de probl\'{e}mes lin\`{e}aires de grande taille sur
multiprocessers}, PhD thesis, Universit\'{e} de Rennes I, Rennes,
France, 1990.


\end{thebibliography}

%\bibliographystyle{siam}
%\bibliography{ref}

\end{document}
