%
% Block Solver Report
%
%\documentstyle[psfig,numinsec]{siam1tex}
%\documentstyle[psfig,numinsec]{siam}
%\documentclass[final]{siamltex}
\documentclass{article}
\usepackage{amssymb}
\usepackage[dvipsone]{epsfig}
\newtheorem{defn}{Definition}[section]
%
\newcommand{\bA}{{\bf A}}
\newcommand{\bB}{{\bf B}}
\newcommand{\bC}{{\bf C}}
\newcommand{\bE}{{\bf E}}
\newcommand{\bF}{{\bf F}}
\newcommand{\bG}{{\bf G}}
\newcommand{\bH}{{\bf H}}
\newcommand{\bI}{{\bf I}}
\newcommand{\bM}{{\bf M}}
\newcommand{\bP}{{\bf P}}
\newcommand{\bQ}{{\bf Q}}
\newcommand{\bR}{{\bf R}}
\newcommand{\bS}{{\bf S}}
\newcommand{\bT}{{\bf T}}
\newcommand{\bU}{{\bf U}}
\newcommand{\bV}{{\bf V}}
\newcommand{\bW}{{\bf W}}
\newcommand{\bX}{{\bf X}}
\newcommand{\bY}{{\bf Y}}
\newcommand{\bZ}{{\bf Z}}
\newcommand{\bx}{{\bf x}}
\newcommand{\bu}{{\bf u}}
\newcommand{\bv}{{\bf v}}
\newcommand{\bw}{{\bf w}}
\newcommand{\bz}{{\bf z}}
\newcommand{\by}{{\bf y}}
\newcommand{\bp}{{\bf p}}
\newcommand{\br}{{\bf r}}
\newcommand{\bs}{{\bf s}}
\newcommand{\bh}{{\bf h}}
\newcommand{\be}{{\bf e}}
\newcommand{\bb}{{\bf b}}
\newcommand{\bff}{{\bf f}}
\newcommand{\bPi}{{\bf \Pi}}
%
\newcommand{\ul}{\underline}
\newcommand{\la}{\langle}
\newcommand{\ra}{\rangle}
\newcommand{\dm}{\begin{displaymath}}
\newcommand{\edm}{\end{displaymath}}
\newcommand{\eq}{\begin{equation}}
\newcommand{\eeq}{\end{equation}}
%
\begin{document}
\newfont{\smrm}{cmr8}
\newfont{\esmrm}{cmr7}

\title{Belos Block Solvers \\
       Algorithm Description Document}

\author{Teri Barth}

\maketitle


\section{Introduction}


Given a linear system of equations
  \eq \bA \bx = \bb,
  \label{eq:linsys}
  \eeq
where $\bA$ is a general $n \times n$ nonsingular matrix and $\bx$
and $\bb \in {\cal C}^n$, and, given an initial guess $\bx^{(0)}$
to (\ref{eq:linsys}), a Krylov subspace method produces a sequence
of approximate solutions $\{ \bx^{(i)} \}$ of the form
 \dm
 \bx^{(i+1)} = \bx^{(0)} + \bz^{(i)}, \quad  {\it where}~\bz^{(i)} \in {\cal
 K}_{i+1}(\br^{(0)},\bA),
 \edm
 $\br^{(0)}$ is the initial residual, and
  \dm
 {\cal K}_{i+1}(\br^{(0)},\bA) = {\rm sp}\{\br^{(0)},\bA \br^{(0)},\ldots,\bA^{i} \br^{(0)}\}
 \edm
denotes the Krylov subspace of dimension $i+1$ generated by the
initial residual and the system matrix $\bA$.  Since the Krylov
subspaces are nested
 \dm
 {\cal K}_j(\br^{(0)}, \bA) \subset {\cal K}_{j+1}(\br^{(0)}, \bA), \quad
 \forall j,
 \edm
the iterates can also be written as
 \dm
 \bx^{(i+1)} = \bx^{(i)} + \bz^{(i)}, \quad  {\it where}~\bz^{(i)} \in {\cal
 K}_{i+1}(\br^{(0)},\bA).
 \edm
The way that $\bz^{(i)}$ is chosen from ${\cal K}_{i+1}(\br^{(0)},
\bA)$ at each step is what distinguishes the various Krylov
methods.

A conjugate gradient (CG) method for the solution of
(\ref{eq:linsys}) is a Krylov subspace method in which the error
is minimized at every step in an inner product norm. We define
this method with respect to a Hermitian positive definite (HPD)
inner product matrix $\bW$ and the system matrix $\bA$ and denote
it as ${\cal CG}(\bW,\bA)$.

\begin{defn} \label{def:cg_method}
{Given an initial guess $\bx^{(0)}$ to (\ref{eq:linsys}), a
conjugate gradient method, ${\cal CG}(\bW,\bA),$ for the solution
of (\ref{eq:linsys}) is a Krylov subspace method whose iterates
are defined uniquely as follows:
 \eq
 \begin{array}{rcll}
 \bx^{(i+1)} &=& \bx^{(i)} + \bz^{(i)}, \quad & {\it where}~\bz^{(i)} \in {\cal
 K}_{i+1}(\br^{(0)},\bA), \\
 ~&~&~&{\it and~is~chosen~s.t.} \\
  \be^{(i+1)} & \perp_{\bW} & {\cal K}_{i+1} (\br^{(0)}, \bA), &~
 \end{array}
 \label{eq:cg_iterates}
  \eeq
or equivalently,
 \dm
 \| \be^{(i+1)} \|_{\bW} ~{\it is~minimized~over}~ {\cal K}_{i+1} (\br^{(0)},\bA),
 \edm
 where $\be^{(i+1)}$ denotes the error at step $i+1$, and $\perp_{\bW}$
 represents orthogonality in the $\bW$ inner product. }
\end{defn}

For example, if $\bA$ HPD (symmetric positive definite (SPD) if
$\bA$ is real), $\la \bA \cdot, \cdot \ra$ defines an inner
product, and we could take $\bW = \bA$.\ In this case, ${\cal
CG}(\bW,\bA)$ minimizes the $\bA$-norm of the error, which results
in the original conjugate gradient method of Hestenes and Stiefel
\cite{HS52}. Another example of a conjugate gradient method is
given by taking $\bW = \bA^H \bA$, where, $\bA^H = {\bar \bA}^T$.
Notice that since we are assuming that $\bA$ is nonsingular,
$\bA^H \bA$ is HPD (or SPD if $\bA$ is real). In this case, ${\cal
CG}(\bW,\bA)$ minimizes
 \dm \| \be^{(i+1)} \|_{\bA^H\bA} = \la \bA^H \bA \be^{(i+1)},\, \be^{(i+1)} \ra^
 {\frac{1}{2}} = \la \br^{(i+1)},\, \br^{(i+1)} \ra^{\frac{1}{2}},
 \edm
yielding a minimal residual method.  GMRES is an implementation of
a minimal residual method \cite{SS86}.  A detailed taxonomy of
conjugate gradient methods can be found in \cite{AMS90}.

CG methods possess a finite termination property. Since the Krylov
subspaces are nested, it follows from the definition of the method
that at step $n$, we are forcing the error to be orthogonal in
some norm to the entire $n$-dimensional space. In exact
arithmetic, this yields $\be^{(n)} = {\bf 0}$, and the iteration
terminates in at most $n$ steps.

CG methods are implemented by constructing an orthogonal basis for
the underlying Krylov subspaces. At each step, this involves a
matrix vector multiplication to bring the iteration up to the next
higher dimensional Krylov subspace. Since at most $n$ steps are
required for convergence in exact arithmetic, at most $n$ matrix
vector multiplications are required to solve the single right-hand
side system using a CG method.

In many situations, it may be advantageous, or possibly, required,
to obtain the solution of several linear systems with the same
coefficient matrix $\bA$ and different right-hand sides
simultaneously. Block versions of Krylov methods for solving the
block linear system
 \eq
 \bA \bX = \bB,
 \label{eq:blocklinsys}
 \eeq
 where, $\bA \in {\cal C}^{n \times n}$, and
 $\bB = [\bb_1, \ldots, \bb_s] \in {\cal C}^{n \times s}$ is a matrix of $s$ right-hand
 sides, have been developed for this purpose. Like their single
 right-hand side counterparts, block Krylov methods produce a
 sequence of block iterates,
  \dm
 \bX^{(i+1)} = \left[ \bx_1^{(i+1)}, \ldots, \bx_s^{(i+1)}
 \right],
 \edm
 of the form
  \dm
  \bX^{(i+1)} = \bX^{(0)} + \bZ^{(i)},
  \edm
 where $\bZ^{(i)}$ is chosen from the block Krylov subspace of
 dimension $i+1$,
  \dm
 {\cal K}_{i+1}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA
 \bR^{(0)},\ldots,\bA^{i}\bR^{(0)}\},
 \edm
and,
  \dm
   \bR^{(0)} = \left[ \br_1^{(0)}, \ldots, \br_s^{(0)} \right]
  \edm
denotes a matrix of $s$ initial residuals. These methods generally
choose $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},\bA)$ so that some
orthogonality or block minimization condition is satisfied.

Analogous to the single right-hand side case, block
generalizations of CG methods for solving the block linear system
(\ref{eq:blocklinsys}) can be defined by:

 \begin{defn} \label{def:blkcgmethod}
 {Given an initial guess
  \dm
  \bX^{(0)} = \left[ \bx_1^{(0)}, \ldots, \bx_s^{(0)} \right],
  \edm
 to (\ref{eq:blocklinsys}), the iterates for a Block CG method, $blk{\cal CG}(\bW,\bA),$
 can be defined as follows:
 \eq
 \begin{array}{rcll}
 \bX^{(i+1)} &=& \bX^{(i)} + \bZ^{(i)}, \quad & {\it where}~ \bZ^{(i)} \in {\cal
 K}_{i+1}(\bR^{(0)},\bA), \\
 ~&~&~& {\it and~is~chosen~s.t.} \\
  \bE^{(i+1)} & \perp_{\bW} & {\cal K}_{i+1} (\bR^{(0)}, \bA), &~
 \end{array}
 \label{eq:blk_iterates}
  \eeq
 where,
  \dm
   \bE^{(i+1)} = \left[ \be_{1}^{(i+1)}, \ldots, \be_{s}^{(i+1)}
   \right],
   \edm
denotes the $(i+1)$'st error block. }
\end{defn}
\vspace{0.1in}

At step $k = ceil(n/s)$, the block Krylov subspace
 \dm
 {\cal K}_{k}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA \bR^{(0)},\ldots,\bA^{k-1}\bR^{(0)}\}
 \edm
spans the entire $n$-dimensional space. In exact arithmetic, it
follows that
 \dm
 \bE^{(k)} \perp_{\bW} {\cal K}_k (\bR^{(0}, \bA) \Rightarrow
 \bE^{(k)} = {\bf 0}_{(n,s)},
 \edm
yielding termination of the block iteration in at most $ceil(n/s)$
steps.

Block generalizations of CG methods are implemented by
constructing a blockwise orthogonal basis for the underlying block
Krylov subspaces. At each step, this involves $s$ (the block size
used by the solver) matrix vector multiplications. Here, the
matrix $\bA$ is applied to $s$ vectors all at once. For users who
have multiple right-hand systems to solve, a block implementation
may provide significant savings in the work required. For example,
to solve for $s$ right-hand sides individually using a single
right-hand side CG implementation would require at most (in exact
arithmetic) $s \times n$ steps, and $s \times n$ matrix vector
multiplications. To solve them using a block implementation with a
block size of $s$ would require at most $ceil(n/s)$ steps, and
roughly $n$ matrix vector multiplications (applied $s$ at a time).
Although real world application problems utilize preconditioning
techniques and generally (to be useful) converge to within some
acceptable tolerance level in much fewer iterations than our exact
arithmetic analysis bounds yield, the goal is to produce block
implementations for solving (\ref{eq:blocklinsys}) that converge
faster that it would take to solve for the $s$ linear systems
consecutively using a single right-hand side implementation. Much
work is still needed to determine the true savings that block
implementations might provide. Any such study should also factor
in the details of the particular block implementation, as these
can make a significant impact on the actual convergence behavior
in finite precision arithmetic.

Because, in a block implementation, the coefficient matrix $\bA$
is applied to several vectors at a time, this provides a
significant benefit to users with matrices $\bA$ that are
expensive to to apply. This expense might result from the cost of
communication when computing a matrix vector product with $\bA$ on
a distributed memory parallel computer, or the memory latency that
exists on floating point rich code that often exhibits poor data
locality, or both.

A block implementation may also provide a significant benefit to
users who have extremely difficult linear systems (poorly
conditioned) to solve. These systems typically arise from
high-fidelity multi-physics simulations. It was pointed out in
\cite{Ol80}  that a block CG implementation was more robust that a
single CG implementation for SPD matrices containing several
extreme eigenvalues widely separated from the others.

Block generalizations of Krylov subspace methods are not new,
however, good robust implementations are needed in order to make
them a viable methodology for solving the large linear systems of
equations with one or more right-hand sides that arise from
today's challenging application problems. From Definition
\ref{def:blkcgmethod} and the earlier discussion, $blk{\cal
CG}(\bA, \bA)$ (for HPD $\bA$) and $blk{\cal CG}(\bA^H \bA, \bA)$
can be viewed as block generalizations of the common CG and GMRES
methods, respectively. Although, by definition, both are CG
methods, for clarity, we will refer to $blk{\cal CG}(\bA, \bA)$ as
BlockCG, and $blk{\cal CG}(\bA^H \bA, \bA)$ as BlockGMRES. The
Belos Block Linear Solver package contains implementations of
BlockCG and BlockGMRES. Future work may include block
generalizations of other Krylov subspace methods.


\section{Notation and Definitions}
\label{sec:notation}

This section establishes the basic notation to be used in this
article. We employ Householder notational conventions. Capital and
lower-case letters denote matrices and vectors, respectively,
while lower-case Greek letters denote scalars. In addition,
bracketed lower-case Greek letters denote coefficient matrices.

The transpose of a vector ${\bf x}$ is denoted by ${\bf x}^T$, and
the complex conjugate of ${\bf x}^T$ is denoted by ${\bf x}^H.$
Similarly, the matrix $\bV^H = {\bar \bV}^T$.

The order of $\bA$ will always be denoted by $n.$ The identity
matrix of order $m$ is denoted by ${\bf I}_{(m,m)}.$ The $m \times
t$ zero matrix is given by ${\bf 0}_{(m,t)}$. The block size used
by the solvers is $s$.

Superscripts on matrices correspond to iteration numbers. For
example, $\bX^{(i)}$ and $\bR^{(i)}$, denote the $i$'th blocks of
iterates and residuals, respectively, produced by the iteration.
The matrix $\bP^{(i)}$ denotes the $i$'th block of direction
vectors associated with the BlockCG iteration, and the matrices
$\bF^{(i)}$ and $\bU^{(i)}$ correspond to the $i$'th blocks of
basis vectors associated with the BlockGMRES iteration.

The remaining notation will be defined as needed, or will be clear
from the context.


\section{Belos General Block Solver Features}

The Belos Block Solvers are applicable only to real matrices $\bA,
~ \bX$ and $\bB$. Thus, for the remainder of this report, we will
assume that $\bA \in {\cal R}^{n \times n}$ and nonsingular, and
$\bX$ and $\bB \in {\cal R}^{n \times p}$.

The block implementations in Belos allow for the total number of
right-hand sides $p$ in the block linear system to be solved to be
different that the block size $s$ used in the solver. The block
size refers to the number of right-hand sides to be solved for
simultaneously within the block solver. The number of right-hand
sides $p$ and the block size $s$ are user specified. This
separation allows a user to:
\begin{itemize}
\item Solve a linear system with a single right-hand side $(p=1)$
using a block implementation $(s>1)$.
\item Solve a system with many (e.g., hundreds) of
right-hand sides, but only solve for several (e.g., 5 - 20) of
them simultaneously.
\item Solve a system with any number of right-hand sides using any
block size.
\end{itemize}
We note here, that very large block sizes may not yield good
solver convergence results. \vspace{0.2in}

The Belos Block Solver routines have an outer loop that iterates
over the $p$ right-hand sides to be solved for, solving them $s$
at a time. This requires
 \dm
 maxrhsiters = {\rm int}[(p+s-1)/s].
 \edm
iterations to solve for all $p$ right-hand sides. This process is
outlined below:
 \vspace{0.2in}

For $rhsiters = 1:maxrhsiters$
\begin{itemize}
\item Set up the blocks containing the next $s$ initial guesses
and $s$ right-hand sides
\item Solve for the $s$ right-hand sides simultaneously using the
block solver
\item Extract the $s$ solutions from the current pass and insert
into a final structure that holds solutions to all $p$ right-hand
sides
\end{itemize}
\vspace{0.2in}

Two routines in the Belos Block Solver library facilitate this
process; {\it SetUpBlocks} and {\it ExtractCurSolnBlock}.

At each iteration through the outer loop, the routine {\it
SetUpBlocks} places the next $s$ initial guesses and right-hand
sides into the current solution and right-hand side blocks
(multivectors) utilized by the block solver. If the number of
remaining right-hand sides to be solved for during this pass is
less than the block size $s$ used by the solver, then zero vectors
are augmented to the initial guess block, and random vectors are
augmented to the right-hand side block, so that each has a total
of $s$ vectors. We note that the user could control the initial
guesses and right-hand sides that are augmented to the block
system by augmenting the block system (yielding one where the
number of right-hand sides $p$ is evenly divided by the block size
$s$) before the call to the solver routine.

The routine {\it ExtractCurSolnBlock} takes the $s$ solutions
obtained after the current pass through the block solver and
stores them in the final solution structure (a multivector with
$p$ columns). If the block size $s$ is greater than the remaining
right-hand sides, $m$, to be solved for at this step, then only
the first $m$ solutions are extracted and placed into the final
solution structure. The remaining $s-m$ solutions correspond to
the augmented systems (that were provided by {\it SetUpBlocks} in
order to yield a block size of $s$ required by the solver).


\section{Belos BlockCG}

\subsection{Introduction}

Block generalizations of the well known CG algorithm for solving
linear systems involving an SPD coefficient matrix $\bA$ are not
new. For example, they have been studied in \cite{Ol80}, and more
recently in (\cite{FOP95}, \cite{NY95}, and \cite{DUB01}).

Analogous to the common single CG method, the BlockCG method for
solving the block linear system of equations
(\ref{eq:blocklinsys}) is implemented via the construction of a
blockwise $\bA$-orthogonal basis $\{ \bP^{(0)}, \bP^{(1)}, \ldots,
\bP^{(i)} \}$ for the underlying block Krylov subspaces
 \dm
 {\cal K}_{i+1}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA
 \bR^{(0)},\ldots,\bA^{i}\bR^{(0)}\}= {\rm sp}\{ \bP^{(0)},
 \bP^{(1)}, \ldots, \bP^{(i)} \},
 \edm
 where,
 \dm
 \bP^{(j)T} \bA \bP^{(k)} = {\bf 0}_{(s,s)}, \quad {\rm
 if}~j \ne k.
 \edm
At each step $i$, the BlockCG iterates,
 \dm
 \bX^{(i+1)} = \bX^{(i)} + \bZ^{(i)}
 \edm
are defined by choosing $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$ such that $\bE^{(i+1)} \perp_{\bA} {\cal K}_{i+1}(\bR^{(0)},
\bA)$.

A common implementation of the BlockCG method is given in Figure
\ref{fig:commonblkcg} below. In the case when $s=1$, this yields
the well known single vector CG implementation.

\begin{figure}[hbt]
\dm
 \begin{array}{|l|}
 \hline
 ~~\\
 {\rm Given} ~ \bX^{(0)}, ~\bR^{(0)} = \bB - \bA \bX^{(0)} ~~~ {\rm Initial\ Guess,\
 Initial\ Residual\ Blocks} \\
 {\rm Set}~ \bP^{(0)} =\bR^{(0)} ~~~ {\rm Initial\ Direction\ Vector\
 Block} \\ \\
 {\rm For}~ i=0,1,..., ~{\rm until\ convergence,\ compute:} \\

\begin{array}{rcl}

[\alpha_i] &=& (\bP^{(i)T} \bA \bP^{(i)})^{-1} (\bR^{(i)T} \bR^{(i)})  \\

\bX^{(i+1)} &=& \bX^{(i)} + \bP^{(i)} [\alpha_i] \\

\bR^{(i+1)} &=& \bR^{(i)} - \bA \bP^{(i)} [\alpha_i] \\

[\beta_i] &=& (\bR^{(i)T} \bR^{(i)})^{-1} (\bR^{(i+1)T} \bR^{(i+1)}) \\

\bP^{(i+1)} &=& \bR^{(i+1)} + \bP^{(i)} [\beta_i]

\end{array} \\
~~\\
 \hline
\end{array}
\edm

\caption{Common implementation of BlockCG, $blk{\cal CG}(\bA,
\bA)$} \label{fig:commonblkcg}
\end{figure}

The blocks of direction and residual vectors produced by the block
algorithm satisfy the following block orthogonality properties
(see for example \cite{Ol80}):

 \eq
 \begin{array}{rcl}
  \bP^{(j)T} \bA \bP^{(k)} = {\bf 0}_{(s,s)}, \qquad j &\ne& k, \\
  \bR^{(j)T} \bR^{(k)} = {\bf 0}_{(s,s)}, \qquad j &\ne& k, \\
  \bP^{(j)T} \bR^{(k)} = {\bf 0}_{(s,s)}, \qquad j &<& k.
  \end{array}
  \label{eq:blkcgorth}
  \eeq

This algorithm can breakdown during the computation of the
coefficient matrices $[\alpha_i]$ and $[\beta_i]$ if either of the
quantities,
 \eq
 (\bP^{(i)T} \bA \bP^{(i)}) ~~{\rm or}~~ (\bR^{(i)T} \bR^{(i)})
 \label{eq:commonbd}
 \eeq
becomes singular at some step during the iteration. This happens
if the columns of either of the matrices $\bP^{(i)}$ or
$\bR^{(i)}$ become linearly dependent. In \cite{Ol80} it was shown
that the ranks of the direction vector blocks $\bP^{(j)}$ and the
residual blocks $\bR^{(j)}$ are the same for every $j$. It follows
that the linear independence (dependence) of these blocks can be
monitored via an orthogonalization procedure involving just the
direction vector blocks. For example, this could be accomplished
via a QR factorization at each step of the new direction vector
block,
  \dm
  \bP^{(i+1)} = {\hat \bP}^{(i+1)} \bG_{i+1}, \qquad {\hat \bP}^{(i+1)T}{\hat
  \bP}^{(i+1)}= I_{(s,s)}.
  \edm
Further, in \cite{Ol80} it was noted that the dependent direction
vectors and the corresponding columns of iterates and residuals
could be dropped from the blocks, and the iteration resumed with a
decreased block size, thus, avoiding breakdown of the algorithm.

Since the columns of ${\hat \bP}^{(i+1)}$ are just linear
combinations of those in $\bP^{(i+1)}$, it follows that
 \dm
  {\hat \bP}^{(j)T} \bA {\hat \bP}^{(k)} = {\bf 0}_{(s,s)}, \qquad {\rm for}~ j
  \ne k,
  \edm
and the ${\hat \bP}^{(j)}$'s also form a blockwise
$\bA$-orthogonal basis for the underlying block Krylov subspaces.
Thus, the BlockCG implementation given in Figure
\ref{fig:commonblkcg} can be easily reformulated in terms of the
${\hat \bP}^{(j)}$'s. If we are already computing the ${\hat
\bP}^{(j)}$'s as a result of an orthogonalization process to
detect linear dependencies, no additional work results from this
formulation. By substituting $\bP^{(i+1)} = {\hat \bP}^{(i+1)}
\bG_{i+1}$ into the algorithm given in Figure
\ref{fig:commonblkcg}, and some algebra, we obtain a version of
the algorithm that is written in terms of the ${\hat
\bP}^{(i+1)}$'s, which is detailed in Figure
\ref{fig:commonblkcg_qr}. The ${\hat \bP}$'s and the corresponding
$\bR$'s produced by this formulation of BlockCG satisfy the same
block orthogonality conditions (\ref{eq:blkcgorth}).

\begin{figure}[hbt]
\dm
 \begin{array}{|l|}
 \hline
 ~~\\
 {\rm Given} ~ \bX^{(0)}, ~\bR^{(0)} = \bB - \bA \bX^{(0)} ~~~ {\rm Initial\ Guess,\
 Initial\ Residual\ Blocks} \\
 {\rm Set}~ \bP^{(0)} =\bR^{(0)} ~~~ {\rm Initial\ Direction\ Vector\
 Block} \\
 {\rm Compute}~ \bP^{(0)} = {\hat \bP}^{(0)} G_{0} \\ \\

 {\rm For}~ i=0,1,..., ~{\rm until\ convergence,\ compute:} \\

\begin{array}{rcl}

[\alpha_i] &=& ({\hat \bP}^{(i)T} \bA {\hat \bP}^{(i)})^{-1} \bG_{i}^{-T} (\bR^{(i)T} \bR^{(i)})  \\

\bX^{(i+1)} &=& \bX^{(i)} + {\hat \bP}^{(i)} [\alpha_i] \\

\bR^{(i+1)} &=& \bR^{(i)} - \bA {\hat \bP}^{(i)} [\alpha_i] \\

[\beta_i] &=& \bG_{i}(\bR^{(i)T} \bR^{(i)})^{-1} (\bR^{(i+1)T} \bR^{(i+1)}) \\

\bP^{(i+1)} &=& {\hat \bP}^{(i+1)} \bG_{i+1} = \bR^{(i+1)} + {\hat
\bP}^{(i)} [\beta_i]

\end{array} \\
~~\\
 \hline
\end{array}
\edm

\caption{Common implementation of BlockCG with QR factorization of
direction vector blocks} \label{fig:commonblkcg_qr}
\end{figure}


Like the single right-hand side CG implementation, the coefficient
matrices, $[\alpha_i]$ and $[\beta_i]$ given in Figure
\ref{fig:commonblkcg} can be written using different formulas. For
example,
 \dm
 \begin{array}{rcl}
  [\alpha_i] &=& (\bP^{(i)T} \bA P^{(i)})^{-1} (\bP^{(i)T} \bR^{(i)}) \\

  [\beta_i] &=& -(\bP^{(i)T} \bA \bP^{(i)})^{-1} (\bP^{(i)T} \bA \bR^{(i+1)}).
 \end{array}
 \edm
This was proven in \cite{HS52} for the single right-hand side CG
implementation. The formulas for the block implementation can be
derived using the orthogonality properties given in
(\ref{eq:blkcgorth}). Using these formulas for the coefficients
yields a variation of the BlockCG implementation which we list in
Figure \ref{fig:blkomin} below.

\begin{figure}[hbt]
\dm
 \begin{array}{|l|}
 \hline
 ~~\\
 {\rm Given} ~ \bX^{(0)}, ~\bR^{(0)} = \bB - \bA \bX^{(0)} ~~~ {\rm Initial\ Guess,\
 Initial\ Residual\ Blocks} \\
 {\rm Set}~ \bP^{(0)} =\bR^{(0)} ~~~ {\rm Initial\ Direction\ Vector\
 Block} \\ \\
 {\rm For}~ i=0,1,..., ~{\rm until\ convergence,\ compute:} \\

\begin{array}{rcl}

[\alpha_i] &=& (\bP^{(i)T} \bA P^{(i)})^{-1} (\bP^{(i)T} \bR^{(i)})  \\

\bX^{(i+1)} &=& \bX^{(i)} + \bP^{(i)} [\alpha_i] \\

\bR^{(i+1)} &=& \bR^{(i)} - \bA \bP^{(i)} [\alpha_i] \\

[\beta_i] &=& -(\bP^{(i)T} \bA \bP^{(i)})^{-1} (\bP^{(i)T} \bA \bR^{(i+1)}) \\

\bP^{(i+1)} &=& \bR^{(i+1)} + \bP^{(i)} [\beta_i]

\end{array} \\
~~\\
 \hline
\end{array}
\edm \caption{Omin implementation of BlockCG} \label{fig:blkomin}
\end{figure}


Like the common BlockCG algorithm given in Figure
\ref{fig:commonblkcg} this implementation of BlockCG can breakdown
during the computation of the coefficient matrices $[\alpha_i]$
and $[\beta_i]$ if $(\bP^{(i)T} \bA \bP^{(i)})$ becomes singular
at some step during the iteration. This happens if the columns of
the matrix $\bP^{(i)}$ become linearly dependent at some step in
the iteration. We note that with this formulation of the BlockCG
algorithm, the quantity $(\bR^{(i)T} \bR^{(i)})^{-1}$ does not
appear explicitly in the computation of the coefficient matrix
$[\beta_i]$. Thus, this formulation may be advantageous
numerically since we only need to be concerned explicitly with the
rank of one quantity. Analogous to the common BlockCG
implementation, once a new block of direction vectors are
computed, dependent columns can be detected via a QR factorization
procedure,
  \dm
  \bP^{(i+1)} = {\hat \bP}^{(i+1)} \bG_{i+1}, \qquad {\hat
  \bP}^{(i+1)T} {\hat \bP}^{(i+1)} = \bI_{(s,s)}.
  \edm
If a dependency is detected, a deflation procedure can be
incorporated that drops dependent vectors, thus, preventing
breakdown of the algorithm. Substituting $\bP^{(i+1)} = {\hat
\bP}^{(i+1)} \bG_{i+1}$ into the algorithm given in Figure
\ref{fig:blkomin}, along with some algebra yields a version of the
algorithm that is written in terms of the ${\hat \bP}^{(i+1)}$.
This version is listed in Figure \ref{fig:blkomin_qr} below. This
implementation of BlockCG has an advantage over the algorithm
given in Figure \ref{fig:commonblkcg_qr} in that the matrices
$\bG_{i}$ do not appear explicitly in the computation of the
coefficients $[\alpha_i]$ and $[\beta_i]$. In the case when the
columns of $\bP^{(i)}$ are nearly dependent, the matrices
$\bG_{i}$ will be ill conditioned, and thus, the systems to be
solved for the coefficients in the common BlockCG implementation
given in Figure \ref{fig:commonblkcg_qr} will be ill conditioned,
which can yield inaccurate results. This was also pointed out in
\cite{DUB01}.


\begin{figure}[ht]
 \dm
 \begin{array}{|l|}
 \hline
 ~~\\
 {\rm Given} ~ \bX^{(0)}, ~\bR^{(0)} = \bB - \bA \bX^{(0)} ~~~ {\rm Initial\ Guess,\
 Initial\ Residual\ Blocks} \\
 {\rm Set}~ \bP^{(0)} =\bR^{(0)} ~~~ {\rm Initial\ Direction\ Vector\
 Block} \\
 {\rm Compute} ~ \bP^{(0)} = {\hat \bP}^{(0)T} \bG_{0} ~~~ {\rm QR\
Factorization} \\ \\

 {\rm For}~ i=0,1,..., ~{\rm until\ convergence,\ compute:} \\

\begin{array}{rcl}

\qquad \qquad [\alpha_i] &=& ( {\hat \bP}^{(i)T} \bA {\hat \bP}^{(i)})^{-1} ( {\hat \bP}^{(i)T} \bR^{(i)})  \\

\bX^{(i+1)} &=& \bX^{(i)} + {\hat \bP}^{(i)} [\alpha_i] \\

\bR^{(i+1)} &=& \bR^{(i)} - \bA {\hat \bP}^{(i)} [\alpha_i] \\

[\beta_i] &=& -( {\hat \bP}^{(i)T} \bA {\hat \bP}^{(i)})^{-1} ( {\hat \bP}^{(i)T} \bA \bR^{(i+1)}) \\

\bP^{(i+1)} &=& {\hat \bP}^{(i+1)T} \bG_{i+1} = \bR^{(i+1)} +
{\hat \bP}^{(i)} [\beta_i]

\end{array} \\
~~\\
 \hline
\end{array}
\edm

 \caption{Omin implementation of BlockCG with QR factorization of
 direction vector blocks} \label{fig:blkomin_qr}
\end{figure}


Belos BlockCG is implemented via the algorithm given in Figure
\ref{fig:blkomin_qr}, together with a deflation mechanism to
prevent numerical breakdowns. A detailed description of the Belos
BlockCG implementation is given below in Section
\ref{sec:belosblkcg}. Our BlockCG implementation is similar to an
implementation presented in \cite{DUB01}. In \cite{DUB01} they
explored various modifications of the BlockCG algorithm, which
included an approach that enforced $\bA$-orthogonality between the
individual vectors within the direction vector blocks. In this
case, a simplification occurs when computing the coefficient
matrices $[\alpha_i]$ and $[\beta_i]$. This approach may be more
expensive, and for large block sizes, loss of $\bA$-orthogonality
could become an issue in the implementation. Their approach also
differs from our implementation in that instead of a deflation
mechanism, they use a procedure that utilizes a change of bases to
supplement rank defects.

\subsection{Belos BlockCG Implementation}
\label{sec:belosblkcg}

The deflation mechanism in the Belos BlockCG implementation is
designed to prevent breakdowns that can occur in the iteration.
This is accomplished by detecting and then dropping both dependent
direction vectors, and residuals that are very small in norm, from
the corresponding blocks, and then continuing the iteration by
enforcing the orthogonality properties of the BlockCG method on
the deflated block system. Since we do not want to be continually
de-allocating and re-allocating memory to handle the changing
block sizes, this process is realized by an indexing mechanism.
Only the direction vectors, residual vectors, and iterates within
the blocks that correspond to certain indices are updated.

Three sets of indices are used by the Belos BlockCG
implementation. The independent indices, $[iidx]$ and $[piidx]$,
are the indices of the linearly independent columns of the current
and previous direction vector blocks, respectively. The quantity
$numind$ refers to the number of independent vectors in the
current direction vector block. The current indices, $[ridx]$, are
the indices corresponding to the nonzero residuals (those
residuals with residual norm greater than some small tolerance,
$rtol$, in the current residual block, and $numcur$ refers to the
number of these indices. In the Belos implementation,
 \dm
 rtol = eps,
 \edm
the machine precision, which is computed via a call to the routine
{\it CGBlkTols}.  As the name implies, the converged indices,
$[cidx]$, are the indices in the current residual block that
correspond to residuals that have converged according to some user
specified residual tolerance, $ctol$, and $numconv$ refers to the
number of converged residuals. For proper execution of the block
iteration, it is necessary for chose $ctol$ greater than or equal
to $rtol$. Thus, $[ridx]$ and $[cidx]$ can be different sets. This
distinction was made for numerical reasons, since residuals that
have converged according to some user specified tolerance may
still be large enough to be used to generate new linearly
independent direction vectors, and thus, further assist in the
convergence of the block linear system. Because the Belos BlockCG
implementation keeps track of the indices of the residuals in the
block that have converged, it would be straight forward to add a
feature that would return the corresponding solutions to the user
as soon as convergence occurs, instead of waiting for convergence
of the entire residual block.

At step $i$ we have the following quantities from the previous
step,
 \dm
 {\hat \bP}^{(i)}_{[piidx]}, ~~ {\rm and} ~~ \bR^{(i)}_{[ridx]},
 \edm
where, ${\hat \bP}^{(i)}_{[piidx]}$ refers to the columns of the
$n \times s$ matrix of direction vectors ${\hat \bP}^{(i)}$ that
are linearly independent, and $\bR^{(i)}_{[ridx]}$ denotes the
nonzero columns of the $n \times s$ matrix of residuals
$\bR^{(i)}$, i.e., those residuals that are greater than $rtol$ in
norm. We will only update the iterates and residuals corresponding
to the indices $[ridx]$ at this step. The other indices within the
blocks correspond to systems whose residuals at this step are too
small to contribute to further convergence of the block system.
The indices of the direction vectors to be updated at this step,
$[iidx]$, may be different from the set $[piidx]$ from the
previous step, depending on the residual update that occurs before
this computation. Below, we detail the steps in the Belos BlockCG
implementation given in Figure \ref{fig:belosblkcg}.

We first describe how to compute the new blocks of iterates
$\bX^{(i+1)}_{[ridx]}$ and residuals $\bR^{(i+1)}_{[ridx]}$ at the
current step. The coefficient matrix $[\alpha_i]$ used in the
computations for $\bX^{(i+1)}_{[ridx]}$ and $\bR^{(i+1)}_{[ridx]}$
is chosen to enforce the definition of the iterates of the method,
$blk{\cal CG}(\bA, \bA)$. That is, at step $i$, the new iterates
$\bX^{(i+1)}_{[ridx]}$ are constructed such that
 \dm
  \bE_{[ridx]}^{(i+1)}  \perp_{\bA}  {\cal K}_{i+1} (\bR^{(0)}, \bA) \\
  ~~~~~{\rm or~equivalently}~~ \\
  \bR_{[ridx]}^{(i+1)}  \perp  {\cal K}_{i+1} (\bR^{(0)}, \bA).
  \edm
Since ${\hat \bP}^{(i)}_{[piidx]} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$, the expression for $[\alpha_i]$ given in Step 1 follows
from enforcing
 \dm
 [ {\bf 0} ] = {\hat \bP}^{(i)T}_{[piidx]} \bR^{(i+1)}_{[ridx]} =
   {\hat \bP}^{(i)T}_{[piidx]} \bR^{(i)}_{[ridx]}
 - ( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \alpha_i ].
 \edm
The coefficient matrix $[\alpha_i]$ has dimensions $numind \times
numcur$. Thus, in the case that deflation has occurred at a
previous step, $[\alpha_i]$ is no longer an $s \times s$ matrix.
Since $\bA$ is SPD, we note that $[\alpha_i]$ is well defined as
long as ${\hat \bP}^{(i)}_{[piidx]}$ has full column rank. This
will be the case as long as our procedure for identifying and then
dropping dependent direction vectors is working properly. The
coefficient matrix $[\alpha_i]$ is actually computed by solving
the block linear system
 \dm
 ( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \alpha_i ] = {\hat \bP}^{(i)T}_{[piidx]} \bR^{(i)}_{[ridx]}.
 \edm
Since $\bA$ is SPD, and ${\hat \bP}^{(i)}_{[piidx]}$ has full
column rank, then $( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat
\bP}^{(i)}_{[piidx]})$ is also SPD, and the above system can be
solved via a Cholesky factorization of the system matrix, and
subsequent forward and back solves for the columns of
$[\alpha_i]$. This is accomplished with LAPACK's routines {\it
POTRF} and {\it POTRS}. Failure to compute $[\alpha_i]$ via the
above routines will result in the termination of the BlockCG
iteration. In this case, the solutions from the previous step will
be returned. After $[\alpha_i]$ is computed, the iterates and
residuals are updated according to Steps 2 and 3.

Once the new residual block $\bR^{(i+1)}_{[ridx]}$ has been
computed, we check for convergence of the individual residuals
(columns of the residual block). The column indices of converged
residuals (those that are smaller in norm than the user specified
tolerance $ctol$) are added to the set of converged residual
indices $[cidx]$, and the $numconv$ is adjusted accordingly. The
iteration will terminate if all residuals in the block have
converged.

This implementation of BlockCG uses the residuals $\bR^{(i+1)}$ to
bring the iteration up to the next higher dimensional Krylov
subspace ${\cal K}_{i+2}(\bR^{(0)}, \bA)$. Only nonzero columns of
residuals are required for this computation. Zero residual columns
will lead to the same zero direction vector columns. So, after
computing the new block of residuals in Step 3, we can delete the
indices corresponding to zero residuals (i.e., those whose norm is
less than $rtol$) from both the sets $[ridx]$ and $[iidx]$, and
adjust the quantities $numcur$ and $numind$. This information will
no longer be useful for creating new linearly independent
direction vectors, and thus, reducing the residual norms further.
Since every time we delete an index from the set $[ridx]$ we also
delete the index from $[iidx]$, it follows that $[iidx] \subseteq
[ridx]$.

The residuals corresponding to the linearly independent columns of
direction vectors, $\bR^{(i+1)}_{[iidx]}$, are used to compute
$\bP^{(i+1)}_{[iidx]}$. We note that in the case the set $[iidx]$
has been adjusted during Step 3, the independent indices form the
previous block of direction vectors $[piidx]$ will be different.
The coefficient matrix $[\beta_i]$ is used in the computation of
the new block of direction vectors given in Step 5. This matrix is
chosen to enforce $\bA$-orthogonality between the new and previous
direction vector blocks. Since ${\hat \bP}^{(i)}_{[piidx]} \in
{\cal K}_{i+1}(\bR^{(0)}, \bA)$, this can be accomplished by
solving
 \dm
 [ {\bf 0} ] = {\hat \bP}^{(i)T}_{[piidx]} \bA \bP^{(i+1)}_{[iidx]} =
   {\hat \bP}^{(i)T}_{[piidx]} \bA \bR^{(i+1)}_{[iidx]}
  + ( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \beta_i ]
 \edm
for $[\beta_i]$. Like the coefficient matrix $[\alpha_i]$, this is
actually computed by solving the block linear system
 \dm
  {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat \bP}^{(i)}_{[piidx]})
 [ \beta_i ] = -{\hat \bP}^{(i)T}_{[piidx]} \bA \bR^{(i+1)}_{[iidx]}.
 \edm
Since a Cholesky factorization of this system matrix has already
been done during the computation of $[\alpha_i]$, we only need to
compute the forward and back solves for the columns of
$[\beta_i]$.

In Step 6, a QR factorization of the new block of direction
vectors $\bP^{(i+1)}_{[iidx]}$ is carried out via an iterated
classical Gram-Schmidt process to check for redundancies. Column
indices corresponding to dependent vectors are dropped from the
set $[iidx]$ and the quantity $numind$ is updated. Subsequent
computations involving ${\hat \bP}^{(i+1)}$ are done only with the
independent columns of ${\hat \bP}^{(i+1)}$, i.e., ${\hat
\bP}^{(i+1)}_{[iidx]}$. If after this step, the set $[iidx] =
\emptyset$, the iteration terminates because numerically, no more
independent direction vectors exist. In this case, the current
solutions will be returned. In the single CG implementation, this
is equivalent to obtaining a zero norm direction vector, which, in
exact arithmetic, means the iteration has converged. Finally, we
set $[piidx] = [iidx]$ and the iteration continues.

We note that the Belos BlockCG implementation handles the case
when the initial residuals have converged or are linearly
dependent. In this case, the indices of the blocks are adjusted in
a similar manner as outlined above (Steps 1 - 6) for a general
step in the iteration. It may be desirable however, to check for
these conditions before calling the block solver routine, because,
otherwise, the solver will start the iteration with a reduced
block size.


{\small
\begin{figure}
\vspace{.05in} \hrule  \vspace{.1in}
\begin{description}
\item[Initialization] ~~~~
\begin{itemize}
 \item Set $[iidx] = [ridx] = [1:s], ~ [cidx] = \emptyset$ \\
 $numind = numcur = s$, and $numconv = 0$
 \item Given $\bX^{(0)}$ a matrix of $s$ initial guesses, \\
       Compute $\bR^{(0)} = \bB - \bA \bX^{(0)}$ a matrix of $s$
       initial residuals \\
       Set $\bP^{(0)} = \bR^{(0)}$ a matrix of $s$ initial
       direction vectors
       \begin{itemize}
       \item Check to see if $\| \br_k^{(0)} \| < ctol$, $\forall
       k \in [cidx]$ \\
       if so, update $[cidx]$ and $numconv$
       \item Check to see if $\| \br_k^{(0)} \| < rtol$, $\forall
       k \in [ridx]$ \\
       If so, update $[iidx]$, $[ridx]$, $numind$, and $numcur$
       \end{itemize}
 \item Compute $\bP^{(0)}_{[iidx]} = {\hat \bP}^{(0)}_{[iidx]}
 \bG_{0}$,~~~~QR factorization
 \begin{itemize}
 \item Update $[iidx]$, and $numind$. Set $[piidx] = [iidx]$
 \end{itemize}
 \end{itemize}
 \item[For $i=0,\ldots,$ until convergence, compute:]~~
 \begin{enumerate}
 \item $[\alpha_i] = ( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat
 \bP}^{(i)}_{[piidx]})^{-1} ( {\hat \bP}^{(i)T}_{[piidx]}
 \bR^{(i)}_{[ridx]})$
 \item $\bX^{(i+1)}_{[ridx]} = \bX^{(i)}_{[ridx]} + {\hat
 \bP}^{(i)}_{[piidx]} [ \alpha_i ]$
 \item $\bR^{(i+1)}_{[ridx]} = \bR^{(i)}_{[ridx]} - \bA {\hat
 \bP}^{(i)}_{[piidx]} [ \alpha_i ]$
 \begin{itemize}
 \item Check to see if $\| \br_k^{(i+1)} \| < ctol$, $\forall
       k \in [cidx]$ \\
 if so, update $[cidx]$ and $numconv$
 \item Check to see if $\| \br_k^{(i+1)} \| < rtol$, $\forall
       k \in [ridx]$ \\
 If so, update $[iidx]$, $[ridx]$, $numind$, and $numcur$
 \end{itemize}
 \item $[\beta_i] = -( {\hat \bP}^{(i)T}_{[piidx]} \bA {\hat
 \bP}^{(i)}_{[piidx]})^{-1} ( {\hat \bP}^{(i)T}_{[piidx]} \bA
 \bR^{(i)}_{[iidx]})$
 \item $\bP^{(i+1)}_{[iidx]} = \bR^{(i+1)}_{[iidx]} + {\hat
 \bP}^{(i)}_{[piidx]} [ \beta_i ]$
 \item $\bP^{(i+1)}_{[iidx]} = {\hat \bP}^{(i+1)}_{[iidx]} G_{i+1}$
 \begin{itemize}
 \item Update $[iidx]$, and $numind$. Set $[piidx] = [iidx]$
 \end{itemize}
 \end{enumerate}
 \end{description}
 \vspace{.1in} \hrule \caption{Belos BlockCG implementation}
 \label{fig:belosblkcg}
 \end{figure}
}

\pagebreak

\section{Belos BlockGMRES}

\subsection{Introduction}
\label{sec:gmresintro}


The BlockGMRES method for solving the block linear system of
equations (\ref{eq:blocklinsys}) is implemented via the
construction of an orthonormal basis
 \dm
 \{ \bU^{(k)} \}_{k=1}^{i+1}
 \edm
for the underlying block Krylov subspaces
  \dm
 {\cal K}_{i+1}(\bR^{(0)},\bA) = {\rm sp}\{\bR^{(0)},\bA
 \bR^{(0)},\ldots,\bA^{i}\bR^{(0)}\}= {\rm sp}\{ \bU^{(1)},
 \ldots, \bU^{(i+1)} \},
 \edm
where,
 \dm
 \begin{array}{rcl}
 \bU^{(j)T} \bU^{(k)} &=& {\bf 0}_{(s,s)}, \quad {\rm
 if}~j \ne k, \\
 \bU^{(j)T} \bU^{(j)} &=& \bI_{(s,s)}.
 \end{array}
 \edm
At each step $i$, the BlockGMRES iterates
 \dm
 \bX^{(i+1)} = \bX^{(0)} + \bZ^{(i)}
 \edm
are defined by choosing $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$ such that the 2-norm of the individual columns of the block
residual $\bR^{(i+1)}$ are minimized.

There are two main steps in a BlockGMRES implementation:
\begin{enumerate}
\item Constructing an orthonormal basis for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$
\item Defining the iterates $\bX^{(i+1)}$ via a block minimization
of the residual norms.
\end{enumerate}
In Section's \ref{sec:blkarn} and \ref{sec:belosblkgmres} we will
describe these processes, and give the details of the Belos
BlockGMRES implementation.


BlockGMRES methods were considered in~\cite{siga:95,siga:96}, but
the implementations required that the number of right-hand sides
$p$ in the block linear system be the same as the block size $s$.
These studies indicated that BlockGMRES methods were not efficient
due to their prohibitive memory and computational requirements. By
formulating the construction of the orthogonal basis for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$ in terms of level 3 BLAS, the
computational requirements are not excessive. Moreover, a robust
and stable algorithm is realized where the application of $\bA$ is
to a matrix.  Thus, as the cost of applying $\bA$ increases
relative to the cost of the orthogonalizations, the efficiency of
our BlockGMRES algorithm will scale with the cost of $\bA
\bR^{(0)}$.

We describe additional notation used in this section below. The
$j$'th canonical basis vector is denoted by ${\bf \epsilon}_j$,
the $j$'th column of the identity matrix, and
 $\bE_j \equiv \left[\begin{array}{ccc}
                    {\bf \epsilon}_{(j-1)s+1} & \cdots & {\bf \epsilon}_{js}
                   \end{array}\right] ,$
where $s$ is a positive integer, and is also the block size used
by the solver. A matrix of lower bandwidth $s$ will be called a
banded upper Hessenberg matrix.  We drop ``upper'' when the
context is clear. Omission of the word {\em band} implies that the
block size is one. We say that a band Hessenberg matrix is
unreduced if all the elements on the $s$'th subdiagonal are
nonzero.

We now define several matrices that will prove useful.  $\bH_j$
denotes a band Hessenberg matrix of order $sj$ of lower bandwidth
$s$; and $\bF^{(j)}$ and $\bU^{(j)}$ denote matrices with $n$ rows
and $s$ columns, where the superscript acts as an iteration index.
On the other hand, $\bV_j$ denotes a matrix with $n$ rows and $sj$
columns. $\bU^{(j)}$ denotes the $j$'th block of $s$ vectors of
$\bV_m$, and $\bG_{i,j}$ denotes the square matrix of order $s$
located in the $i,j$'th block of order $s$ of $\bH_m.$ Note that
$\bG_{j+1,j}$ is an upper triangular matrix. These matrices will
define the dimensions of other matrices used throughout this
section.


\subsection{Belos BlockGMRES: Constructing an Orthonormal Basis}
\label{sec:blkarn}

An orthonormal basis for the block Krylov subspaces
 \dm
 {\cal K}_{i+1}(\bR^{(0)}, \bA) = {\rm sp} \{ \bR^{(0)}, \bA
 \bR^{(0)}, \ldots, \bA^i \bR^{(0)} \}
 \edm
can be constructed via a block Arnoldi process.

 Let $\bA $ be a matrix of order $n$ and $s > 0$ be the block size.
 We say that
 \begin{equation}
 \bA \bV_m = \bV_m \bH_m + \bF^{(m)} \bE^T_m \label{eq:blk_arn_red}
 \end{equation}
is a block Arnoldi reduction of length $m$ when
$\bV_m^T\bA\bV_m=\bH_m$ is a banded upper Hessenberg matrix,
$\bV_m^T\bV_m={\bf I}_{(m s,m s)}$, and $\bV_m^T\bF^{(m)}={\bf
0}_{(ms,s)}.$ Let $\bU^{(m+1)} \bG_{m+1,m}$ denote the QR
factorization of $\bF^{(m)}.$ We have
\begin{eqnarray*}
\bA \, \bV_m
     &=& \left[\begin{array}{ccc}
         \bU^{(1)} & \cdots & \bU^{(m)} \end{array}\right]
    \left[\begin{array}{cccc}
      \bG_{1,1}&\cdots& \cdots & \bG_{1,m}\\
      \bG_{2,1}&  \ddots& \vdots & \vdots \\
       \vdots & \ddots & \vdots & \vdots \\
     {\bf 0}& \cdots &  \bG_{m,m-1} & \bG_{m,m} \end{array}\right]\\
    & & + \bU^{(m+1)} \bG_{m+1,m}\bE_m^T. \\ \label{block_not} \nonumber
\end{eqnarray*}
or equivalently,
 \dm
 \begin{array}{rcl}
 \bA \bV_m &=& \left[ \bV_m ~~ \bU^{(m+1)} \right] \left[
 \begin{array}{c}
 \bH_m \\
 \bG_{m+1,m} \end{array} \right] \\ \\
 ~&=& \bV_{m+1} {\widetilde \bH}_m,
 \end{array}
 \edm
where, the columns of $\bV_m$ form an orthonormal basis for the
block Krylov subspace ${\cal K}_m ( \bR^{(0)}, \bA)$.

If $ m > \bar{m} \equiv \mbox{ceiling}(n/s)$, then $\bF^{(m)} =
{\bf 0}_{(n,s)}$ and $\bH_{\bar{m}}$ is the orthogonal reduction
of $\bA$ into banded upper Hessenberg form.  If $\bF^{(m)}$ is of
full rank and the diagonal elements of $\bG_{m+1,m}$ are positive,
an extension of the implicit Q theorem \cite{govl:96} gives that
$\bF^{(m)}$ is (uniquely) specified by the starting block
$\bU^{(1)}.$ The algorithm in Figure~\ref{fig:blk_arn_red_alg}
describes the process for extending a block Arnoldi reduction,
thus obtaining an orthonormal basis for the next higher
dimensional block Krylov subspace. Below, we detail the steps of
this process.

{\small
\begin{figure}
\vspace{.05in} \hrule \vspace{.1in}
\begin{description}
\item[Start] ~~
Let $\bA \bV_m = \bV_m \bH_m + \bF^{(m)} \bE^T_m$ be a length-$m$
block Arnoldi reduction from the previous step, where $\bV_m^T
\bF^{(m)}= {\bf 0}_{(ms,s)}$,~$\bU^{(m+1)} \bG_{m+1,m} =
\bF^{(m)}$, and $ \bV_{m+1} = \left[ \bV_m ~~ \bU^{(m+1)}
\right]$.

\item[Compute:] ~~

\begin{enumerate}

\item $ \bW^{(m+1)} = \bA \bU^{(m+1)} $.

\item If $depflg = false$:
\begin{enumerate}
\item $ \begin{array}{rcl} \bF^{(m+1)} &=& \bW^{(m+1)} - \bV_{m+1} \left[ \bV_{m+1}^T
\bW^{(m+1)}
\right] \\
~&=& \bW^{(m+1)} - \left[ \bV_m ~~ \bU^{(m+1)} \right] \left[
\begin{array}{c} \bV_m^T \bW^{(m+1)} \\
                 \bG_{m+1,m+1}  \end{array} \right],
\end{array} $ \\
where, $\bG_{m+1,m+1} = \bU^{(m+1)T} \bW^{(m+1)}. $

\item Check for dependencies between $\bW^{(m+1)}$ and $\bV_{m+1}$.
\begin{itemize}

\item If no dependencies exist, compute
\begin{itemize}
\item $ \bF^{(m+1)} = \bU^{(m+2)} \bG_{m+2,m+1}$
     \begin{itemize}

      \item If the columns of $\bF^{(m+1)}$ are linearly
      independent, set \\

      $\bH_{m+1} = \left[\begin{array}{cc}
                             \bH_m & \bV_m^T \bW^{(m+1)} \\
                             \bG_{m+1,m} \bE_m^T & \bG_{m+1,m+1}
                        \end{array}\right]$, \\
                         ~~~~~~${\widetilde \bH}_{m+1} = \left[
                        \begin{array}{c} \bH_{m+1} \\
             \bG_{m+2,m+1} \end{array} \right] $, and \\
      $\bV_{m+2} = \left[ \bV_{m+1} ~~ \bU^{(m+2)} \right]$.


      \item If the columns of $\bF^{(m+1)}$ are linearly
      dependent, set $depflg = true$, and recompute $\bF^{(m+1)}$ using step 3.

      \end{itemize}
\end{itemize}

\item If dependencies exist
\begin{itemize}
\item Set $depflg = true$, recompute $\bF^{(m+1)}$
      using step 3 below.
\end{itemize}

\end{itemize}

\end{enumerate}

\item If $depflg = true$: \\
Set $\bV_{prev} = \bV_{m+1}$ ~(all previous Arnoldi vectors) \\
For $j = 1, \ldots, s$,~~Compute:
\begin{enumerate}

\item ${\bf f}_j^{(m+1)} = \bw_j^{(m+1)} - \bV_{prev} \left[ \bV_{prev}^T \bw_j^{(m+1)} \right]$.
\item Set the $(ms+j)$'th column of ${\widetilde \bH}_{m+1}$ to
$\bV_{prev}^T \bw_j^{(m+1)}$.
\item Compute $\bu_j^{(m+2)}$.
\item Set $\bV_{prev} = \left[ \bV_{prev} ~~ \bu_j^{(m+2)} \right]$.

\end{enumerate}

\end{enumerate}
\end{description}
\vspace{.1in} \hrule \caption{Extending a Block Arnoldi Reduction}
\label{fig:blk_arn_red_alg}
\end{figure}
}


Step 1 allows the application of $\bA$ to a group of $s$ vectors.
This might prove essential when accessing $\bA$ is expensive.
Clearly, the goal is to amortize the cost of applying $\bA$ over
several vectors.

Step's 2 and 3 detail two separate processes for computing a new
block of orthonormal vectors at a given step in the iteration. The
process outlined in Step 2 is more efficient and is used by the
Belos BlockGMRES solver until dependencies are detected in the
Arnoldi basis vectors. This may not occur at all, or if it does,
it usually occurs towards the very end of the iteration when some
of the systems comprising the block converge before others. When a
dependency is detected, a flag is set, $depflg = true$, indicating
this is the case, and relegating the construction of the current
and remaining blocks of basis vectors to the process given in Step
3. This decision was made because preliminary testing showed the
approach in Step 3 was more stable under these circumstances. The
solver routines, {\it BlkOrth} and {\it BlkOrthSing}, implement
the processes given in Step's 2 and 3, respectively.

As written, Step 2a is one step of block classical Gram-Schmidt
(bCGR). This allows the use of the Level 3 {\small
BLAS}~\cite{dddh:90} matrix-matrix multiplication subroutine {\tt
\_GEMM} for computing $\bV_{m+1}^T \bW^{(m+1)}.$ To ensure the
orthogonality of $\bF^{(m+1)}$ with $\bV_{m+1}$, a second step of
bCGR is performed. Once $\bF^{(m+1)}$ is computed, we check for
rank deficiencies in Step 2b. If any of the columns of
$\bW^{(m+1)}$~ ($\bw_j^{(m+1)},~j=1,\ldots,s$) are dependent on
previous Arnoldi vectors (columns of $\bV_{m+1}$), then the sine
of the angle between $\bw_j^{(m+1)}$ and the subspace spanned by
the columns of $\bV_{m+1}$ will be very small. Thus, for $j =
1,\ldots,s$, we check to see if
 \dm
 sin~\theta = \| {\bf f}_j^{(m+1)} \| / \| \bw_j^{(m+1)} \| < {\it btol},
 \edm
where, for this purpose we have chosen $btol = 10 * sqrt(eps)$,
and where $eps$ denotes the machine precision. This tolerance is
set in the routine {\it SetGmresBlkTols}. If any dependencies are
detected between the vectors in $\bW^{(m+1)}$ (the component that
brings the iteration up to the next higher dimensional block
Krylov subspace) and the previous Arnoldi vectors, $depflg$ is set
to true, and $\bF^{(m+1)}$ is recomputed according to the process
outlined in Step 3 below.

If no dependencies are detected, a QR factorization is used to
obtain an orthonormal set $\bU^{(m+2)}$ from $\bF^{(m+1)}$. This
procedure is also used to check for dependencies within the blocks
$\bF^{(m+1)}$. The QR factorization in Step 2b is computed via an
iterated classical Gram-Schmidt (CGS) process. A simple test in
DGKS~\cite{dgks:76} is used to determine whether a second
orthogonalization (correction) step is needed to ensure
orthogonality (see \cite{Le95} and \cite{lesy:98} for details).
One benefit of this scheme is that it allows the use of the Level
2 {\small BLAS}~\cite{ddhh:88} matrix-vector multiplication
subroutine {\tt \_GEMV}. After constructing column $j$ of
$\bU^{(m+2)}$ by this process, the sine of the angle between ${\bf
f}_j^{(m+1)}$ and the space spanned by the previous $j-1$ columns
of $\bU^{(m+2)}$ is measured. If this is smaller than $btol$, then
we conclude that ${\bf f}_j^{(m+1)}$ is linearly dependent on the
previous $j-1$ columns of $\bU^{(m+2)}$. If dependencies are
detected during this process, $depflg$ is set to true, and the
block $\bF^{(m+1)}$ is recomputed according to Step 3 below.

The block orthogonalization process outlined in Step 2 allows for
the computation of the matrix coefficients $\bV_{m+1}^T
\bW^{(m+1)}$ to be separated from the QR factorization of
$\bF^{(m+1)}.$ This is advantageous in that it reduces the cost of
I/O by a factor of the block size and increase the amount of
floating-point operations per memory reference.

In the case that a rank deficiency is produced during the block
orthogonalization process (described in Step 2) at either the
current, or a previous step in the iteration, $depflg$ will be set
to true. The new block of Arnoldi vectors will be constructed one
at a time via the process outlined in Step 3.

In Step 3, we denote the matrix whose columns correspond to all
previously generated Arnoldi vectors as $\bV_{prev}$. The
orthogonalization process described in Step 3, iterates over the
$s$ columns of $\bW^{(m+1)}$ producing the corresponding columns
of $\bF^{(m+1)}$ and ${\widetilde \bH}_{m+1}$ via an iterated CGS
process. If needed, a correction step is used to ensure
orthogonality of ${\bf f}_j^{(m+1)}$ to all previous Arnoldi
vectors. To check for dependencies, the sine of the angle between
$\bw_j^{(m+1)}$ and all previous Arnoldi vectors is measured.,
i.e.,
 \dm
 sin~\theta = \| {\bf f}_j^{(m+1)} \| / \| \bw_j^{(m+1)} \|.
 \edm
If this quantity is smaller than $stol$, then $\bw_j^{(m+1)}$ is
linearly dependent on the columns of $\bV_{prev}$. Here, the value
$stol = 10 * eps$, is smaller (tighter) than the tolerance $btol$
used during the block orthogonalization process in Step 2. The
tolerance $stol$ is also computed by the routine {\it
SetGmresBlkTols} during the solver initialization phase. If
$\bw_j^{(m+1)}$ is independent, then ${\bf f}_j^{(m+1)}$ is
normalized to produce the $j$'th column $\bu_j^{(m+2))}$ of
$\bU^{(m+2)}$, and added to the set of previous orthonormal
Arnoldi vectors, $\bV_{prev} = \left[ \bV_{prev} ~~ \bu_j^{(m+2)}
\right]$. If $\bw_j^{(m+1)}$ is dependent on previous vectors, the
subdiagonal element in the $(ms+j)$'th column of ${\widetilde
\bH}_{m+1}$ is set to zero, and a random vector is generated and
subsequently orthogonalized against all previous Arnoldi vectors.
The resulting vector ${\hat \bu}_j^{(m+2)}$ is normalized
producing the $j$'th column of $\bU^{(m+2)}$, and is then
augmented to the matrix $\bV_{prev}$. Note that after all $s$
columns have been orthogonalized, $\bV_{prev} = \bV_{m+2}$. This
approach for producing $\bV_{m+2}$ is a modification of that used
by Ruhe in \cite{ruhe:79} for symmetric matrices. However, this
process is more efficient than that given in \cite{ruhe:79} in
that it allows for the matrix vector multiplication to take place
between $\bA$ and a group of vectors $\bU^{(m+1)}$ instead of
performing $s$ individual matrix vector multiplications.



\subsection{Belos BlockGMRES: Defining the Iterates}
\label{sec:belosblkgmres}

This section gives the details of how to update the iterates
defined by the Belos BlockGMRES iteration. This development is
essentially the same as that given in (\cite{Sa96}, pp. 200-201),
and extends the results in \cite{SS86} to a block implementation.

Let $\bX^{(0)}$ be the matrix of $s$ vectors (the guesses) and let
$\bR^{(0)} = \bB - \bA \bX^{(0)}$ be a matrix of full column rank
(the initial residuals). Compute the QR factorization $\bU^{(1)}
\bG_{1,0} = \bR^{(0)}$ where $\bU^{(1)}$ has $s$ orthonormal
columns. Use the algorithm in Figure~\ref{fig:blk_arn_red_alg} to
build a block Arnoldi reduction $\bA \bV_{i+1} = \bV_{i+1}
\bH_{i+1} + \bF^{(i+1)} \bE^T_{i+1}$ of length $i+1$ and let
 \dm
 \widetilde{\bH}_{i+1} =
 \left[
 \begin{array}{c}
                            \bH_{i+1} \\
                            \bG_{i+2,i+1} \end{array} \right].
 \edm
The columns of
 \dm
 \bV_{i+1} = \left[ \bU^{(1)} \ldots \bU^{(i+1)} \right]
 \edm
form an orthonormal basis for ${\cal K}_{i+1}(\bR^{(0)}, \bA)$.
Denote $\bR^{(i+1)} = \left[ \br_1^{(i+1)}, \ldots, \br_s^{(i+1)}
\right]$ as the $i+1$'st residual block. The BlockGMRES iterates
 \dm
 \bX^{(i+1)} = \bX^{(0)} + \bZ^{(i)}
 \edm
are defined by choosing $\bZ^{(i)} \in {\cal K}_{i+1}(\bR^{(0)},
\bA)$ such that
 \eq
 \|\br_j^{(i+1)}\|_2 ~~~{\rm
 is~minimized}~, {\rm for}~ j = 1,\ldots, s.
 \label{eq:gmresmin}
 \eeq
Since the columns of $\bV_{i+1}$ form a basis for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$, we can write $\bZ^{(i)} = \bV_{i+1}
\bY$, for some coefficient matrix $\bY$ with $(i+1)s$ rows and $s$
columns.

Simple manipulation gives
 \eq
 \begin{array}{rcl}
 \bR^{(i+1)} & = & \bB - \bA( \bX^{(0)} + \bV_{i+1} \bY) \\
      & = & \bR^{(0)} -  \bA \bV_{i+1} \bY \\
      & = & \bU^{(1)} \bG_{1,0} - \bA \bV_{i+1} \bY \\
      & = & \bV_{i+2}(\bE_1 \bG_{1,0} - \widetilde{\bH}_{i+1} \bY).
 \end{array} \label{eq:gmresresids}
 \eeq
Denote the $j$'th column of $\bE_1 \bG_{1,0}$ as ${\bf g}_j$, with
length $(i+2)s$.

The individual BlockGMRES approximations $\bx_j^{(i+1)}, j = 1,
\ldots, s$, have the form
 \dm
 \bx_j^{(0)} + \bV_{i+1} \by_j, \quad {\rm for}~ j = 1, \ldots, s,
 \edm
where $\by_j$ is a vector of length $(i+1)s$. Thus, in BlockGMRES
the vectors $\by_j$ are chosen to minimize the individual columns
in (\ref{eq:gmresresids}). Since $\bV_{i+2}$ has orthonormal
columns, this is equivalent to solving the $s$ least squares
problems
\begin{equation}
{\rm min}\| {\bf g}_j - \widetilde{\bH}_{i+1} \by_j \|_2, \quad
{\rm for}~ j = 1, \ldots, s.
\end{equation}
Assuming that $\widetilde{\bH}_{i+1}$ is of full column rank,
denote by $\bY_{i+1}$ the matrix whose columns are the $s$ unique
minimizers. If $\widetilde{\bQ} \widetilde{\bT}$ is a QR
factorization of $\widetilde{\bH}_{i+1} $ with $\widetilde{\bT} =
\left[
\begin{array}{l} \bT_{i+1} \\ {\bf 0}_{(s,(i+1)s)} \end{array}
\right]$ an upper triangular matrix with $(i+2)s$ rows and
$(i+1)s$ columns, then the $s$ least squares solutions are given
by solving
\begin{equation}
\bT_{i+1} \bY_{i+1} = \bS_1 \,\,\,\mbox{with} \,\,\,
\widetilde{\bQ}^T \bE_1 \bG_{1,0} = \left[ \begin{array}{c} \bS_1
\\ \bS_2
\end{array} \right] \begin{array}{l} \} \, (i+1)s \mbox{ rows}
\\ \} \, s \mbox{ rows} \end{array}
\end{equation}

Hence, $\bE_1 \bG_{1,0} - \widetilde{\bH}_{i+1} \bY_{i+1} =
\widetilde{\bQ} \left[
\begin{array}{l} {\bf 0}_{((i+1) \cdot s, s)} \\ \bS_2 \end{array}
\right] $ is the projection of $\bE_1 \bG_{1,0}$ onto the
orthogonal compliment of the range of $\widetilde{\bH}_{i+1}$. And
thus, if
 \dm
 \bX^{(i+1)} = \bX^{(0)} + \bV_{i+1} \bY_{i+1}
 \edm
and
$$\bR^{(i+1)} = \bB - \bA \bX^{(i+1)} = \bV_{i+2}(\bE_1
\bG_{1,0} - \widetilde{\bH}_{i+1} \bY_{i+1}),$$ it follows that
\begin{equation}
\| \br_j^{(i+1)} \|_2 = \|{\bf g}_j - \widetilde{\bH}_{i+1}
\bY_{i+1} \|_2 = \|\bs_j\|_2, ~{\rm for}~ j = 1,\ldots,s,
\end{equation}
where $\bs_j$ denotes the $j$'th column of $\bS_2$. This is
significant as it allows us to easily determine the quality of the
GMRES approximation during every iteration without having to
update the iterates $\bX^{(i+1)}$, because $\bS_2$ is easily
computed during the $s$ least squares solutions.

{\small
\begin{figure}[hbt]
\vspace{.05in} \hrule \vspace{.1in}
\begin{description}
\item[Initialization]
Set $ \bX^{(cur)} = \bX^{(0)}$ to be the  matrix of current
initial guesses, and set $restartiter=0$
\item[While] ($restartiter < numrestarts$, and not converged)
\begin{itemize}
\item Set $depflg = false$
\item Set $\bR^{(0)} = \bB - \bA \bX^{(cur)}$
\item Compute the QR factorization $\bU^{(1)} \bG_{1,0} =
\bR^{(0)}$
\item Set $\bV_1 = \left[ \bU^{(1)} \right]$, and $i=0$
\end{itemize}
\begin{description}
\item [While] ($i < maxits$, and not converged)
\begin{enumerate}
\item  Extend the block Arnoldi reduction by one so that a
length $i+1$ block Arnoldi reduction now exists for ${\cal
K}_{i+1}(\bR^{(0)}, \bA)$

\item Compute the $s$ least squares minimizers,\\
${\rm min}\| {\bf g}_j - \widetilde{\bH}_{i+1} \by_j \|_2, \quad
{\rm for}~ j = 1, \ldots, s, $  via a QR factorization
$\widetilde{\bQ} \widetilde{\bT}$ of $\widetilde{\bH}_{i+1}$ and
store in the matrix $\bY_{i+1}$

\item For $j = 1,\ldots, s$, compute $\| \br_j^{(i+1)} \|_2 = \| \bs_j \|_2$, where
$\bs_j$ is the $j$'th column of $\bS_2$ (an order $s$ sub-matrix
in the last $s$ rows of $\widetilde{\bQ}^T \bE_1 \bG_{1,0}$)

\begin{itemize}
\item If $\| \br_j^{(i+1)} \|_2 $ is small enough, for every $j$, or $maxits$ is reached,
compute the $s$ approximations $\bX^{(i+1)} = \bX^{(0)} +
\bV_{i+1} \bY_{i+1}$
\item Set $i = i+1$
\end{itemize}

\end{enumerate}

\item [Set] $\bX^{(cur)} = \bX^{(i+1)}$, and $restartiter =
restartiter + 1$
\end{description}

\end{description}
\vspace{.1in} \hrule \caption{Belos BlockGMRES implementation}
\label{fig:bGMRES}
\end{figure}
}

The Belos BlockGMRES implementation contains a restart mechanism,
where at each restart iteration, $restariter$, the final iterates
(if not converged sooner) are computed from a block Krylov
subspace of dimension $maxits$. The iteration is then restarted
using this solution as an initial guess for the next
$restartiter$. We have observed that if the iteration is close to
convergence, the new initial residuals produced after a restart
cycle can be linearly dependent. The Belos implementation handles
the case when this occurs during the initial QR factorization of
the residual block.

We summarize the Belos BlockGMRES algorithm in
Figure~\ref{fig:bGMRES}. This implementation is a careful
reorganization of the scheme due to B. Vital~\cite{vita:90} given
in the appendix of~\cite{siga:95}. There are three major
differences between our scheme and that listed in~\cite{siga:95}.
The first is that we use a classical Gram--Schmidt scheme via the
level 3 BLAS instead of a modified Gram--Schmidt scheme. The
second difference is that we do not explicitly compute
$\bR^{(i+1)}$ via an application of $\bA$. The final distinction
is that we consider using a BlockGMRES iteration regardless of the
number of right hand sides.

\pagebreak

\begin{thebibliography}{AAAA99}

\bibitem[1]{AMS90} S. F. Ashby, T. A. Manteuffel and P. E.
Saylor, {\em A Taxonomy for Conjugate Gradient Methods}, SIAM J.
Numer. Anal., 27 (1990), pp. 1542-1568.

\bibitem[2]{bcrr:98} J. Baglama, D. Calvetti, L. Reichel, and A.
Ruttan, {\em Computation of a few close eigenvalues of a large
matrix with application to liquid crystal modeling}, Journal of
Computational Physics, 146 (1998), pp. 203-226.

\bibitem[3]{dgks:76} J. Daniel, W. B. Gragg, L. Kaufman, and G. W.
Stewart, {\em Reorthogonalization and stable algorithms for
updating the Gram-Schmidt QR factorization}, Mathematics of
Computation, 30 (1976), pp. 772-795.

\bibitem[4]{dddh:90} J. J. Dongarra, J. DuCroz, I. S. Duff, and S.
Hammarling, {\em A set of Level 3 Basic Linear Algebra
Subprograms}, ACM Trans. Mathematical Software, 16 (1990), pp.
1-17.

\bibitem[5]{ddhh:88} J. J. Dongarra, J. DuCroz, S. Hammarling, and
R. J. Hanson, {\em An extended set of Fortran Basic Linear Algebra
Subprograms}, ACM Trans. Mathematical Software, 14 (1988), pp.
1-17.

\bibitem[6]{DUB01} A. Dubrulle, {\em Retooling The Method Of
Block Conjugate Gradients}, ETNA, 12 (2001), pp. 216-233.

\bibitem[7]{FOP95} Y. Feng, D. Owen, and D. Peric, {\em A
block conjugate gradient method applied to linear systems with
multiple right-hand sides}, Computer Methods in Applied Mechanics
and Engineering, 127 (1995), pp. 203-215.

\bibitem[8]{govl:96} G. H. Golub and C. F. V. Loan, {\em Matrix
Computations}, Johns Hopkins University Press, Baltimore, third
ed., 1996.

\bibitem[9]{HS52} M. Hestenes, and E. Stiefel, {\em Methods of
Conjugate Gradients for Solving Linear Systems}, J. Research of
the National Bureau of Standards, Vol. 49, No. 6 (1952), pp.
409-436.

\bibitem[10]{Le95} R. B. Lehoucq, {\em Analysis and Implementation
of an Implicitly Restarted Arnoldi Iteration}, Ph.D. Thesis, Rice
University, Houston, TX, 1995.

\bibitem[11]{lesy:98} R. B. Lehoucq, D. C. Sorensen, and C. Yang,
{\em ARPACK USERS GUIDE: Solution of Large Scale Eigenvalue
Problems with Implicitly Restarted Arnoldi Methods}, SIAM,
Philidelphia, PA, 1998.

\bibitem[12]{NY95} A. Nikishin, and Y. Yeremin, {\em Variable
Block CG Algorithms For Solving Large Sparse Symmetric Positive
Definite Linear Systems On Parallel Computers, I: General
Iterative Scheme}, SIAM J. Matrix Anal. Appl., Vol. 16, No 4,
(1995), pp. 1135-1153.

\bibitem[13]{Ol80} D. P. O'Leary, {\em The Block Conjugate Gradient
Algorithm and Related Methods}, Linear Algebra And Its
Applications, 29 (1980), pp. 293-322.

\bibitem[14]{ruhe:79} A. Ruhe, {\em Implementation aspects of band
Lanczos algorithms for computation of eigenvalues of large sparse
symmetric matrices}, Mathematics of Computation, 33 (1979), pp.
680-687.

\bibitem[15]{SS86} Y. Saad, and M. H. Schultz, {\em GMRES: A
generalized minimal residual algorithm for solving nonsymmetric
linear systems}, SIAM J. Scientific Computing, 7 (1986), pp.
856-869.

\bibitem[16]{Sa96} Y. Saad, {\em Iterative Methods for Sparse
Linear Systems}, PWS Publishing Company, Boston, 1996.

\bibitem[17]{siga:95} V. Simoncini and E. Gallopoulos, {\em An
iterative method for nonsymmetric systems with multiple right-hand
sides}, SIAM J. Scientific Computing, 16 (1995), pp. 917-933.

\bibitem[18]{siga:96} V. Simoncini and E. Gallopoulos, {\em A
hybrid block GMRES method for nonsymmetric systems with multiple
right-hand sides}, Journal of Computational and Applied
Mathematics, 66 (1996), pp. 457-469.

\bibitem[19]{vita:90} B. Vital, {\em Etude de quelques m\`{e}thodes de
r\`{e}solution de probl\'{e}mes lin\`{e}aires de grande taille sur
multiprocessers}, PhD thesis, Universit\'{e} de Rennes I, Rennes,
France, 1990.


\end{thebibliography}

\end{document}
