@article {,
	title = {Ginkgo: A Modern Linear Operator Algebra Framework for High Performance Computing},
	journal = {ACM Transactions on Mathematical Software},
	volume = {48},
	year = {2022},
	month = {2022-03},
	pages = {1 - 33},
	abstract = {In this article, we present Ginkgo, a modern C++ math library for scientific high performance computing. While classical linear algebra libraries act on matrix and vector objects, Ginkgo{\textquoteright}s design principle abstracts all functionality as {\textquotedblleft}linear operators,{\textquotedblright} motivating the notation of a {\textquotedblleft}linear operator algebra library.{\textquotedblright} Ginkgo{\textquoteright}s current focus is oriented toward providing sparse linear algebra functionality for high performance graphics processing unit (GPU) architectures, but given the library design, this focus can be easily extended to accommodate other algorithms and hardware architectures. We introduce this sophisticated software architecture that separates core algorithms from architecture-specific backends and provide details on extensibility and sustainability measures. We also demonstrate Ginkgo{\textquoteright}s usability by providing examples on how to use its functionality inside the MFEM and deal.ii finite element ecosystems. Finally, we offer a practical demonstration of Ginkgo{\textquoteright}s high performance on state-of-the-art GPU architectures.},
	issn = {0098-3500},
	doi = {10.1145/3480935},
	url = {https://dl.acm.org/doi/10.1145/3480935},
	author = {Anzt, Hartwig and Cojean, Terry and Flegar, Goran and G{\"o}bel, Fritz and Gr{\"u}tzmacher, Thomas and Nayak, Pratik and Ribizel, Tobias and Tsai, Yuhsiang Mike and Quintana-Ort{\'\i}, Enrique S}
}
@article {,
	title = {Load-Balancing Sparse Matrix Vector Product Kernels on GPUs},
	journal = {ACM Transactions on Parallel Computing},
	volume = {7},
	year = {2020},
	month = {2020-03},
	abstract = {Efficient processing of Irregular Matrices on Single Instruction, Multiple Data (SIMD)-type architectures is a persistent challenge. Resolving it requires innovations in the development of data formats, computational techniques, and implementations that strike a balance between thread divergence, which is inherent for Irregular Matrices, and padding, which alleviates the performance-detrimental thread divergence but introduces artificial overheads. To this end, in this article, we address the challenge of designing high performance sparse matrix-vector product (SpMV) kernels designed for Nvidia Graphics Processing Units (GPUs). We present a compressed sparse row (CSR) format suitable for unbalanced matrices. We also provide a load-balancing kernel for the coordinate (COO) matrix format and extend it to a hybrid algorithm that stores part of the matrix in SIMD-friendly Ellpack format (ELL) format. The ratio between the ELL- and the COO-part is determined using a theoretical analysis of the nonzeros-per-row distribution. For the over 2,800 test matrices available in the Suite Sparse matrix collection, we compare the performance against SpMV kernels provided by NVIDIA{\textquoteright}s cuSPARSE library and a heavily-tuned sliced ELL (SELL-P) kernel that prevents unnecessary padding by considering the irregular matrices as a combination of matrix blocks stored in ELL format.},
	doi = {https://doi.org/10.1145/3380930},
	author = {Hartwig Anzt and Terry Cojean and Chen Yen-Chen and Jack Dongarra and Goran Flegar and Pratik Nayak and Stanimire Tomov and Yuhsiang M. Tsai and Weichung Wang}
}
@conference {,
	title = {Multiprecision Block-Jacobi for Iterative Triangular Solves},
	booktitle = {European Conference on Parallel Processing (Euro-Par 2020)},
	year = {2020},
	month = {2020-08},
	publisher = {Springer},
	organization = {Springer},
	abstract = {Recent research efforts have shown that Jacobi and block-Jacobi relaxation methods can be used as an effective and highly parallel approach for the solution of sparse triangular linear systems arising in the application of ILU-type preconditioners. Simultaneously, a few independent works have focused on designing efficient high performance adaptive-precision block-Jacobi preconditioning (block-diagonal scaling), in the context of the iterative solution of sparse linear systems, on manycore architectures. In this paper, we bridge the gap between relaxation methods based on regular splittings and preconditioners by demonstrating that iterative refinement can be leveraged to construct a relaxation method from the preconditioner. In addition, we exploit this insight to construct a highly-efficient sparse triangular system solver for graphics processors that combines iterative refinement with the block-Jacobi preconditioner available in the Ginkgo library.},
	keywords = {Block-Jacobi, graphics processing units (GPUs), incomplete factorization preconditioning, multiprecision, sparse linear algebra},
	doi = {https://doi.org/10.1007/978-3-030-57675-2_34},
	author = {Fritz Goebel and Hartwig Anzt and Terry Cojean and Goran Flegar and Enrique S. Quintana-Orti}
}
@article {1319,
	title = {Adaptive Precision in Block-Jacobi Preconditioning for Iterative Sparse Linear System Solvers},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {31},
	number = {6},
	year = {2019},
	month = {2019-03},
	pages = {e4460},
	abstract = {Summary We propose an adaptive scheme to reduce communication overhead caused by data movement by selectively storing the diagonal blocks of a block-Jacobi preconditioner in different precision formats (half, single, or double). This specialized preconditioner can then be combined with any Krylov subspace method for the solution of sparse linear systems to perform all arithmetic in double precision. We assess the effects of the adaptive precision preconditioner on the iteration count and data transfer cost of a preconditioned conjugate gradient solver. A preconditioned conjugate gradient method is, in general, a memory bandwidth-bound algorithm, and therefore its execution time and energy consumption are largely dominated by the costs of accessing the problem{\textquoteright}s data in memory. Given this observation, we propose a model that quantifies the time and energy savings of our approach based on the assumption that these two costs depend linearly on the bit length of a floating point number. Furthermore, we use a number of test problems from the SuiteSparse matrix collection to estimate the potential benefits of the adaptive block-Jacobi preconditioning scheme.},
	keywords = {adaptive precision, block-Jacobi preconditioning, communication reduction, energy efficiency, Krylov subspace methods, sparse linear systems},
	doi = {https://doi.org/10.1002/cpe.4460},
	url = {https://onlinelibrary.wiley.com/doi/abs/10.1002/cpe.4460},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Nicholas J. Higham and Enrique S. Quintana-Orti}
}
@conference {1367,
	title = {Are we Doing the Right Thing? {\textendash} A Critical Analysis of the Academic HPC Community},
	booktitle = {2019 IEEE International Parallel and Distributed Processing Symposium Workshops (IPDPSW)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {Like in any other research field, academically surviving in the High Performance Computing (HPC) community generally requires to publish papers, in the bast case many of them and in high-ranked journals or at top-tier conferences. As a result, the number of scientific papers published each year in this relatively small community easily outnumbers what a single researcher can read. At the same time, many of the proposed and analyzed strategies, algorithms, and hardware-optimized implementations never make it beyond the prototype stage, as they are abandoned once they served the single purpose of yielding (another) publication. In a time and field where high-quality manpower is a scarce resource, this is extremely inefficient. In this position paper we promote a radical paradigm shift towards accepting high-quality software patches to community software packages as legitimate conference contributions. In consequence, the reputation and appointability of researchers is no longer based on the classical scientific metrics, but on the quality and documentation of open source software contributions - effectively improving and accelerating the collaborative development of community software.},
	doi = {10.1109/IPDPSW.2019.00122},
	author = {Hartwig Anzt and Goran Flegar}
}
@article {1369,
	title = {A Customized Precision Format Based on Mantissa Segmentation for Accelerating Sparse Linear Algebra},
	journal = {Concurrency and Computation: Practice and Experience},
	volume = {40319},
	year = {2019},
	month = {2019-01},
	issn = {1532-0626},
	doi = {https://doi.org/10.1002/cpe.5418},
	author = {Thomas Gruetzmacher and Terry Cojean and Goran Flegar and Fritz G{\"o}bel and Hartwig Anzt}
}
@conference {1436,
	title = {ParILUT {\textendash} A Parallel Threshold ILU for GPUs},
	booktitle = {IEEE International Parallel and Distributed Processing Symposium (IPDPS)},
	year = {2019},
	month = {2019-05},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Rio de Janeiro, Brazil},
	abstract = {In this paper, we present the first algorithm for computing threshold ILU factorizations on GPU architectures. The proposed ParILUT-GPU algorithm is based on interleaving parallel fixed-point iterations that approximate the incomplete factors for an existing nonzero pattern with a strategy that dynamically adapts the nonzero pattern to the problem characteristics. This requires the efficient selection of thresholds that separate the values to be dropped from the incomplete factors, and we design a novel selection algorithm tailored towards GPUs. All components of the ParILUT-GPU algorithm make heavy use of the features available in the latest NVIDIA GPU generations, and outperform existing multithreaded CPU implementations.},
	doi = {https://doi.org/10.1109/IPDPS.2019.00033},
	author = {Hartwig Anzt and Tobias Ribizel and Goran Flegar and Edmond Chow and Jack Dongarra}
}
@article {1317,
	title = {Toward a Modular Precision Ecosystem for High-Performance Computing},
	journal = {The International Journal of High Performance Computing Applications},
	volume = {33},
	year = {2019},
	month = {2019-11},
	pages = {1069-1078},
	abstract = {With the memory bandwidth of current computer architectures being significantly slower than the (floating point) arithmetic performance, many scientific computations only leverage a fraction of the computational power in today{\textquoteright}s high-performance architectures. At the same time, memory operations are the primary energy consumer of modern architectures, heavily impacting the resource cost of large-scale applications and the battery life of mobile devices. This article tackles this mismatch between floating point arithmetic throughput and memory bandwidth by advocating a disruptive paradigm change with respect to how data are stored and processed in scientific applications. Concretely, the goal is to radically decouple the data storage format from the processing format and, ultimately, design a {\textquotedblleft}modular precision ecosystem{\textquotedblright} that allows for more flexibility in terms of customized data access. For memory-bounded scientific applications, dynamically adapting the memory precision to the numerical requirements allows for attractive resource savings. In this article, we demonstrate the potential of employing a modular precision ecosystem for the block-Jacobi preconditioner and the PageRank algorithm{\textemdash}two applications that are popular in the communities and at the same characteristic representatives for the field of numerical linear algebra and data analytics, respectively.},
	keywords = {conjugate gradient, GPUs, Jacobi method, Modular precision, multicore processors, PageRank, parallel numerical linear algebra},
	issn = {1094-3420},
	doi = {https://doi.org/10.1177/1094342019846547},
	author = {Hartwig Anzt and Goran Flegar and Thomas Gruetzmacher and Enrique S. Quintana-Orti}
}
@conference {1318,
	title = {Towards Continuous Benchmarking},
	booktitle = {Platform for Advanced Scientific Computing Conference (PASC 2019)},
	year = {2019},
	month = {2019-06},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Zurich, Switzerland},
	abstract = {We present an automated performance evaluation framework that enables an automated workflow for testing and performance evaluation of software libraries. Integrating this component into an ecosystem enables sustainable software development, as a community effort, via a web application for interactively evaluating the performance of individual software components. The performance evaluation tool is based exclusively on web technologies, which removes the burden of downloading performance data or installing additional software. We employ this framework for the Ginkgo software ecosystem, but the framework can be used with essentially any software project, including the comparison between different software libraries. The Continuous Integration (CI) framework of Ginkgo is also extended to automatically run a benchmark suite on predetermined HPC systems, store the state of the machine and the environment along with the compiled binaries, and collect results in a publicly accessible performance data repository based on Git. The Ginkgo performance explorer (GPE) can be used to retrieve the performance data from the repository, and visualizes it in a web browser. GPE also implements an interface that allows users to write scripts, archived in a Git repository, to extract particular data, compute particular metrics, and visualize them in many different formats (as specified by the script). The combination of these approaches creates a workflow which enables performance reproducibility and software sustainability of scientific software. In this paper, we present example scripts that extract and visualize performance data for Ginkgo{\textquoteright}s SpMV kernels that allow users to identify the optimal kernel for specific problem characteristics.},
	isbn = {9781450367707},
	doi = {https://doi.org/10.1145/3324989.3325719},
	author = {Hartwig Anzt and Yen Chen Chen and Terry Cojean and Jack Dongarra and Goran Flegar and Pratik Nayak and Enrique S. Quintana-Orti and Yuhsiang M. Tsai and Weichung Wang}
}
@article {1220,
	title = {Variable-Size Batched Gauss-Jordan Elimination for Block-Jacobi Preconditioning on Graphics Processors},
	journal = {Parallel Computing},
	volume = {81},
	year = {2019},
	month = {2019-01},
	pages = {131-146},
	abstract = {In this work, we address the efficient realization of block-Jacobi preconditioning on graphics processing units (GPUs). This task requires the solution of a collection of small and independent linear systems. To fully realize this implementation, we develop a variable-size batched matrix inversion kernel that uses Gauss-Jordan elimination (GJE) along with a variable-size batched matrix{\textendash}vector multiplication kernel that transforms the linear systems{\textquoteright} right-hand sides into the solution vectors. Our kernels make heavy use of the increased register count and the warp-local communication associated with newer GPU architectures. Moreover, in the matrix inversion, we employ an implicit pivoting strategy that migrates the workload (i.e., operations) to the place where the data resides instead of moving the data to the executing cores. We complement the matrix inversion with extraction and insertion strategies that allow the block-Jacobi preconditioner to be set up rapidly. The experiments on NVIDIA{\textquoteright}s K40 and P100 architectures reveal that our variable-size batched matrix inversion routine outperforms the CUDA basic linear algebra subroutine (cuBLAS) library functions that provide the same (or even less) functionality. We also show that the preconditioner setup and preconditioner application cost can be somewhat offset by the faster convergence of the iterative solver.},
	keywords = {Batched algorithms, Block-Jacobi, Gauss{\textendash}Jordan elimination, Graphics processor, matrix inversion, sparse linear systems},
	doi = {https://doi.org/10.1016/j.parco.2017.12.006},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@conference {1234,
	title = {Variable-Size Batched Condition Number Calculation on GPUs},
	booktitle = {SBAC-PAD},
	year = {2018},
	month = {2018-09},
	address = {Lyon, France},
	url = {https://ieeexplore.ieee.org/document/8645907},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Thomas Gruetzmacher}
}
@inproceedings {998,
	title = {Batched Gauss-Jordan Elimination for Block-Jacobi Preconditioner Generation on GPUs},
	journal = {Proceedings of the 8th International Workshop on Programming Models and Applications for Multicores and Manycores},
	year = {2017},
	month = {2017-02},
	pages = {1{\textendash}10},
	publisher = {ACM},
	address = {New York, NY, USA},
	abstract = {In this paper, we design and evaluate a routine for the efficient generation of block-Jacobi preconditioners on graphics processing units (GPUs). Concretely, to exploit the architecture of the graphics accelerator, we develop a batched Gauss-Jordan elimination CUDA kernel for matrix inversion that embeds an implicit pivoting technique and handles the entire inversion process in the GPU registers. In addition, we integrate extraction and insertion CUDA kernels to rapidly set up the block-Jacobi preconditioner.

Our experiments compare the performance of our implementation against a sequence of batched routines from the MAGMA library realizing the inversion via the LU factorization with partial pivoting. Furthermore, we evaluate the costs of different strategies for the block-Jacobi extraction and insertion steps, using a variety of sparse matrices from the SuiteSparse matrix collection. Finally, we assess the efficiency of the complete block-Jacobi preconditioner generation in the context of an iterative solver applied to a set of computational science problems, and quantify its benefits over a scalar Jacobi preconditioner.},
	keywords = {block-Jacobi preconditioner, Gauss-Jordan elimination, graphics processing units (GPUs), iterative methods, matrix inversion, sparse linear systems},
	isbn = {978-1-4503-4883-6},
	doi = {10.1145/3026937.3026940},
	url = {http://doi.acm.org/10.1145/3026937.3026940},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@article {1163,
	title = {Flexible Batched Sparse Matrix Vector Product on GPUs},
	year = {2017},
	month = {2017-11},
	publisher = {ScalA{\textquoteright}17: 8th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems},
	address = {Denver, Colorado},
	author = {Hartwig Anzt and Collins, Gary and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@conference {,
	title = {Flexible Batched Sparse Matrix-Vector Product on GPUs},
	booktitle = {8th Workshop on Latest Advances in Scalable Algorithms for Large-Scale Systems (ScalA {\textquoteright}17)},
	year = {2017},
	month = {2017-11},
	publisher = {ACM Press},
	organization = {ACM Press},
	address = {Denver, CO},
	abstract = {	
We propose a variety of batched routines for concurrently processing a large collection of small-size, independent sparse matrix-vector products (SpMV) on graphics processing units (GPUs). These batched SpMV kernels are designed to be flexible in order to handle a batch of matrices which differ in size, nonzero count, and nonzero distribution. Furthermore, they support three most commonly used sparse storage formats: CSR, COO and ELL. Our experimental results on a state-of-the-art GPU reveal performance improvements of up to 25X compared to non-batched SpMV routines.},
	doi = {http://dx.doi.org/10.1145/3148226.3148230},
	author = {Hartwig Anzt and Gary Collins and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
@techreport {1130,
	title = {MAGMA-sparse Interface Design Whitepaper},
	journal = {Innovative Computing Laboratory Technical Report},
	number = {ICL-UT-17-05},
	year = {2017},
	month = {2017-09},
	type = {Technical Report},
	abstract = {In this report we describe the logic and interface we develop for the MAGMA-sparse library
to allow for easy integration as third-party library into a top-level software ecosystem. The
design choices are based on extensive consultation with other software library developers, in
particular the Trilinos software development team. The interface documentation is at this point
not exhaustive, but a first proposal for setting a standard. Although the interface description
targets the MAGMA-sparse software module, we hope that the design choices carry beyond this
specific library, and are attractive for adoption in other packages.
This report is not intended as static document, but will be updated over time to reflect the agile
software development in the ECP 1.3.3.11 STMS11-PEEKS project.},
	author = {Hartwig Anzt and Erik Boman and Jack Dongarra and Goran Flegar and Mark Gates and Mike Heroux and Mark Hoemmen and Jakub Kurzak and Piotr Luszczek and Sivasankaran Rajamanickam and Stanimire Tomov and Stephen Wood and Ichitaro Yamazaki}
}
@inproceedings {1088,
	title = {Variable-Size Batched Gauss-Huard for Block-Jacobi Preconditioning},
	journal = {International Conference on Computational Science (ICCS 2017)},
	volume = {108},
	year = {2017},
	month = {2017-06},
	pages = {1783-1792},
	publisher = {Procedia Computer Science},
	address = {Zurich, Switzerland},
	abstract = {In this work we present new kernels for the generation and application of block-Jacobi precon-ditioners that accelerate the iterative solution of sparse linear systems on graphics processing units (GPUs). Our approach departs from the conventional LU factorization and decomposes the diagonal blocks of the matrix using the Gauss-Huard method. When enhanced with column pivoting, this method is as stable as LU with partial/row pivoting. Due to extensive use of GPU registers and integration of implicit pivoting, our variable size batched Gauss-Huard implementation outperforms the batched version of LU factorization. In addition, the application kernel combines the conventional two-stage triangular solve procedure, consisting of a backward solve followed by a forward solve, into a single stage that performs both operations simultaneously.},
	doi = {https://doi.org/10.1016/j.procs.2017.05.186},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti and Andres E. Thomas}
}
@conference {1160,
	title = {Variable-Size Batched LU for Small Matrices and Its Integration into Block-Jacobi Preconditioning},
	booktitle = {46th International Conference on Parallel Processing (ICPP)},
	year = {2017},
	month = {2017-08},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Bristol, United Kingdom},
	abstract = {We present a set of new batched CUDA kernels for the LU factorization of a large collection of independent problems of different size, and the subsequent triangular solves. All kernels heavily exploit the registers of the graphics processing unit (GPU) in order to deliver high performance for small problems. The development of these kernels is motivated by the need for tackling this embarrassingly parallel scenario in the context of block-Jacobi preconditioning that is relevant for the iterative solution of sparse linear systems.},
	keywords = {graphics processing units, Jacobian matrices, Kernel, linear systems, Parallel processing, Sparse matrices},
	doi = {10.1109/ICPP.2017.18},
	url = {http://ieeexplore.ieee.org/abstract/document/8025283/?reload=true},
	author = {Hartwig Anzt and Jack Dongarra and Goran Flegar and Enrique S. Quintana-Orti}
}
