DiagonalPreconditioner wrapper on Eigen-3.3.5 for Matrix-Free sparse solver CG

Question

[Question]: Could some help me with a DiagonalPreconditioner wrapper for matrix-free solver on Eigen-3.3.5?

I am trying to modify the example on the link bellow, so I can use the DiagonalPreconditioner with CG sparse interactive solver.
[Eigen-3.3.5] https://eigen.tuxfamily.org/dox/group__MatrixfreeSolverExample.html

On Eigen-3.2.10 they provide wrapper for the preconditioner, but I got stuck on modifying the solver part of this example and gave up on using it due significant changes between the two versions. [Eigen-3.2.10] http://eigen.tuxfamily.org/dox-3.2/group__MatrixfreeSolverExample.html

[Background]: I am compiling cg solver on Mex so I can use it on Matlab. Reason, Eigen solver is more efficient than the pcg solver that Matlab has for the size of linear system(LS) I am working right now, 250k x 250k (it could be as big as 1e6 x 1e6, but luckily it is extremely sparse). Moreover, the values of right hand side(RHS) and left hand side (LHS) of the LS changes every iteration, but the sparsity pattern is fixed. For this reason I thought Eigen would be a good choice, since one could decompose the "compute" steps, as explained here: https://eigen.tuxfamily.org/dox/group__TopicSparseSystems.html

[OS]: CentOS 7

[code]: The code bellow works if one comment the parts related with the matrix-free solver and uncomment the the parts indicated (I believe it is well indicated).

#include "mex.h"
#include "math.h"
#include "matrix.h"
#include <Eigen/Sparse>
#include <Eigen/Dense>
#include <Eigen/Core>
#include <vector>
#include <algorithm>
#include <iostream>
#include <omp.h>
#include <fstream>

//using namespace Eigen;
using namespace std;
typedef Eigen::Triplet<double> T;

class MatrixReplacement;
using Eigen::SparseMatrix;

namespace Eigen {
namespace internal {
  // MatrixReplacement looks-like a SparseMatrix, so let's inherits its traits:
  template<>
  struct traits<MatrixReplacement> :  public Eigen::internal::traits<Eigen::SparseMatrix<double> >
  {};
}
}
// Example of a matrix-free wrapper from a user type to Eigen's compatible type
// For the sake of simplicity, this example simply wrap a Eigen::SparseMatrix.
class MatrixReplacement : public Eigen::EigenBase<MatrixReplacement> {
public:
  // Required typedefs, constants, and method:
  typedef double Scalar;
  typedef double RealScalar;
  typedef int StorageIndex;
  enum {
    ColsAtCompileTime = Eigen::Dynamic,
    MaxColsAtCompileTime = Eigen::Dynamic,
    IsRowMajor = false
  };
  Index rows() const { return mp_mat->rows(); }
  Index cols() const { return mp_mat->cols(); }
  //Index outerSize() const { return mp_mat->outerSize(); }
  template<typename Rhs>
  Eigen::Product<MatrixReplacement,Rhs,Eigen::AliasFreeProduct> operator*(const Eigen::MatrixBase<Rhs>& x) const {
    return Eigen::Product<MatrixReplacement,Rhs,Eigen::AliasFreeProduct>(*this, x.derived());
  }
  // Custom API:
  MatrixReplacement() : mp_mat(0) {}
  void attachMyMatrix(const SparseMatrix<double> &mat) {
    mp_mat = &mat;
  }
  const SparseMatrix<double> my_matrix() const { return *mp_mat; }
private:
  const SparseMatrix<double> *mp_mat;
};
// Implementation of MatrixReplacement * Eigen::DenseVector though a specialization of internal::generic_product_impl:
namespace Eigen {
namespace internal {
  template<typename Rhs>
  struct generic_product_impl<MatrixReplacement, Rhs, SparseShape, DenseShape, GemvProduct> // GEMV stands for matrix-vector
  : generic_product_impl_base<MatrixReplacement,Rhs,generic_product_impl<MatrixReplacement,Rhs> >
  {
    typedef typename Product<MatrixReplacement,Rhs>::Scalar Scalar;
    template<typename Dest>
    static void scaleAndAddTo(Dest& dst, const MatrixReplacement& lhs, const Rhs& rhs, const Scalar& alpha)
    {
      // This method should implement "dst += alpha * lhs * rhs" inplace,
      // however, for iterative solvers, alpha is always equal to 1, so let's not bother about it.
      assert(alpha==Scalar(1) && "scaling is not implemented");
      EIGEN_ONLY_USED_FOR_DEBUG(alpha);
      // Here we could simply call dst.noalias() += lhs.my_matrix() * rhs,
      // but let's do something fancier (and less efficient):
      for(Index i=0; i<lhs.cols(); ++i)
        dst += rhs(i) * lhs.my_matrix().col(i);
    }
  };
}
}

// the gateway function
void mexFunction( int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
    //
    Eigen::initParallel();
    //input vars
    double *A_ir;
    double *A_jc;
    double *A_val;
    double *b;
    double *x0;
    int maxiter;
    double tol;
    double icholShift;
    int nWorkers;
    //output vars
    double *x;
    // long int numite;
    // double resrel;
    //temp vars
    long int nrows;
    long int nnz;
    std::vector<T> tripletList;
    //-----------------
    // GetData
    //-----------------
    if(nrhs != 9){
        mexErrMsgIdAndTxt("MEX:c_sol_pcg_eigen:rhs",
                "This function takes too much input arguments.");
    }
    A_ir = mxGetPr(prhs[0]);
    A_jc = mxGetPr(prhs[1]);
    A_val = mxGetPr(prhs[2]);
    b = mxGetPr(prhs[3]);
    x0 = mxGetPr(prhs[4]);
    maxiter = mxGetScalar(prhs[5]);
    tol = mxGetScalar(prhs[6]);
    icholShift = mxGetScalar(prhs[7]);
    nrows = mxGetM(prhs[3]);
    nnz = mxGetM(prhs[0]);
    nWorkers = mxGetScalar(prhs[8]);
    plhs[0] = mxCreateDoubleMatrix(nrows,1,mxREAL);
    x = mxGetPr(plhs[0]);
    Eigen::setNbThreads(nWorkers);

    //-----------------
    //calculations
    //-----------------
    //covert A_ir, A_jc, A_val to Eigen Sparse matrix
    tripletList.reserve(nnz);
    for(long int i=0; i<nnz; i++){
        tripletList.push_back(T(A_ir[i]-1, A_jc[i]-1, A_val[i]));
    }
    Eigen::SparseMatrix<double> A_eigen(nrows, nrows);
    A_eigen.setFromTriplets(tripletList.begin(), tripletList.end());


    // call matrix-free API
    MatrixReplacement AF_eigen; // <-comment here for matrix context
    AF_eigen.attachMyMatrix(A_eigen); // <-comment here for matrix context


    Eigen::Map<Eigen::VectorXd> b_eigen(b, nrows);
    //CG solver of Eigen
    Eigen::VectorXd x_eigen;
    Eigen::Map<Eigen::VectorXd> x0_eigen(x0, nrows);
    // solve problem
    //Eigen::ConjugateGradient<SparseMatrix<double>, Eigen::Lower|Eigen::Upper,Eigen::DiagonalPreconditioner<double>> cg; // <-uncomment here for matrix context

    // matrix free solver
    Eigen::ConjugateGradient<MatrixReplacement, Eigen::Lower|Eigen::Upper,Eigen::DiagonalPreconditioner<double>>  cg; // <-comment here for matrix context

    cg.setTolerance(tol);
    cg.setMaxIterations(maxiter);
    //cg.compute(A_eigen);  // <-uncomment here for matrix context

    cg.compute(AF_eigen); //  <-comment here for matrix context

    x_eigen = cg.solveWithGuess(b_eigen, x0_eigen);
    for(long int i=0; i<nrows; i++){
        x[i] = x_eigen(i);
    }
}

If one compile as it is, the following error is returned (I have removed my local machine paths and substituted by path_to_eigen, or path_to_mex_file in the error log):

    Error using mex
    In file included from
    /path_to_eigen/include/eigen3/Eigen/IterativeLinearSolvers:39:0,
                     from
    /path_to_eigen/include/eigen3/Eigen/Sparse:33,
                     from
    /path_to_mex_file/mex/EigenPCGIcholOMPFREE.cpp:23:
    /path_to_eigen/include/eigen3/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h:
    In instantiation of ‘Eigen::DiagonalPreconditioner<_Scalar>&
    Eigen::DiagonalPreconditioner<_Scalar>::factorize(const MatType&) [with MatType
    = MatrixReplacement; _Scalar = double]’:
    /path_to_eigen/include/eigen3/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h:84:27:
    required from ‘Eigen::DiagonalPreconditioner<_Scalar>&
    Eigen::DiagonalPreconditioner<_Scalar>::compute(const MatType&) [with MatType =
    MatrixReplacement; _Scalar = double]’
 /path_to_eigen/include/eigen3/Eigen/src/IterativeLinearSolvers/IterativeSolverBase.h:241:5:
    required from ‘Derived& Eigen::IterativeSolverBase<Derived>::compute(const
    Eigen::EigenBase<OtherDerived>&) [with MatrixDerived = MatrixReplacement;
    Derived = Eigen::ConjugateGradient<MatrixReplacement, 3,
    Eigen::DiagonalPreconditioner<double> >]’
    /path_to_mex_file/EigenPCGIcholOMPFREE.cpp:175:24:
    required from here
    /path_to_eigen/include/eigen3/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h:68:26:
    error: ‘const class MatrixReplacement’ has no member named ‘outerSize’
           for(int j=0; j<mat.outerSize(); ++j)
                          ~~~~^~~~~~~~~
    /path_to_eigen/include/eigen3/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h:70:41:
    error: no type named ‘InnerIterator’ in ‘class MatrixReplacement’
             typename MatType::InnerIterator it(mat,j);
                                             ^~
    /path_to_eigen/include/eigen3/Eigen/src/IterativeLinearSolvers/BasicPreconditioners.h:70:41:
    error: no type named ‘InnerIterator’ in ‘class MatrixReplacement’

I was able to fix 'outerSize' by adding Index outerSize() const { return mp_mat->outerSize(); } Index outerSize() const { return mp_mat->outerSize(); } below Index cols() const { return mp_mat->cols(); } Index cols() const { return mp_mat->cols(); } . However, I could not figure how to work around the InnerIterator.

I believe Mex is not the issue, but here is the Matlab file for compiling on mex:

MEXOPTS={'-v','-largeArrayDims','-DMEX','-DNDEBUG'};
MSSE42='CXXFLAGS=$CXXFLAGS -msse4.2';
STDCPP11='CXXFLAGS=$CXXFLAGS -fopenmp';
LDFLAGS = 'LDFLAGS=$LDFLAGS -fopenmp';

EIGEN_INC{1}='-I/path_to_eigen/include/eigen3'; // version 3.3.5

%% not sure if boost is need, I have being carrying this from other functions
BOOST_INC='-I/path_to_boost_include/include';
BOOST_LIB{1}='-L/path_to_local_boost_lib/boost/lib'; // version 1.67
BOOST_LIB{2}='-lboost_thread';
BOOST_LIB{3}='-lboost_system';

GMP_INC='-I/apps/cent7/gcc/6.3.0/gmp-6.1.0/include';
GMP_LIB{1}='-L/apps/cent7/gcc/6.3.0/gmp-6.1.0/lib';
GMP_LIB{2}='-lgmpxx';
GMP_LIB{3}='-lgmp';

MPFR_INC='-I/apps/cent7/gcc/6.3.0/mpfr-3.1.5/include';
MPFR_LIB{1}='-L/apps/cent7/gcc/6.3.0/mpfr-3.1.5/lib';
MPFR_LIB{2}='-lmpfr';

MPC_INC='-I/apps/cent7/gcc/6.3.0/mpc-1.0.3/include';
MPC_LIB{1}='-L/apps/cent7/gcc/6.3.0/mpc-1.0.3/lib';
MPC_LIB{2}='-lmpc';

mex( MEXOPTS{:}, MSSE42,STDCPP11,LDFLAGS,...
    BOOST_INC, BOOST_LIB{:}, EIGEN_INC{:}, GMP_INC, GMP_LIB{:}, ...
    MPFR_INC,MPFR_LIB{:}, MPC_INC, MPC_LIB{:}, ...
    'EigenPCGIcholOMPFREE.cpp');

Modifying this wrapper has being to much for my "procedural mindset". I hope you guys can help me.

Thank you for you help in advance!

Answer 1

I know this post is old, and you probably don't need the answer anymore, but I want to leave an answer here for people who found this post through Google like myself. I am quite a novice to the Eigen library myself, but hopefully the following solution is better than not having a solution.

To implement a diagonal preconditioner, you'd obviously need some way to extract the diagonal coefficients from your linear operator. How this is done would depend on the implementation detail of your matrix-free operator class, but at worse you can pass in a basis vector to your operator (all zeros except a one at the column of interest). This would return a column vector of the matrix representation of your linear operator, which you can extract the diagonal element from. This is basically equivalent to computing a dense matrix representation of your operator and just not storing most of the values, so you really don't want to do that unless there's no other options. Depending on the efficiency of this approach, it might be better to just use the identity preconditioner instead.

Looking in the source code of Eigen::DiagonalPreconditioner , you will see it uses the MatrixReplacement::InnerIterator to extract the diagonal coefficients like the following:

       for(int j=0; j<mat.outerSize(); ++j)
       {
         typename MatType::InnerIterator it(mat,j);
         while(it && it.index()!=j) ++it;
         if(it && it.index()==j && it.value()!=Scalar(0))
           m_invdiag(j) = Scalar(1)/it.value();
         else
           m_invdiag(j) = Scalar(1);
       }

You need to create a iterator class inside your MatrixReplacement that will return the diagonal coefficient for a specified row/column of the matrix representation of your operator. My implementation follows:

    class InnerIterator {
    public:
        InnerIterator(const MatrixReplacement& mat, Index row)
            : mat(mat), has_val(false), row(row), col(row){ }

        operator bool() { return row==col; }

        InnerIterator& operator++(){
            col++;
            has_val = false;
            return *this;
        }

        Index index(){ return col; }

        Scalar value(){
            if(!has_val){ // cache value since this function is called twice
                stored_val = mat.diagonalCoefficient(row); // your implementation here
                has_val = true;
            }
            return stored_val;
        }

    private:
        const MatrixReplacement& mat;
        bool has_val;
        Index row, col;
        Scalar stored_val;
    };