IPCC/_k_n_m_p_i_manager_8cpp_source.html

 #include "stdafx.h"

 #include "CKNGlobal.h"

 #include "KNMPIManager.h"

 #include "KNIPCCUtility.h"

 #include "KNTimeMeasurement.h"

 #include <math.h>

 #include <string.h>

 #include <algorithm>


 int CKNMPIManager::m_nCurrentRank = 0;

 int CKNMPIManager::m_nCommWorldRank = 0;

 int CKNMPIManager::m_nTotalNode = 1;

 bool CKNMPIManager::m_bStartMPI = false;

 int *CKNMPIManager::m_pLoadBalance = NULL;;

 CKNMPIManager::LPCOMPLEX_NUMBER CKNMPIManager::m_pCommBuffer = NULL;

 CKNMPIManager::LPCOMPLEX_NUMBER CKNMPIManager::m_pConvertingBuffer = NULL;

 int* CKNMPIManager::m_pRecvCount = NULL;

 int* CKNMPIManager::m_pSendCount = NULL;

 int*  CKNMPIManager::m_pBankInfo = NULL;

 int* CKNMPIManager::m_pDispls = NULL;

 MPI_Request CKNMPIManager::m_SendDoubleAsyncRequest = MPI_REQUEST_NULL;

 MPI_Request CKNMPIManager::m_ReceiveDoubleAsyncRequest = MPI_REQUEST_NULL;

 unsigned int CKNMPIManager::m_nMPILevel = 1;

 bool CKNMPIManager::m_bNeedPostOperation[10] = { false, false, false, false, false, false, false, false, false, false };

 MPI_Comm CKNMPIManager::m_mpiCommIndex = MPI_COMM_WORLD;

 MPI_Comm CKNMPIManager::m_deflationComm = MPI_COMM_NULL;

 MPI_Group CKNMPIManager::m_lanczosGroup = MPI_GROUP_EMPTY;

 MPI_Group CKNMPIManager::m_deflationGroup = MPI_GROUP_EMPTY;

 bool CKNMPIManager::m_bMultiLevel = false;

 unsigned int CKNMPIManager::m_nLanczosGroupIndex = 0;


 CKNMPIManager::CKNMPIManager()

 {

 }


 CKNMPIManager::~CKNMPIManager()

 {

 }


 bool CKNMPIManager::InitLevel(int nMPILevel, int nFindingDegeneratedEVCount)

 {

     bool                bRtn = true;

     int                 nNeedNodeCount = 1;

     int                 world_size, rank;

     int                 nPerGroupNode;

     int                 nLanczosGroupIndex;

     int                 *pNewGroupRank = NULL;

     unsigned int        i;

     MPI_Group           commWorldGroup;


     if( 1 == nMPILevel )

     {

         m_mpiCommIndex = MPI_COMM_WORLD;

         return bRtn;

     }


     if( nFindingDegeneratedEVCount > 1 )

     {

         m_bNeedPostOperation[1] = true;

         nNeedNodeCount *= nFindingDegeneratedEVCount;

         nMPILevel--;

     }


     if( nMPILevel == 1 )

     {

         bRtn = CheckDeflationNodeCount(nNeedNodeCount);

         if( !bRtn )

             return bRtn;

     }


     m_nCommWorldRank = GetCurrentRank();

     nPerGroupNode = GetTotalNodeCount() / nNeedNodeCount;

     nLanczosGroupIndex = GetCurrentRank() / nPerGroupNode;

     pNewGroupRank  = (int*)malloc(sizeof(int)*nPerGroupNode);


     for( i = 0; i < nPerGroupNode ; ++i)

         pNewGroupRank [i] = nLanczosGroupIndex * nPerGroupNode + i;


     MPI_Comm_group(MPI_COMM_WORLD,&commWorldGroup);

     MPI_Group_incl(commWorldGroup,nPerGroupNode,pNewGroupRank ,&m_lanczosGroup);

     MPI_Comm_create(MPI_COMM_WORLD,m_lanczosGroup,&m_mpiCommIndex);

     MPI_Comm_size(m_mpiCommIndex, &world_size);

     MPI_Comm_rank(m_mpiCommIndex, &rank);

     SetMPIEnviroment(rank, world_size);

     m_nLanczosGroupIndex = nLanczosGroupIndex;


     pNewGroupRank  = (int*)realloc(pNewGroupRank , sizeof(int)*nFindingDegeneratedEVCount);

     for( i = 0; i < nFindingDegeneratedEVCount ; ++i)

         pNewGroupRank [i] = i * nPerGroupNode + GetCurrentRank();


     MPI_Comm_group(MPI_COMM_WORLD,&commWorldGroup);

     MPI_Group_incl(commWorldGroup,nFindingDegeneratedEVCount,pNewGroupRank ,&m_deflationGroup);

     MPI_Comm_create(MPI_COMM_WORLD,m_deflationGroup,&m_deflationComm);

     MPI_Comm_rank(m_deflationComm, &rank);

     MPI_Comm_size(m_mpiCommIndex, &world_size);


     m_bMultiLevel = true;


     FREE_MEM(pNewGroupRank );


     if( false == CKNMPIManager::IsDeflationRoot() || false == CKNMPIManager::IsLanczosComputeRoot() )

         CKNIPCCUtility::SetShow(false);


     bRtn = true;

     return bRtn;

 }


 bool CKNMPIManager::CheckDeflationNodeCount(int nNeedNodeCount)

 {

     if( 0 == m_nTotalNode % nNeedNodeCount )

         return true;

     else

         return false;

 }


 void CKNMPIManager::SetMPIEnviroment(int nRank, int nTotalNode)

 {

     m_nCurrentRank = nRank;

     m_nTotalNode = nTotalNode;

     m_bStartMPI = true;

 }


 void CKNMPIManager::LoadBlancingForLanczos(int nRowCount)

 {

     int                 i, j, temp;

     m_pLoadBalance = (int *)malloc(sizeof(int)*(m_nTotalNode)); // For communication size

     m_pLoadBalance[0] = nRowCount;


 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     if (nRowCount < 10 * m_nTotalNode){

         CKNIPCCUtility::ShowMsg("ERROR :: matrix dimension < 10 * number of CPU for Lanczos "); // Error out

     }


     if (0 == (nRowCount / 10) % m_nTotalNode){

         for (i = 0; i<m_nTotalNode; i++){

             m_pLoadBalance[i] = nRowCount / m_nTotalNode;

         }

     }

     else{

         temp = nRowCount;

         for (i = m_nTotalNode - 1; i>-1; i--){

             m_pLoadBalance[i] = ((int)(nRowCount / m_nTotalNode / 10 + 1)) * 10;

             temp -= m_pLoadBalance[i];

             if ((temp / 10) % i == 0){

                 for (j = 0; j < i; j++){

                     m_pLoadBalance[j] = temp / i;

                 }

                 temp = 0;

                 break;

             }

         }

     }

 }


 void CKNMPIManager::LoadBlancing(int nElementCount)

 {

     m_pLoadBalance = (int *)malloc(sizeof(int)*(m_nTotalNode)); // For communication size


 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     MPI_Allgather(&nElementCount, 1, MPI_INTEGER, m_pLoadBalance, 1, MPI_INTEGER, m_mpiCommIndex);

 }


 int CKNMPIManager::GetLoadBalanceCount(int nRank)

 {

 #ifdef DISABLE_MPI_ROUTINE

     return 0;

 #endif

     if (nRank > m_nTotalNode)

         return 0;


     return m_pLoadBalance[nRank];

 }


 bool CKNMPIManager::IsRootRank()

 {

     bool            bRtn = true;


 #ifdef DISABLE_MPI_ROUTINE

     return bRtn;

 #endif


     if (0 == GetCurrentRank())

         return bRtn;


     bRtn = false;

     return bRtn;

 }


 bool CKNMPIManager::IsRootRank(MPI_Comm comm)

 {

     if (MPI_COMM_NULL == comm)

         return true;


     if( 0 == GetCurrentRank(comm) )

         return true;

     else

         return false;

 }


 int CKNMPIManager::GetCurrentRank(MPI_Comm comm)

 {

     int             rank;


     MPI_Comm_rank(comm, &rank);

     return rank;

 }


 void CKNMPIManager::BroadcastVector(CKNMatrixOperation::CKNVector *pVector)

 {

 #ifdef DISABLE_MPI_ROUTINE

     if (!IsInMPIRoutine())

         return;

 #endif

 }


 void CKNMPIManager::MergeVector(CKNMatrixOperation::CKNVector *pVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize)

 {

     LPCOMPLEX_NUMBER    lpSendBuffer = NULL;

     unsigned int        i;


 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     double                  *pBuffer = (double*)malloc(sizeof(double)*nMergeSize);

     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     MPI_Allgatherv(pVector->m_vectValueRealBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,

         pBuffer, m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);


     for (i = 0; i < nMergeSize; i++)

         pResultVector->m_vectValueRealBuffer[i] = pBuffer[i];


     MPI_Allgatherv(pVector->m_vectValueImaginaryBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,

         pBuffer, m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);


     for (i = 0; i < nMergeSize; i++)

         pResultVector->m_vectValueImaginaryBuffer[i] = pBuffer[i];


     FREE_MEM(pBuffer);


     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);


 }


 void CKNMPIManager::MergeVectorEx_Optimal(CKNMatrixOperation::CKNVector *pSrcVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize, double fFirstIndex, unsigned int nSizeFromPrevRank, unsigned int nSizeFromNextRank, unsigned int nSizetoPrevRank, unsigned int nSizetoNextRank, unsigned int *mPos)

 {

     if (m_nTotalNode <= 3)

     {

         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);

         MPI_Allgatherv(pSrcVector->m_vectValueRealBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE, pResultVector->m_vectValueRealBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);

         MPI_Allgatherv(pSrcVector->m_vectValueImaginaryBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE, pResultVector->m_vectValueImaginaryBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);

         mPos[0] = -1; mPos[1] = -1; mPos[2] = -1;

     }

     else

     {

         long long       fCurrentRankPos = fFirstIndex;

         int         nPrevRank = (m_nCurrentRank - 1 + m_nTotalNode) % m_nTotalNode;

         int         nNextRank = (m_nCurrentRank + 1) % m_nTotalNode;

         long long       fPrevRankPos = -1, fNextRankPos = -1;

         double          *pSendBuffer = NULL, *pRecvBuffer = NULL;

         MPI_Request     req[2];

         MPI_Status      status[2];


         if (0 == m_nCurrentRank)

             fPrevRankPos = nMergeSize - nSizeFromPrevRank;

         else

             fPrevRankPos = fFirstIndex - nSizeFromPrevRank;


         if (m_nCurrentRank == m_nTotalNode - 1)

             fNextRankPos = 0;

         else

             fNextRankPos = fFirstIndex + GetLoadBalanceCount(m_nCurrentRank);


         mPos[0] = (unsigned int)fPrevRankPos; mPos[1] = (unsigned int)fCurrentRankPos; mPos[2] = (unsigned int)fNextRankPos;


         //printf("Rank %d: myload=%d, nSizeFromPrevRank=%d, nSizeFromNextRank=%d, nSizetoPrevRank=%d, nSizetoNextRank=%d, fPrevRankPos=%d, fNextRankPos=%d\n", m_nCurrentRank, GetLoadBalanceCount(m_nCurrentRank), nSizeFromPrevRank, nSizeFromNextRank, nSizetoPrevRank, nSizetoNextRank, fPrevRankPos, fNextRankPos);


         pSendBuffer = pSrcVector->m_vectValueRealBuffer.data();

         pRecvBuffer = pResultVector->m_vectValueRealBuffer.data();


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);

         MPI_Irecv(pRecvBuffer + fPrevRankPos, nSizeFromPrevRank, MPI_DOUBLE, nPrevRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);

         MPI_Isend(pSendBuffer + GetLoadBalanceCount(m_nCurrentRank) - nSizetoNextRank, nSizetoNextRank, MPI_DOUBLE, nNextRank, nNextRank, m_mpiCommIndex, &req[1]);

         MPI_Waitall(2, req, status);


         MPI_Irecv(pRecvBuffer + fNextRankPos, nSizeFromNextRank, MPI_DOUBLE, nNextRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);

         MPI_Isend(pSendBuffer, nSizetoPrevRank, MPI_DOUBLE, nPrevRank, nPrevRank, m_mpiCommIndex, &req[1]);

         MPI_Waitall(2, req, status);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);


         pSendBuffer = pSrcVector->m_vectValueImaginaryBuffer.data();

         pRecvBuffer = pResultVector->m_vectValueImaginaryBuffer.data();


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);

         MPI_Irecv(pRecvBuffer + fPrevRankPos, nSizeFromPrevRank, MPI_DOUBLE, nPrevRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);

         MPI_Isend(pSendBuffer + GetLoadBalanceCount(m_nCurrentRank) - nSizetoNextRank, nSizetoNextRank, MPI_DOUBLE, nNextRank, nNextRank, m_mpiCommIndex, &req[1]);

         MPI_Waitall(2, req, status);


         MPI_Irecv(pRecvBuffer + fNextRankPos, nSizeFromNextRank, MPI_DOUBLE, nNextRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);

         MPI_Isend(pSendBuffer, nSizetoPrevRank, MPI_DOUBLE, nPrevRank, nPrevRank, m_mpiCommIndex, &req[1]);

         MPI_Waitall(2, req, status);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);


         memcpy(pResultVector->m_vectValueRealBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueRealBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));

         memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueImaginaryBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));


     }

 }


 void CKNMPIManager::MergeVectorOptimal(CKNMatrixOperation::CKNVector *pSrcVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize, double fFirstIndex)

 {

     if( m_nTotalNode <= 3 )

     {

     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);

     MPI_Allgatherv(pSrcVector->m_vectValueRealBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,

         pResultVector->m_vectValueRealBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);

     MPI_Allgatherv(pSrcVector->m_vectValueImaginaryBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,

         pResultVector->m_vectValueImaginaryBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);

     }

     else

     {

         double          fCurrentRankPos = fFirstIndex;

         long long       fPrevRankPos = -1, fNextRankPos = -1;

         int             nPrevRank = (m_nCurrentRank-1+m_nTotalNode)%m_nTotalNode;

         int             nNextRank = (m_nCurrentRank+1)%m_nTotalNode;

         long long       nMax;

         double          *pSendBuffer = NULL, *pRecvBuffer = NULL;

         MPI_Request     req[2];

         MPI_Status      status[2];


         if( 0 == m_nCurrentRank )

             fPrevRankPos = nMergeSize - GetLoadBalanceCount(nPrevRank);

         else

             fPrevRankPos = fFirstIndex - GetLoadBalanceCount(nPrevRank);


         if (m_nCurrentRank == m_nTotalNode - 1)

             fNextRankPos = 0;

         else

             fNextRankPos = fCurrentRankPos + GetCurrentLoadBalanceCount();


 #ifdef _WIN32

             nMax = max(GetLoadBalanceCount(nPrevRank), GetLoadBalanceCount(nNextRank));

 #else //_WIN32

             nMax = std::max(GetLoadBalanceCount(nPrevRank), GetLoadBalanceCount(nNextRank));

 #endif//


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_MALLOC);

         pRecvBuffer = (double*)malloc(sizeof(double)*nMax*2);

         pSendBuffer = (double*)malloc(sizeof(double)*GetLoadBalanceCount(m_nCurrentRank)*2);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_MALLOC);


         memcpy(pSendBuffer, pSrcVector->m_vectValueRealBuffer.data(), sizeof(double)*GetLoadBalanceCount(m_nCurrentRank));

         memcpy(pSendBuffer+ GetLoadBalanceCount(m_nCurrentRank), pSrcVector->m_vectValueImaginaryBuffer.data() , sizeof(double)*GetLoadBalanceCount(m_nCurrentRank));


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);

         MPI_Irecv(pRecvBuffer, 2 * GetLoadBalanceCount(nPrevRank), MPI_DOUBLE, nPrevRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);

         MPI_Isend(pSendBuffer, 2 * GetLoadBalanceCount(m_nCurrentRank), MPI_DOUBLE, nNextRank, nNextRank, m_mpiCommIndex, &req[1]);

         MPI_Waitall(2, req, status);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);


         memcpy(pResultVector->m_vectValueRealBuffer.data() + fPrevRankPos, pRecvBuffer, GetLoadBalanceCount(nPrevRank) * sizeof(double));

         memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + fPrevRankPos, pRecvBuffer + GetLoadBalanceCount(nPrevRank), GetLoadBalanceCount(nPrevRank) * sizeof(double));


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);

         MPI_Irecv(pRecvBuffer, 2 * GetLoadBalanceCount(nNextRank), MPI_DOUBLE, nNextRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);

         MPI_Isend(pSendBuffer, 2 * GetLoadBalanceCount(m_nCurrentRank), MPI_DOUBLE, nPrevRank, nPrevRank, m_mpiCommIndex, &req[1]);

         MPI_Waitall(2, req, status);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);

         memcpy(pResultVector->m_vectValueRealBuffer.data() + fNextRankPos, pRecvBuffer, GetLoadBalanceCount(nNextRank) * sizeof(double));

         memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + fNextRankPos, pRecvBuffer + GetLoadBalanceCount(nNextRank), GetLoadBalanceCount(nNextRank) * sizeof(double));


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_FREE_MEM);

         FREE_MEM(pRecvBuffer);

         FREE_MEM(pSendBuffer);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_FREE_MEM);


         memcpy(pResultVector->m_vectValueRealBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueRealBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));

         memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueImaginaryBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));

     }

 }


 void CKNMPIManager::SplitVector(CKNMatrixOperation::CKNVector *pVector, int nRootRank)

 {

     LPCOMPLEX_NUMBER    lpSendBuffer = NULL;

     LPCOMPLEX_NUMBER    lpRecvBuffer = NULL;

     unsigned int        i;

     int                 *pSendCounts = NULL, *pDispls = NULL;


 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);

     pSendCounts = (int*)malloc(sizeof(int)*GetTotalNodeCount());

     pDispls = (int*)malloc(sizeof(int)*GetTotalNodeCount());

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);


     pDispls[0] = 0;

     pSendCounts[0] = 2 * GetLoadBalanceCount(0);

     for (i = 1; i < (unsigned int)GetTotalNodeCount(); i++)

     {

         pSendCounts[i] = 2 * GetLoadBalanceCount(i);

         pDispls[i] = pDispls[i - 1] + 2 * GetLoadBalanceCount(i - 1);

     }


     if (nRootRank == GetCurrentRank())

     {

         lpSendBuffer = ConvertVectorToMPIComplexBuffer(pVector);

         pVector->Finalize();

     }


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);

     lpRecvBuffer = (LPCOMPLEX_NUMBER)malloc(sizeof(COMPLEX_NUMBER)*GetCurrentLoadBalanceCount() * 2);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);


     if (NULL == lpRecvBuffer)

         throw ERROR_MALLOC;


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     MPI_Scatterv(lpSendBuffer, pSendCounts, pDispls, MPI_DOUBLE,

         lpRecvBuffer, 2 * GetCurrentLoadBalanceCount(), MPI_DOUBLE, nRootRank, m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);


     pVector->SetSize(GetCurrentLoadBalanceCount());

     for (i = 0; i < (unsigned int)GetCurrentLoadBalanceCount(); i++)

         pVector->SetAt(i, lpRecvBuffer[i].fReal, lpRecvBuffer[i].fImginary);


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::FREE_MEM);

     FREE_MEM(pSendCounts);

     FREE_MEM(pDispls);

     FREE_MEM(lpRecvBuffer);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::FREE_MEM);

 }


 void CKNMPIManager::BroadcastBool(bool *boolValue, int nRootRank)

 {

 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     MPI_Bcast(boolValue, 1, MPI_C_BOOL, nRootRank, m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

 }


 void CKNMPIManager::BroadcastDouble(double *pValue, unsigned int nSize, int nRootRank, MPI_Comm comm)

 {

 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     if( MPI_COMM_NULL == comm )

         MPI_Bcast(pValue, nSize, MPI_DOUBLE, nRootRank, m_mpiCommIndex);

     else

         MPI_Bcast(pValue, nSize, MPI_DOUBLE, nRootRank, comm);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

 }


 void CKNMPIManager::BroadcastInt(int *pValue, unsigned int nSize, int nRootRank, MPI_Comm comm)

 {

 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     if( MPI_COMM_NULL == comm )

         MPI_Bcast(pValue, nSize, MPI_INT, nRootRank, m_mpiCommIndex);

     else

         MPI_Bcast(pValue, nSize, MPI_INT, nRootRank, comm);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

 }


 void CKNMPIManager::BroadcastLanczosResult(CKNLanczosMethod::LPEIGENVALUE_RESULT lpResult, int nIterationCount)

 {

     unsigned int        nCastData[4];

     unsigned int        i;


 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     if( IsRootRank() )

     {

         nCastData[0] = lpResult->nEigenValueCount;

         nCastData[1] = lpResult->nEigenValueCountForMemeory;

         nCastData[2] = lpResult->nMaxEigenValueFoundIteration;

         nCastData[3] = lpResult->nEigenVectorSize;

     }


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     MPI_Bcast(nCastData, 4, MPI_INT, GetRootRank(), m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);


     if( !IsRootRank() )

     {

         lpResult->nEigenValueCount = nCastData[0];

         lpResult->nEigenValueCountForMemeory = nCastData[1];

         lpResult->nMaxEigenValueFoundIteration = nCastData[2];

         lpResult->nEigenVectorSize = nCastData[3];


         CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);

         lpResult->pEigenValueFoundIteration = (unsigned int*)malloc(sizeof(unsigned int)*lpResult->nEigenValueCount);

         lpResult->pEigenVectors = (double**)malloc(sizeof(double*)*lpResult->nEigenValueCount);

         for (i = 0; i < lpResult->nEigenValueCount; ++i)

             lpResult->pEigenVectors[i] = (double*)malloc(sizeof(double)*lpResult->nEigenVectorSize);

         CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);

     }


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     MPI_Bcast(lpResult->pEigenValueFoundIteration, lpResult->nEigenValueCount, MPI_INT, GetRootRank(), m_mpiCommIndex);

     for (i = 0; i < lpResult->nEigenValueCount; ++i)

         MPI_Bcast(lpResult->pEigenVectors[i], lpResult->nEigenVectorSize, MPI_DOUBLE, GetRootRank(), m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

 }


 void CKNMPIManager::AllReduceComlex(CKNComplex *pNumber, CKNTimeMeasurement::MEASUREMENT_INDEX INDEX)

 {

 #ifdef DISABLE_MPI_ROUTINE

     return;

 #endif


     double          fSend[2], fRecv[2];


     fSend[0] = pNumber->GetRealNumber();

     fSend[1] = pNumber->GetImaginaryNumber();


     CKNTimeMeasurement::MeasurementStart(INDEX);

     MPI_Allreduce(fSend, fRecv, 2, MPI_DOUBLE, MPI_SUM, m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(INDEX);


     pNumber->SetComplexNumber(fRecv[0], fRecv[1]);

 }


 double CKNMPIManager::AllReduceDouble(double fNumber)

 {

 #ifdef DISABLE_MPI_ROUTINE

     return fNumber;

 #endif


     double          fRecv;


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);

     MPI_Allreduce(&fNumber, &fRecv, 1, MPI_DOUBLE, MPI_SUM, m_mpiCommIndex);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);


     return fRecv;

 }


 CKNMPIManager::LPCOMPLEX_NUMBER CKNMPIManager::ConvertVectorToMPIComplexBuffer(CKNMatrixOperation::CKNVector *pVector)

 {

     LPCOMPLEX_NUMBER            lpResult = NULL;

     unsigned int                i, nCount;


     nCount = pVector->GetSize();

     lpResult = m_pConvertingBuffer;


     for (i = 0; i < nCount; i++)

     {

         lpResult[i].fReal = pVector->GetAt(i).GetRealNumber();

         lpResult[i].fImginary = pVector->GetAt(i).GetImaginaryNumber();

     }


     return lpResult;

 }


 int CKNMPIManager::GetCurrentLoadBalanceCount()

 {

 #ifdef DISABLE_MPI_ROUTINE

     return 0;

 #endif


     return m_pLoadBalance[m_nCurrentRank];

 }


 void CKNMPIManager::FinalizeManager()

 {

     m_bStartMPI = false;

     m_nCurrentRank = 0;

     m_nTotalNode = 1;

     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::FREE_MEM);

     FREE_MEM(m_pLoadBalance);

     FREE_MEM(m_pRecvCount);

     FREE_MEM(m_pSendCount);

     FREE_MEM(m_pBankInfo);

     FREE_MEM(m_pDispls);

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::FREE_MEM);

     if( MPI_GROUP_EMPTY != m_lanczosGroup )

         MPI_Group_free(&m_lanczosGroup);

     if( MPI_GROUP_EMPTY != m_deflationGroup )

         MPI_Group_free(&m_deflationGroup);

     if( MPI_COMM_NULL != m_mpiCommIndex && MPI_COMM_WORLD != m_mpiCommIndex)

         MPI_Comm_free(&m_mpiCommIndex);

     if( MPI_COMM_NULL != m_deflationComm )

         MPI_Comm_free(&m_deflationComm);

 }


 void CKNMPIManager::InitCommunicationBufferMetric()

 {

     unsigned int                i;


     CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);

     m_pRecvCount = (int*)malloc(sizeof(int)*GetTotalNodeCount());

     m_pSendCount = (int*)malloc(sizeof(int)*GetTotalNodeCount());

     m_pDispls = (int*)malloc(sizeof(int)*GetTotalNodeCount());

     m_pDispls[0] = 0;

     m_pRecvCount[0] = GetLoadBalanceCount(0);

     for (i = 1; i < (unsigned int)GetTotalNodeCount(); i++)

     {

         m_pRecvCount[i] = GetLoadBalanceCount(i);

         m_pDispls[i] = m_pDispls[i - 1] + GetLoadBalanceCount(i - 1);

     }

     CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);

 }


 void CKNMPIManager::SendDoubleBufferSync(int nTargetRank, double *pBuffer, int nSize, MPI_Request *req, MPI_Comm commWorld)

 {

     int nRank = CKNMPIManager::GetCurrentRank();

     if( MPI_COMM_NULL == commWorld)

         //MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, 0, m_mpiCommIndex);

         MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, nTargetRank, m_mpiCommIndex);

     else

         if( MPI_SUCCESS != MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, nTargetRank, commWorld) )

         //if( MPI_SUCCESS != MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, 0, commWorld) )

             printf("Oh my god!\n");

 }


 void CKNMPIManager::WaitSendDoubleBufferSync(MPI_Request *req)

 {

     MPI_Status      status;


     MPI_Wait(&m_SendDoubleAsyncRequest, &status);

 }


 void CKNMPIManager::ReceiveDoubleBufferSync(int nSourceRank, double *pBuffer, int nSize, MPI_Request *req, MPI_Comm commWorld)

 {

     MPI_Status status;

     if( MPI_COMM_NULL == commWorld)

         //MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, 0, m_mpiCommIndex, &status);

         MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, CKNMPIManager::GetCurrentRank(), m_mpiCommIndex, &status);

     else

         //MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, 0, commWorld, &status);

         MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, CKNMPIManager::GetCurrentRank(commWorld), commWorld, &status);

 }


 void CKNMPIManager::WaitReceiveDoubleBufferAsync(MPI_Request *req)

 {

     MPI_Status      status;


     MPI_Wait(&m_ReceiveDoubleAsyncRequest, &status);

 }


 void CKNMPIManager::BarrierAllComm()

 {

     MPI_Barrier(MPI_COMM_WORLD);

 }


 int* CKNMPIManager::GetEigenvalueCountFromDeflationGroup(int nDeflationGroupCount, int nLocalEVCount)

 {

     int             *pEVCount = NULL;


     if( IsRootRank(m_deflationComm ) )

         pEVCount = (int*)malloc(sizeof(int)*nDeflationGroupCount);


     MPI_Gather(&nLocalEVCount, 1, MPI_INT, pEVCount, 1, MPI_INT, 0, m_deflationComm);


     return pEVCount;

 }


 void CKNMPIManager::GatherVDouble(int nSourceCount, double *pReceiveBuffer, int *pSourceCount, double *pSendBuffer, int nSendCount, MPI_Comm comm)

 {

     int             *pReceiveCount = NULL;

     int             *pDisp = NULL;

     unsigned int    i;


     if( IsDeflationRoot() )

     {

         pDisp = (int*)malloc(sizeof(int)*nSourceCount);

         pDisp[0] = 0;

         for( i = 1; i < nSourceCount ; ++i)

             pDisp[i] = pDisp[i-1] + pSourceCount[i-1];

     }


     if( MPI_COMM_NULL == comm )

         MPI_Gatherv(pSendBuffer, nSendCount, MPI_DOUBLE, pReceiveBuffer, pSourceCount, pDisp, MPI_DOUBLE, 0, m_mpiCommIndex);

     else

         MPI_Gatherv(pSendBuffer, nSendCount, MPI_DOUBLE, pReceiveBuffer, pSourceCount, pDisp, MPI_DOUBLE, 0, comm);


     FREE_MEM(pDisp);

 }


 void CKNMPIManager::GatherVInt(int nSourceCount, int *pReceiveBuffer, int *pSourceCount, int *pSendBuffer, int nSendCount, MPI_Comm comm)

 {

     int             *pReceiveCount = NULL;

     int             *pDisp = NULL;

     unsigned int    i;


     if( IsDeflationRoot() )

     {

         pDisp = (int*)malloc(sizeof(int)*nSourceCount);

         pDisp[0] = 0;

         for( i = 1; i < nSourceCount ; ++i)

             pDisp[i] = pDisp[i-1] + pSourceCount[i-1];

     }


     if( MPI_COMM_NULL == comm )

         MPI_Gatherv(pSendBuffer, nSendCount, MPI_INT, pReceiveBuffer, pSourceCount, pDisp, MPI_INT, 0, m_mpiCommIndex);

     else

         MPI_Gatherv(pSendBuffer, nSendCount, MPI_INT, pReceiveBuffer, pSourceCount, pDisp, MPI_INT, 0, comm);


     FREE_MEM(pDisp);

 }


 void CKNMPIManager::ExchangeCommand(double *pfCommand, MPI_Comm comm)

 {

     BroadcastDouble(pfCommand, COMMAND_SIZE, 0, comm);

 }


 void CKNMPIManager::SendVectorSync(int nTargetRank, CKNMatrixOperation::CKNVector *pVector, int nSize, MPI_Request *req, MPI_Comm commWorld)

 {

     double              *pBuffer = NULL;


     pBuffer = (double*)malloc(sizeof(double)*nSize*2);

     pVector->Serialize(pBuffer, false);


     SendDoubleBufferSync(nTargetRank, pBuffer, nSize * 2, req, commWorld);


     FREE_MEM(pBuffer);

 }


 void CKNMPIManager::ReceiveVectorSync(int nSourceRank, CKNMatrixOperation::CKNVector *pVector, int nSize, MPI_Request *req, MPI_Comm commWorld)

 {

     double              *pBuffer = NULL;


     pBuffer = (double*)malloc(sizeof(double)*nSize*2);


     ReceiveDoubleBufferSync(nSourceRank, pBuffer, nSize * 2, req, commWorld);


     pVector->Serialize(pBuffer, true);


     FREE_MEM(pBuffer);

 }

CKNMPIManager::BroadcastBool
static void BroadcastBool(bool *boolValue, int nRootRank=0)
Broadcst boolean value.
Definition: KNMPIManager.cpp:524

CKNMPIManager::m_pSendCount
static int * m_pSendCount
Sending count variable for MPI comminication.
Definition: KNMPIManager.h:94

CKNMatrixOperation::CKNVector::SetSize
void SetSize(unsigned int nSize)
Set Vector elements size.
Definition: KNMatrixOperation.cpp:40

CKNMPIManager::InitCommunicationBufferMetric
static void InitCommunicationBufferMetric()
Initializing MPI Communication buffer for MVMul.
Definition: KNMPIManager.cpp:718

CKNComplex::GetImaginaryNumber
double GetImaginaryNumber() const
Get imaginary part.
Definition: KNComplex.h:27

CKNMPIManager::BroadcastInt
static void BroadcastInt(int *pValue, unsigned int nSize, int nRootRank=0, MPI_Comm comm=MPI_COMM_NULL)
Broadcst boolean value.
Definition: KNMPIManager.cpp:560

CKNMPIManager::FinalizeManager
static void FinalizeManager()
Get Root rank.
Definition: KNMPIManager.cpp:693

CKNTimeMeasurement::MALLOC
Definition: KNTimeMeasurement.h:24

CKNIPCCUtility::ShowMsg
static void ShowMsg(char *pszBuffer)
Show message.
Definition: KNIPCCUtility.cpp:35

CKNMPIManager::SetMPIEnviroment
static void SetMPIEnviroment(int nRank, int nTotalNode)
Set MPI Enviroment.
Definition: KNMPIManager.cpp:139

CKNMPIManager::WaitSendDoubleBufferSync
static void WaitSendDoubleBufferSync(MPI_Request *req)
Waiting sending double buffer sync function.
Definition: KNMPIManager.cpp:757

CKNMPIManager::AllReduceComlex
static void AllReduceComlex(CKNComplex *pNumber, CKNTimeMeasurement::MEASUREMENT_INDEX INDEX=CKNTimeMeasurement::COMM)
Do all reduce function with CKNComplex.
Definition: KNMPIManager.cpp:625

CKNMPIManager::GetEigenvalueCountFromDeflationGroup
static int * GetEigenvalueCountFromDeflationGroup(int nDeflationGroupCount, int nLocalEVCount)
Checking is root rank of Deflation computation.
Definition: KNMPIManager.cpp:802

CKNMPIManager::BroadcastLanczosResult
static void BroadcastLanczosResult(CKNLanczosMethod::LPEIGENVALUE_RESULT lpResult, int nIterationCount)
Broadcast Lanczos result.
Definition: KNMPIManager.cpp:578

CKNMPIManager::InitLevel
static bool InitLevel(int nMPILevel, int nFindingDegeneratedEVCount)
Init MPI Level, most low level is for multi node cacluation for Lanczos.
Definition: KNMPIManager.cpp:51

CKNMPIManager::m_nMPILevel
static unsigned int m_nMPILevel
MPI Level.
Definition: KNMPIManager.h:99

CKNMPIManager::m_pCommBuffer
static LPCOMPLEX_NUMBER m_pCommBuffer
Data buffer for MPI Communication.
Definition: KNMPIManager.h:91

CKNMPIManager::BroadcastDouble
static void BroadcastDouble(double *pValue, unsigned int nSize, int nRootRank=0, MPI_Comm comm=MPI_COMM_NULL)
Broadcst boolean value.
Definition: KNMPIManager.cpp:540

CKNComplex::GetRealNumber
double GetRealNumber() const
Get real part.
Definition: KNComplex.h:26

CKNMPIManager::SplitVector
static void SplitVector(CKNMatrixOperation::CKNVector *pVector, int nRootRank)
Split vector to sub rank.
Definition: KNMPIManager.cpp:467

CKNMPIManager::GetCurrentLoadBalanceCount
static int GetCurrentLoadBalanceCount()
Get Current node's rank load balancing number.
Definition: KNMPIManager.cpp:684

CKNTimeMeasurement::MeasurementEnd
static void MeasurementEnd(MEASUREMENT_INDEX index)
Measurement end for part.
Definition: KNTimeMeasurement.cpp:87

CKNLanczosMethod::EIGENVALUE_RESULT::nEigenVectorSize
unsigned int nEigenVectorSize
Definition: KNLanczosMethod.h:34

CKNMPIManager::MergeVector
static void MergeVector(CKNMatrixOperation::CKNVector *pVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize)
Merge vector to sub rank.
Definition: KNMPIManager.cpp:276

CKNMPIManager::GetTotalNodeCount
static int GetTotalNodeCount()
Definition: KNMPIManager.h:44

CKNMPIManager::COMPLEX_NUMBER
Complex number.
Definition: KNMPIManager.h:27

CKNMPIManager::CheckDeflationNodeCount
static bool CheckDeflationNodeCount(int nNeedNodeCount)
Checking node counts fix to deflation group.
Definition: KNMPIManager.cpp:127

CKNMPIManager::ReceiveVectorSync
static void ReceiveVectorSync(int nSourceRank, CKNMatrixOperation::CKNVector *pVector, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
Receiving Vector with sync.
Definition: KNMPIManager.cpp:910

CKNMPIManager::GetLoadBalanceCount
static int GetLoadBalanceCount(int nRank)
Definition: KNMPIManager.cpp:203

CKNMPIManager::ConvertVectorToMPIComplexBuffer
static LPCOMPLEX_NUMBER ConvertVectorToMPIComplexBuffer(CKNMatrixOperation::CKNVector *pVector)
Convert vector class to MPI_COMPLEX array.
Definition: KNMPIManager.cpp:667

CKNMPIManager::AllReduceDouble
static double AllReduceDouble(double fNumber)
Do all reduce function with CKNComplex.
Definition: KNMPIManager.cpp:647

CKNMPIManager::GetRootRank
static int GetRootRank()
Definition: KNMPIManager.h:60

CKNMPIManager::m_deflationComm
static MPI_Comm m_deflationComm
Deflation computing MPI_Comm.
Definition: KNMPIManager.h:102

CKNMPIManager::m_pRecvCount
static int * m_pRecvCount
Reciving count variable for MPI comminication.
Definition: KNMPIManager.h:93

CKNMPIManager::m_nCommWorldRank
static int m_nCommWorldRank
MPI Rank before split.
Definition: KNMPIManager.h:87

CKNMPIManager::m_ReceiveDoubleAsyncRequest
static MPI_Request m_ReceiveDoubleAsyncRequest
Request for receving double.
Definition: KNMPIManager.h:98

CKNTimeMeasurement::MV_COMM
Definition: KNTimeMeasurement.h:24

CKNMPIManager::IsInMPIRoutine
static bool IsInMPIRoutine()
Definition: KNMPIManager.h:47

CKNMPIManager::m_nTotalNode
static int m_nTotalNode
Total node count.
Definition: KNMPIManager.h:88

CKNMPIManager::LoadBlancing
static void LoadBlancing(int nElementCount)
Load blancing for MPI, This function for lanczos solving with geometric constrcution.
Definition: KNMPIManager.cpp:189

CKNIPCCUtility::SetShow
static void SetShow(bool bShow)
Definition: KNIPCCUtility.h:30

CKNMPIManager::GatherVInt
static void GatherVInt(int nSourceCount, int *pReceiveBuffer, int *pSourceCount, int *pSendBuffer, int nSendCount, MPI_Comm comm=MPI_COMM_NULL)
GahterV for int wrapping function.
Definition: KNMPIManager.cpp:852

CKNMPIManager::m_SendDoubleAsyncRequest
static MPI_Request m_SendDoubleAsyncRequest
Request for sending double.
Definition: KNMPIManager.h:97

CKNMPIManager::CKNMPIManager
CKNMPIManager()
Constructor.
Definition: KNMPIManager.cpp:38

CKNGlobal.h
Common definition for Solver.

CKNMPIManager::SendVectorSync
static void SendVectorSync(int nTargetRank, CKNMatrixOperation::CKNVector *pVector, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
Getting Deflation computing group MPI_Comm.
Definition: KNMPIManager.cpp:890

CKNMPIManager::m_pBankInfo
static int * m_pBankInfo
After MPI Split bank infomation.
Definition: KNMPIManager.h:95

CKNMatrixOperation::CKNVector::GetAt
CKNComplex GetAt(unsigned int nIndex)
Get element value from specific index.
Definition: KNMatrixOperation.cpp:100

CKNMPIManager::MergeVectorEx_Optimal
static void MergeVectorEx_Optimal(CKNMatrixOperation::CKNVector *pVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize, double fFirstIndex, unsigned int nSizeFromPrevRank, unsigned int nSizeFromNextRank, unsigned int nSizetoPrevRank, unsigned int nSizetoNextRank, unsigned int *)
Merge vector for 1 layer exchanging.
Definition: KNMPIManager.cpp:317

CKNMatrixOperation::CKNVector::Serialize
bool Serialize(double *pBuffer, bool bStore)
Serialize vector.
Definition: KNMatrixOperation.cpp:401

CKNMPIManager::IsLanczosComputeRoot
static bool IsLanczosComputeRoot()
Barrier current deflation group.
Definition: KNMPIManager.h:71

CKNLanczosMethod::EIGENVALUE_RESULT
Structure for engienvalue computing.
Definition: KNLanczosMethod.h:30

CKNTimeMeasurement::MEASUREMENT_INDEX
MEASUREMENT_INDEX
Definition: KNTimeMeasurement.h:24

CKNLanczosMethod::EIGENVALUE_RESULT::pEigenValueFoundIteration
unsigned int * pEigenValueFoundIteration
Definition: KNLanczosMethod.h:39

KNTimeMeasurement.h
Time measurement class.

CKNTimeMeasurement::FREE_MEM
Definition: KNTimeMeasurement.h:24

CKNMatrixOperation::CKNVector::m_vectValueImaginaryBuffer
double_vector_t m_vectValueImaginaryBuffer
A member variable for saving none zero elements.
Definition: KNMatrixOperation.h:105

CKNLanczosMethod::EIGENVALUE_RESULT::nMaxEigenValueFoundIteration
unsigned int nMaxEigenValueFoundIteration
Definition: KNLanczosMethod.h:38

CKNMPIManager::m_bMultiLevel
static bool m_bMultiLevel
Flag for Multilevel MPI group.
Definition: KNMPIManager.h:106

CKNMPIManager::~CKNMPIManager
~CKNMPIManager()
Destructor.
Definition: KNMPIManager.cpp:42

CKNMatrixOperation::CKNVector::Finalize
void Finalize()
Free allocated memory for vector elements.
Definition: KNMatrixOperation.cpp:365

KNIPCCUtility.h
This class includes functions for matrix debugging.

CKNMPIManager::m_pConvertingBuffer
static LPCOMPLEX_NUMBER m_pConvertingBuffer
Data buffer for Vector converting.
Definition: KNMPIManager.h:92

CKNMPIManager::m_pDispls
static int * m_pDispls
Displ for MPI comminication.
Definition: KNMPIManager.h:96

CKNMPIManager::ExchangeCommand
static void ExchangeCommand(double *pfCommand, MPI_Comm comm)
Gather eigenvalue finding iteration number from deflation group.
Definition: KNMPIManager.cpp:878

CKNMPIManager::GetCurrentRank
static int GetCurrentRank()
Definition: KNMPIManager.h:42

CKNMPIManager::COMPLEX_NUMBER::fReal
double fReal
Definition: KNMPIManager.h:28

CKNMatrixOperation::CKNVector::m_vectValueRealBuffer
double_vector_t m_vectValueRealBuffer
A member variable for saving none zero elements.
Definition: KNMatrixOperation.h:104

CKNMPIManager::GatherVDouble
static void GatherVDouble(int nSourceCount, double *pReceiveBuffer, int *pSourceCount, double *pSendBuffer, int nSendCount, MPI_Comm comm=MPI_COMM_NULL)
GatherV for double wrapping function.
Definition: KNMPIManager.cpp:822

CKNMPIManager::MergeVectorOptimal
static void MergeVectorOptimal(CKNMatrixOperation::CKNVector *pSrcVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize, double fFirstIndex)
Merge vector to sub rank, operated without vector class member function call.
Definition: KNMPIManager.cpp:389

CKNLanczosMethod::EIGENVALUE_RESULT::pEigenVectors
double ** pEigenVectors
Definition: KNLanczosMethod.h:33

CKNTimeMeasurement::MV_FREE_MEM
Definition: KNTimeMeasurement.h:24

CKNMatrixOperation::CKNVector::SetAt
void SetAt(unsigned int nIndex, CKNComplex value)
Set element value in specific index, Call by value.
Definition: KNMatrixOperation.cpp:54

CKNMPIManager::m_deflationGroup
static MPI_Group m_deflationGroup
MPI Group for Deflation computation.
Definition: KNMPIManager.h:104

CKNLanczosMethod::EIGENVALUE_RESULT::nEigenValueCount
unsigned int nEigenValueCount
Definition: KNLanczosMethod.h:36

FREE_MEM
#define FREE_MEM(pointer)
Macro for memory allocation and assign null value.
Definition: CKNGlobal.h:20

CKNMatrixOperation::CKNVector::GetSize
unsigned int GetSize()
Return Vector elements size.
Definition: KNMatrixOperation.h:71

CKNLanczosMethod::EIGENVALUE_RESULT::nEigenValueCountForMemeory
unsigned int nEigenValueCountForMemeory
Definition: KNLanczosMethod.h:41

CKNMPIManager::m_pLoadBalance
static int * m_pLoadBalance
Load blancing for MPI Communication.
Definition: KNMPIManager.h:90

CKNTimeMeasurement::MV_MALLOC
Definition: KNTimeMeasurement.h:24

ERROR_MALLOC
const unsigned long ERROR_MALLOC
Error code that means error occur during memory allocation.
Definition: CKNGlobal.h:62

CKNComplex
This class for complex operation and saving value.
Definition: KNComplex.h:18

KNMPIManager.h
MPI Mangement class.

CKNMPIManager::BarrierAllComm
static void BarrierAllComm()
Is Multilevel MPI Setting.
Definition: KNMPIManager.cpp:791

CKNMPIManager::m_mpiCommIndex
static MPI_Comm m_mpiCommIndex
Lanczos Method MPI_Comm.
Definition: KNMPIManager.h:101

CKNTimeMeasurement::MeasurementStart
static void MeasurementStart(MEASUREMENT_INDEX index)
Measurement start for part.
Definition: KNTimeMeasurement.cpp:73

CKNMPIManager::WaitReceiveDoubleBufferAsync
static void WaitReceiveDoubleBufferAsync(MPI_Request *req)
Waiting recevinging double buffer sync function.
Definition: KNMPIManager.cpp:784

CKNMPIManager::LoadBlancingForLanczos
static void LoadBlancingForLanczos(int nRowCount)
Load blancing for MPI, this function only for lanczos solving without geometric constrcution.
Definition: KNMPIManager.cpp:150

CKNMPIManager::m_bNeedPostOperation
static bool m_bNeedPostOperation[10]
MPI Level.
Definition: KNMPIManager.h:100

CKNMPIManager::m_lanczosGroup
static MPI_Group m_lanczosGroup
MPI Group for Lanczos computation.
Definition: KNMPIManager.h:103

CKNMPIManager::m_nCurrentRank
static int m_nCurrentRank
Getting Lanczos group index.
Definition: KNMPIManager.h:83

CKNMPIManager::IsRootRank
static bool IsRootRank()
Get Total node count.
Definition: KNMPIManager.cpp:217

CKNTimeMeasurement::COMM
Definition: KNTimeMeasurement.h:24

COMMAND_SIZE
#define COMMAND_SIZE
Definition: CKNGlobal.h:102

CKNMPIManager::m_bStartMPI
static bool m_bStartMPI
MPI_Init call or not.
Definition: KNMPIManager.h:89

CKNMPIManager::SendDoubleBufferSync
static void SendDoubleBufferSync(int nTargetRank, double *pBuffer, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
Sending buffer for double data array with sync.
Definition: KNMPIManager.cpp:742

CKNComplex::SetComplexNumber
void SetComplexNumber(double fReal, double fImaginaray)
Set Complex number using real part and imaginary part.
Definition: KNComplex.cpp:59

CKNMPIManager::LPCOMPLEX_NUMBER
struct CKNMPIManager::COMPLEX_NUMBER * LPCOMPLEX_NUMBER

CKNMPIManager::m_nLanczosGroupIndex
static unsigned int m_nLanczosGroupIndex
MPI Group index for Lanczos group.
Definition: KNMPIManager.h:105

CKNMPIManager::COMPLEX_NUMBER::fImginary
double fImginary
Definition: KNMPIManager.h:29

CKNMatrixOperation::CKNVector
This class for describing vector for Lanczos method.
Definition: KNMatrixOperation.h:60

CKNMPIManager::BroadcastVector
static void BroadcastVector(CKNMatrixOperation::CKNVector *pVector)
Check this processing running on MPI Enviorment.
Definition: KNMPIManager.cpp:263

CKNMPIManager::IsDeflationRoot
static bool IsDeflationRoot()
Checking is root rank of Lanczos computation.
Definition: KNMPIManager.h:72

CKNMPIManager::ReceiveDoubleBufferSync
static void ReceiveDoubleBufferSync(int nSourceRank, double *pBuffer, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
Receivinging buffer for double data array with sync.
Definition: KNMPIManager.cpp:770