CKNMPIManager Class Reference

MPI Mangement class. More...

#include "KNMPIManager.h"

Collaboration diagram for CKNMPIManager:

Classes
struct	COMPLEX_NUMBER
	Complex number. More...

Public Member Functions
	CKNMPIManager ()
	Constructor. More...
	~CKNMPIManager ()
	Destructor. More...

Static Public Member Functions
static bool	InitLevel (int nMPILevel, int nFindingDegeneratedEVCount)
	Init MPI Level, most low level is for multi node cacluation for Lanczos. More...
static bool	CheckDeflationNodeCount (int nNeedNodeCount)
	Checking node counts fix to deflation group. More...
static void	SetMPIEnviroment (int nRank, int nTotalNode)
	Set MPI Enviroment. More...
static void	LoadBlancingForLanczos (int nRowCount)
	Load blancing for MPI, this function only for lanczos solving without geometric constrcution. More...
static void	LoadBlancing (int nElementCount)
	Load blancing for MPI, This function for lanczos solving with geometric constrcution. More...
static int	GetCurrentLoadBalanceCount ()
	Get Current node's rank load balancing number. More...
static int	GetLoadBalanceCount (int nRank)
static int	GetCurrentRank ()
static int	GetCurrentRank (MPI_Comm comm)
	Get Current node's rank number. More...
static int	GetTotalNodeCount ()
static bool	IsRootRank ()
	Get Total node count. More...
static bool	IsRootRank (MPI_Comm comm)
	Check this node is root rank in 'comm' MPI_Comm. More...
static bool	IsInMPIRoutine ()
static void	BroadcastVector (CKNMatrixOperation::CKNVector *pVector)
	Check this processing running on MPI Enviorment. More...
static void	BroadcastBool (bool *boolValue, int nRootRank=0)
	Broadcst boolean value. More...
static void	BroadcastDouble (double *pValue, unsigned int nSize, int nRootRank=0, MPI_Comm comm=MPI_COMM_NULL)
	Broadcst boolean value. More...
static void	BroadcastInt (int *pValue, unsigned int nSize, int nRootRank=0, MPI_Comm comm=MPI_COMM_NULL)
	Broadcst boolean value. More...
static void	BroadcastLanczosResult (CKNLanczosMethod::LPEIGENVALUE_RESULT lpResult, int nIterationCount)
	Broadcast Lanczos result. More...
static void	SplitVector (CKNMatrixOperation::CKNVector *pVector, int nRootRank)
	Split vector to sub rank. More...
static void	MergeVector (CKNMatrixOperation::CKNVector *pVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize)
	Merge vector to sub rank. More...
static void	MergeVectorOptimal (CKNMatrixOperation::CKNVector *pSrcVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize, double fFirstIndex)
	Merge vector to sub rank, operated without vector class member function call. More...
static void	MergeVectorEx_Optimal (CKNMatrixOperation::CKNVector *pVector, CKNMatrixOperation::CKNVector *pResultVector, unsigned int nMergeSize, double fFirstIndex, unsigned int nSizeFromPrevRank, unsigned int nSizeFromNextRank, unsigned int nSizetoPrevRank, unsigned int nSizetoNextRank, unsigned int *)
	Merge vector for 1 layer exchanging. More...
static LPCOMPLEX_NUMBER	ConvertVectorToMPIComplexBuffer (CKNMatrixOperation::CKNVector *pVector)
	Convert vector class to MPI_COMPLEX array. More...
static void	AllReduceComlex (CKNComplex *pNumber, CKNTimeMeasurement::MEASUREMENT_INDEX INDEX=CKNTimeMeasurement::COMM)
	Do all reduce function with CKNComplex. More...
static double	AllReduceDouble (double fNumber)
	Do all reduce function with CKNComplex. More...
static int	GetRootRank ()
static void	FinalizeManager ()
	Get Root rank. More...
static void	InitCommunicationBufferMetric ()
	Initializing MPI Communication buffer for MVMul. More...
static void	SendDoubleBufferSync (int nTargetRank, double *pBuffer, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
	Sending buffer for double data array with sync. More...
static void	WaitSendDoubleBufferSync (MPI_Request *req)
	Waiting sending double buffer sync function. More...
static void	ReceiveDoubleBufferSync (int nSourceRank, double *pBuffer, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
	Receivinging buffer for double data array with sync. More...
static void	WaitReceiveDoubleBufferAsync (MPI_Request *req)
	Waiting recevinging double buffer sync function. More...
static MPI_Comm	GetMPIComm ()
static bool	IsMultiLevelMPI ()
	Get MPI_Comm. More...
static void	BarrierAllComm ()
	Is Multilevel MPI Setting. More...
static void	Barrier ()
static bool	IsLanczosComputeRoot ()
	Barrier current deflation group. More...
static bool	IsDeflationRoot ()
	Checking is root rank of Lanczos computation. More...
static int *	GetEigenvalueCountFromDeflationGroup (int nDeflationGroupCount, int nLocalEVCount)
	Checking is root rank of Deflation computation. More...
static void	GatherVDouble (int nSourceCount, double *pReceiveBuffer, int *pSourceCount, double *pSendBuffer, int nSendCount, MPI_Comm comm=MPI_COMM_NULL)
	GatherV for double wrapping function. More...
static void	GatherVInt (int nSourceCount, int *pReceiveBuffer, int *pSourceCount, int *pSendBuffer, int nSendCount, MPI_Comm comm=MPI_COMM_NULL)
	GahterV for int wrapping function. More...
static void	GatherEVFromDeflationGroup (int nSourceCount, double *pReceiveBuffer, int *pSourceCount, double *pSendBuffer, int nSendCount)
static void	GatherEVIterationFromDeflationGroup (int nSourceCount, int *pReceiveBuffer, int *pSourceCount, int *pSendBuffer, int nSendCount)
	Gather eigenvalue from deflation group. More...
static void	ExchangeCommand (double *pfCommand, MPI_Comm comm)
	Gather eigenvalue finding iteration number from deflation group. More...
static MPI_Comm	GetLanczosComputComm ()
static MPI_Comm	GetDeflationComm ()
	Getting Lanczos computing group MPI_Comm. More...
static void	SendVectorSync (int nTargetRank, CKNMatrixOperation::CKNVector *pVector, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
	Getting Deflation computing group MPI_Comm. More...
static void	ReceiveVectorSync (int nSourceRank, CKNMatrixOperation::CKNVector *pVector, int nSize, MPI_Request *req, MPI_Comm commWorld=MPI_COMM_NULL)
	Receiving Vector with sync. More...
static unsigned int	GetLanczosGroupIndex ()

Private Types
typedef struct CKNMPIManager::COMPLEX_NUMBER *	LPCOMPLEX_NUMBER

Static Private Attributes
static int	m_nCurrentRank = 0
	Getting Lanczos group index. More...
static int	m_nCommWorldRank = 0
	MPI Rank before split. More...
static int	m_nTotalNode = 1
	Total node count. More...
static bool	m_bStartMPI = false
	MPI_Init call or not. More...
static int *	m_pLoadBalance = NULL
	Load blancing for MPI Communication. More...
static LPCOMPLEX_NUMBER	m_pCommBuffer = NULL
	Data buffer for MPI Communication. More...
static LPCOMPLEX_NUMBER	m_pConvertingBuffer = NULL
	Data buffer for Vector converting. More...
static int *	m_pRecvCount = NULL
	Reciving count variable for MPI comminication. More...
static int *	m_pSendCount = NULL
	Sending count variable for MPI comminication. More...
static int *	m_pBankInfo = NULL
	After MPI Split bank infomation. More...
static int *	m_pDispls = NULL
	Displ for MPI comminication. More...
static MPI_Request	m_SendDoubleAsyncRequest = MPI_REQUEST_NULL
	Request for sending double. More...
static MPI_Request	m_ReceiveDoubleAsyncRequest = MPI_REQUEST_NULL
	Request for receving double. More...
static unsigned int	m_nMPILevel = 1
	MPI Level. More...
static bool	m_bNeedPostOperation [10] = { false, false, false, false, false, false, false, false, false, false }
	MPI Level. More...
static MPI_Comm	m_mpiCommIndex = MPI_COMM_WORLD
	Lanczos Method MPI_Comm. More...
static MPI_Comm	m_deflationComm = MPI_COMM_NULL
	Deflation computing MPI_Comm. More...
static MPI_Group	m_lanczosGroup = MPI_GROUP_EMPTY
	MPI Group for Lanczos computation. More...
static MPI_Group	m_deflationGroup = MPI_GROUP_EMPTY
	MPI Group for Deflation computation. More...
static unsigned int	m_nLanczosGroupIndex = 0
	MPI Group index for Lanczos group. More...
static bool	m_bMultiLevel = false
	Flag for Multilevel MPI group. More...

Detailed Description

MPI Mangement class.

Date

2014/8/25

Author

Kyu Nam Cho(mysto.nosp@m.us@g.nosp@m.mail..nosp@m.com)

Definition at line 21 of file KNMPIManager.h.

Member Typedef Documentation

typedef struct CKNMPIManager::COMPLEX_NUMBER * CKNMPIManager::LPCOMPLEX_NUMBER

private

Constructor & Destructor Documentation

CKNMPIManager::CKNMPIManager

(

)

Constructor.

Definition at line 38 of file KNMPIManager.cpp.

{
}

CKNMPIManager::~CKNMPIManager

(

)

Destructor.

Definition at line 42 of file KNMPIManager.cpp.

{
}

Member Function Documentation

void CKNMPIManager::AllReduceComlex ( CKNComplex *  pNumber, CKNTimeMeasurement::MEASUREMENT_INDEX  INDEX = CKNTimeMeasurement::COMM  )

static

Do all reduce function with CKNComplex.

Parameters

pNumber	Variable that want to sum
INDEX	Time measurement index

Definition at line 625 of file KNMPIManager.cpp.

References CKNComplex::GetImaginaryNumber(), CKNComplex::GetRealNumber(), m_mpiCommIndex, CKNTimeMeasurement::MeasurementEnd(), CKNTimeMeasurement::MeasurementStart(), and CKNComplex::SetComplexNumber().

Referenced by CKNMatrixOperation::VVDot().

{
#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    double          fSend[2], fRecv[2];

    fSend[0] = pNumber->GetRealNumber();
    fSend[1] = pNumber->GetImaginaryNumber();

    CKNTimeMeasurement::MeasurementStart(INDEX);
    MPI_Allreduce(fSend, fRecv, 2, MPI_DOUBLE, MPI_SUM, m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(INDEX);

    pNumber->SetComplexNumber(fRecv[0], fRecv[1]);
}

Here is the call graph for this function:

Here is the caller graph for this function:

double CKNMPIManager::AllReduceDouble ( double  fNumber )

static

Do all reduce function with CKNComplex.

Parameters

fNumber

Variable that want to sum

Returns

Reducing result

Definition at line 647 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, m_mpiCommIndex, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNGeometricShape::CalculateUnitcellCount(), CKNMatrixOperation::CKNVector::GetNorm(), and CKNTBMS_Solver::Launching_TBMS_Solver().

{
#ifdef DISABLE_MPI_ROUTINE
    return fNumber;
#endif

    double          fRecv;

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    MPI_Allreduce(&fNumber, &fRecv, 1, MPI_DOUBLE, MPI_SUM, m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

    return fRecv;
}

Here is the call graph for this function:

Here is the caller graph for this function:

static void CKNMPIManager::Barrier ( )

inlinestatic

Definition at line 70 of file KNMPIManager.h.

References m_mpiCommIndex.

Referenced by CKNLanczosMethod::SaveLanczosResult().

{ MPI_Barrier(m_mpiCommIndex); }; 

Here is the caller graph for this function:

void CKNMPIManager::BarrierAllComm ( )

static

Is Multilevel MPI Setting.

Barrier MPI_COMM_WORLD

< Caution! This function wait all rank even if Comm split into several colors

Definition at line 791 of file KNMPIManager.cpp.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
    MPI_Barrier(MPI_COMM_WORLD);
}

Here is the caller graph for this function:

void CKNMPIManager::BroadcastBool ( bool *  boolValue, int  nRootRank = 0  )

static

Broadcst boolean value.

Parameters

boolValue	bool variable for want to broadcasting
nRootRank	Root rank index

Definition at line 524 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, m_mpiCommIndex, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNLanczosMethod::LanczosIterationLoop().

{
#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    MPI_Bcast(boolValue, 1, MPI_C_BOOL, nRootRank, m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::BroadcastDouble ( double *  pValue, unsigned int  nSize, int  nRootRank = 0, MPI_Comm  comm = MPI_COMM_NULL  )

static

Broadcst boolean value.

Parameters

pValue	double data buffer for want to broadcasting
nSize	Data buffer size
nRootRank	Root rank index

Definition at line 540 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, m_mpiCommIndex, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNLanczosMethod::DoResidualCheck(), CKNGeometricShape::ExchangeAtomInfoBetweenNode(), ExchangeCommand(), and CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    if( MPI_COMM_NULL == comm )
        MPI_Bcast(pValue, nSize, MPI_DOUBLE, nRootRank, m_mpiCommIndex);
    else
        MPI_Bcast(pValue, nSize, MPI_DOUBLE, nRootRank, comm);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::BroadcastInt ( int *  pValue, unsigned int  nSize, int  nRootRank = 0, MPI_Comm  comm = MPI_COMM_NULL  )

static

Broadcst boolean value.

Parameters

[in,out]	pValue	Broadcasting variable
	nSize	Broadcasting variable size
	nRootRank	Root rank number
	comm	Broadcating MPI_Comm range

Definition at line 560 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, m_mpiCommIndex, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    if( MPI_COMM_NULL == comm )
        MPI_Bcast(pValue, nSize, MPI_INT, nRootRank, m_mpiCommIndex);
    else
        MPI_Bcast(pValue, nSize, MPI_INT, nRootRank, comm);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::BroadcastLanczosResult ( CKNLanczosMethod::LPEIGENVALUE_RESULT  lpResult, int  nIterationCount  )

static

Broadcast Lanczos result.

Parameters

lpResult	Lanczos method result that want to boardcasting
nIterationCount	Current iteration count

Definition at line 578 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, GetRootRank(), IsRootRank(), m_mpiCommIndex, CKNTimeMeasurement::MALLOC, CKNTimeMeasurement::MeasurementEnd(), CKNTimeMeasurement::MeasurementStart(), CKNLanczosMethod::EIGENVALUE_RESULT::nEigenValueCount, CKNLanczosMethod::EIGENVALUE_RESULT::nEigenValueCountForMemeory, CKNLanczosMethod::EIGENVALUE_RESULT::nEigenVectorSize, CKNLanczosMethod::EIGENVALUE_RESULT::nMaxEigenValueFoundIteration, CKNLanczosMethod::EIGENVALUE_RESULT::pEigenValueFoundIteration, and CKNLanczosMethod::EIGENVALUE_RESULT::pEigenVectors.

Referenced by CKNLanczosMethod::LanczosIteration().

{
    unsigned int        nCastData[4];
    unsigned int        i;

#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    if( IsRootRank() )
    {
        nCastData[0] = lpResult->nEigenValueCount;
        nCastData[1] = lpResult->nEigenValueCountForMemeory;
        nCastData[2] = lpResult->nMaxEigenValueFoundIteration;
        nCastData[3] = lpResult->nEigenVectorSize;
    }

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    MPI_Bcast(nCastData, 4, MPI_INT, GetRootRank(), m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

    if( !IsRootRank() )
    {
        lpResult->nEigenValueCount = nCastData[0];
        lpResult->nEigenValueCountForMemeory = nCastData[1];
        lpResult->nMaxEigenValueFoundIteration = nCastData[2];
        lpResult->nEigenVectorSize = nCastData[3];

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);
        lpResult->pEigenValueFoundIteration = (unsigned int*)malloc(sizeof(unsigned int)*lpResult->nEigenValueCount);
        lpResult->pEigenVectors = (double**)malloc(sizeof(double*)*lpResult->nEigenValueCount);
        for (i = 0; i < lpResult->nEigenValueCount; ++i)
            lpResult->pEigenVectors[i] = (double*)malloc(sizeof(double)*lpResult->nEigenVectorSize);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);
    }

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    MPI_Bcast(lpResult->pEigenValueFoundIteration, lpResult->nEigenValueCount, MPI_INT, GetRootRank(), m_mpiCommIndex);
    for (i = 0; i < lpResult->nEigenValueCount; ++i)
        MPI_Bcast(lpResult->pEigenVectors[i], lpResult->nEigenVectorSize, MPI_DOUBLE, GetRootRank(), m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::BroadcastVector ( CKNMatrixOperation::CKNVector *  pVector )

static

Check this processing running on MPI Enviorment.

Broadcast vector to sub rank

Parameters

pVector

Vector for want to broadcast

Remarks

This function currently not used 2/Feb/2015

Definition at line 263 of file KNMPIManager.cpp.

References IsInMPIRoutine().

{
#ifdef DISABLE_MPI_ROUTINE
    if (!IsInMPIRoutine())
        return;
#endif
}

Here is the call graph for this function:

bool CKNMPIManager::CheckDeflationNodeCount ( int  nNeedNodeCount )

static

Checking node counts fix to deflation group.

Parameters

nNeedNodeCount

Deflation group count

Returns

Can make defaltion group or not

Definition at line 127 of file KNMPIManager.cpp.

References m_nTotalNode.

Referenced by InitLevel().

{
    if( 0 == m_nTotalNode % nNeedNodeCount )
        return true;
    else
        return false;
}

Here is the caller graph for this function:

CKNMPIManager::LPCOMPLEX_NUMBER CKNMPIManager::ConvertVectorToMPIComplexBuffer ( CKNMatrixOperation::CKNVector *  pVector )

static

Convert vector class to MPI_COMPLEX array.

Parameters

pVector

Converting target verctor

Returns

Conveting result that structure for complex part array (AoS type)

Remarks

For performance AoS type should be changed into SoA type

Definition at line 667 of file KNMPIManager.cpp.

References CKNMPIManager::COMPLEX_NUMBER::fImginary, CKNMPIManager::COMPLEX_NUMBER::fReal, CKNMatrixOperation::CKNVector::GetAt(), CKNComplex::GetImaginaryNumber(), CKNComplex::GetRealNumber(), CKNMatrixOperation::CKNVector::GetSize(), and m_pConvertingBuffer.

Referenced by SplitVector().

{
    LPCOMPLEX_NUMBER            lpResult = NULL;
    unsigned int                i, nCount;

    nCount = pVector->GetSize();
    lpResult = m_pConvertingBuffer;

    for (i = 0; i < nCount; i++)
    {
        lpResult[i].fReal = pVector->GetAt(i).GetRealNumber();
        lpResult[i].fImginary = pVector->GetAt(i).GetImaginaryNumber();
    }

    return lpResult;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::ExchangeCommand ( double *  pfCommand, MPI_Comm  comm  )

static

Gather eigenvalue finding iteration number from deflation group.

Exchanging command between MPI_Comm

Parameters

pfCommand	Command buffer
comm	MPI_Comm for exchaning command

Definition at line 878 of file KNMPIManager.cpp.

References BroadcastDouble(), and COMMAND_SIZE.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
    BroadcastDouble(pfCommand, COMMAND_SIZE, 0, comm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::FinalizeManager ( )

static

Get Root rank.

Release all memory

Definition at line 693 of file KNMPIManager.cpp.

References FREE_MEM, CKNTimeMeasurement::FREE_MEM, m_bStartMPI, m_deflationComm, m_deflationGroup, m_lanczosGroup, m_mpiCommIndex, m_nCurrentRank, m_nTotalNode, m_pBankInfo, m_pDispls, m_pLoadBalance, m_pRecvCount, m_pSendCount, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNLanczosTest::COmpareWIthMatLabSeOrthMPI(), CKNTBMS_Solver::FinalEvn(), CKNLanczosTest::LargeSizeMatrixMPI(), and CKNGeometricConstructionLaunch::LaunchingGeometricConstructionMPI().

{
    m_bStartMPI = false;
    m_nCurrentRank = 0;
    m_nTotalNode = 1;
    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::FREE_MEM);
    FREE_MEM(m_pLoadBalance);
    FREE_MEM(m_pRecvCount);
    FREE_MEM(m_pSendCount);
    FREE_MEM(m_pBankInfo);
    FREE_MEM(m_pDispls);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::FREE_MEM);
    if( MPI_GROUP_EMPTY != m_lanczosGroup )
        MPI_Group_free(&m_lanczosGroup);
    if( MPI_GROUP_EMPTY != m_deflationGroup )
        MPI_Group_free(&m_deflationGroup);
    if( MPI_COMM_NULL != m_mpiCommIndex && MPI_COMM_WORLD != m_mpiCommIndex)
        MPI_Comm_free(&m_mpiCommIndex);
    if( MPI_COMM_NULL != m_deflationComm )
        MPI_Comm_free(&m_deflationComm);
}

Here is the call graph for this function:

Here is the caller graph for this function:

static void CKNMPIManager::GatherEVFromDeflationGroup ( int  nSourceCount, double *  pReceiveBuffer, int *  pSourceCount, double *  pSendBuffer, int  nSendCount  )

inlinestatic

Definition at line 76 of file KNMPIManager.h.

References GatherVDouble(), and m_deflationComm.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{ GatherVDouble(nSourceCount, pReceiveBuffer, pSourceCount, pSendBuffer, nSendCount, m_deflationComm); }; 

Here is the call graph for this function:

Here is the caller graph for this function:

static void CKNMPIManager::GatherEVIterationFromDeflationGroup ( int  nSourceCount, int *  pReceiveBuffer, int *  pSourceCount, int *  pSendBuffer, int  nSendCount  )

inlinestatic

Gather eigenvalue from deflation group.

Definition at line 77 of file KNMPIManager.h.

References GatherVInt(), and m_deflationComm.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{ GatherVInt(nSourceCount, pReceiveBuffer, pSourceCount, pSendBuffer, nSendCount, m_deflationComm); }; 

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::GatherVDouble ( int  nSourceCount, double *  pReceiveBuffer, int *  pSourceCount, double *  pSendBuffer, int  nSendCount, MPI_Comm  comm = MPI_COMM_NULL  )

static

GatherV for double wrapping function.

Parameters

	nSourceCount	Gather double buffer source count
[out]	pReceiveBuffer	Saving buffer
	pSourceCount	Srouce counts (ref. MPI_Gatherv)
	pSendBuffer	Sending buffer
	nSendCount	Sending counts
	comm	MPI_Comm for gather data

Definition at line 822 of file KNMPIManager.cpp.

References FREE_MEM, IsDeflationRoot(), and m_mpiCommIndex.

Referenced by GatherEVFromDeflationGroup().

{
    int             *pReceiveCount = NULL;
    int             *pDisp = NULL;
    unsigned int    i;

    if( IsDeflationRoot() )
    {
        pDisp = (int*)malloc(sizeof(int)*nSourceCount);
        pDisp[0] = 0;
        for( i = 1; i < nSourceCount ; ++i)
            pDisp[i] = pDisp[i-1] + pSourceCount[i-1];
    }

    if( MPI_COMM_NULL == comm )
        MPI_Gatherv(pSendBuffer, nSendCount, MPI_DOUBLE, pReceiveBuffer, pSourceCount, pDisp, MPI_DOUBLE, 0, m_mpiCommIndex);
    else
        MPI_Gatherv(pSendBuffer, nSendCount, MPI_DOUBLE, pReceiveBuffer, pSourceCount, pDisp, MPI_DOUBLE, 0, comm);

    FREE_MEM(pDisp);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::GatherVInt ( int  nSourceCount, int *  pReceiveBuffer, int *  pSourceCount, int *  pSendBuffer, int  nSendCount, MPI_Comm  comm = MPI_COMM_NULL  )

static

GahterV for int wrapping function.

Parameters

	nSourceCount	Gather double buffer source count
[out]	pReceiveBuffer	Saving buffer
	pSourceCount	Srouce counts (ref. MPI_Gatherv)
	pSendBuffer	Sending buffer
	nSendCount	Sending counts
	comm	MPI_Comm for gather data

Definition at line 852 of file KNMPIManager.cpp.

References FREE_MEM, IsDeflationRoot(), and m_mpiCommIndex.

Referenced by GatherEVIterationFromDeflationGroup().

{
    int             *pReceiveCount = NULL;
    int             *pDisp = NULL;
    unsigned int    i;

    if( IsDeflationRoot() )
    {
        pDisp = (int*)malloc(sizeof(int)*nSourceCount);
        pDisp[0] = 0;
        for( i = 1; i < nSourceCount ; ++i)
            pDisp[i] = pDisp[i-1] + pSourceCount[i-1];
    }

    if( MPI_COMM_NULL == comm )
        MPI_Gatherv(pSendBuffer, nSendCount, MPI_INT, pReceiveBuffer, pSourceCount, pDisp, MPI_INT, 0, m_mpiCommIndex);
    else
        MPI_Gatherv(pSendBuffer, nSendCount, MPI_INT, pReceiveBuffer, pSourceCount, pDisp, MPI_INT, 0, comm);

    FREE_MEM(pDisp);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int CKNMPIManager::GetCurrentLoadBalanceCount ( )

static

Get Current node's rank load balancing number.

Definition at line 684 of file KNMPIManager.cpp.

References m_nCurrentRank, and m_pLoadBalance.

Referenced by CKNLanczosMethod::BuildWaveFunction(), CKNLanczosMethod::CalculateEigenVector(), CKNLanczosMethod::DoResidualCheck(), CKNLanczosMethod::InitLanczosVector(), CKNLanczosMethod::LanczosIteration(), CKNLanczosMethod::LanczosIterationLoop(), CKNLanczosTest::LargeSizeMatrixMPI(), CKNLanczosMethod::MergeDegeneratedEigenvalues(), MergeVector(), MergeVectorEx_Optimal(), MergeVectorOptimal(), CKNLanczosMethod::RecalcuWaveFunction(), CKNLanczosMethod::SortSolution(), SplitVector(), CKNLanczosTest::TestCSRBuildingViaFileLoad(), and CKNLanczosTest::TestCSRBuildingViaFileLoad_().

{
#ifdef DISABLE_MPI_ROUTINE
    return 0;
#endif

    return m_pLoadBalance[m_nCurrentRank];
}

Here is the caller graph for this function:

static int CKNMPIManager::GetCurrentRank ( )

inlinestatic

Definition at line 42 of file KNMPIManager.h.

References m_nCurrentRank.

Referenced by CKNMatrixOperation::AllocateLocalCSR(), CKNGeometricShape::ArrangeUnitCell(), CKNGeometricShape::CalculateUnitcellCount(), CKNGeometricShape::ConstructMapInfo(), CKNMatrixDebug::DumpCSR(), CKNIPCCUtility::DumpCSR(), CKNMatrixDebug::DumpCSRBinary(), CKNIPCCUtility::DumpCSRBinary(), CKNGeometricShape::ExchangeAtomInfoBetweenNode(), CKNGeometricShape::GetPeriodicDirection(), InitLevel(), CKNGeometricShape::IsInBoundaryCondition(), IsRootRank(), CKNLanczosMethod::LanczosIterationLoop(), CKNLanczosTest::LargeSizeMatrixMPI(), CKNTBMS_Solver::Launching_TBMS_Solver(), CKNLanczosLaunching::LaunchingLanczos(), CKNLanczosMethod::MergeDegeneratedEigenvalues(), CKNMatrixOperation::MVMulEx_AsyncCommWithLocalBlocks(), CKNGeometricShape::PeriodicUnitCellNumbering(), ReceiveDoubleBufferSync(), CKNLanczosMethod::SaveLanczosResult(), SendDoubleBufferSync(), CKNGeometricConstructionLaunch::SetShapeInformation(), CKNGeometricShape::SetShapeInformation(), SplitVector(), CKNLanczosTest::TestCSRBuildingViaFileLoad(), and CKNLanczosTest::TestCSRBuildingViaFileLoad_().

{ return m_nCurrentRank;}; 

Here is the caller graph for this function:

int CKNMPIManager::GetCurrentRank ( MPI_Comm  comm )

static

Get Current node's rank number.

Get Current node's rank number

Parameters

comm

MPI_Comm

Returns

Current rank in 'comm' MPI_Comm

Definition at line 251 of file KNMPIManager.cpp.

{
    int             rank;

    MPI_Comm_rank(comm, &rank);
    return rank;
}

static MPI_Comm CKNMPIManager::GetDeflationComm ( )

inlinestatic

Getting Lanczos computing group MPI_Comm.

Definition at line 80 of file KNMPIManager.h.

References m_deflationComm.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{ return m_deflationComm; }; 

Here is the caller graph for this function:

int * CKNMPIManager::GetEigenvalueCountFromDeflationGroup ( int  nDeflationGroupCount, int  nLocalEVCount  )

static

Checking is root rank of Deflation computation.

Collecting total eigenvalue count from All deflation group

Parameters

nDeflationGroupCount	Deflation group counts
nLocalEVCount	Local deflation group eigenvalue counts

Returns

Merged eigenvalue counts

Definition at line 802 of file KNMPIManager.cpp.

References IsRootRank(), and m_deflationComm.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
    int             *pEVCount = NULL;

    if( IsRootRank(m_deflationComm ) )
        pEVCount = (int*)malloc(sizeof(int)*nDeflationGroupCount);

    MPI_Gather(&nLocalEVCount, 1, MPI_INT, pEVCount, 1, MPI_INT, 0, m_deflationComm);

    return pEVCount;
}

Here is the call graph for this function:

Here is the caller graph for this function:

static MPI_Comm CKNMPIManager::GetLanczosComputComm ( )

inlinestatic

Definition at line 79 of file KNMPIManager.h.

References m_mpiCommIndex.

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{ return m_mpiCommIndex; }; 

Here is the caller graph for this function:

static unsigned int CKNMPIManager::GetLanczosGroupIndex ( )

inlinestatic

Definition at line 83 of file KNMPIManager.h.

Referenced by CKNLanczosMethod::LanczosIteration(), and CKNLanczosMethod::MergeDegeneratedEigenvalues().

{ return m_nLanczosGroupIndex; }; 

Here is the caller graph for this function:

int CKNMPIManager::GetLoadBalanceCount ( int  nRank )

static

Parameters

nRank

Target rank index

Definition at line 203 of file KNMPIManager.cpp.

References m_nTotalNode, and m_pLoadBalance.

Referenced by CKNMatrixOperation::AllocateLocalCSR(), InitCommunicationBufferMetric(), CKNLanczosTest::LargeSizeMatrixMPI(), MergeVectorEx_Optimal(), MergeVectorOptimal(), SplitVector(), CKNLanczosTest::TestCSRBuildingViaFileLoad(), and CKNLanczosTest::TestCSRBuildingViaFileLoad_().

{
#ifdef DISABLE_MPI_ROUTINE
    return 0;
#endif
    if (nRank > m_nTotalNode)
        return 0;

    return m_pLoadBalance[nRank];
}

Here is the caller graph for this function:

static MPI_Comm CKNMPIManager::GetMPIComm ( )

inlinestatic

Definition at line 67 of file KNMPIManager.h.

References m_mpiCommIndex.

Referenced by CKNMatrixOperation::AllocateLocalCSR(), CKNLanczosMethod::LanczosIterationLoop(), and CKNMatrixOperation::MVMulEx_AsyncCommWithLocalBlocks().

{ return m_mpiCommIndex; }; 

Here is the caller graph for this function:

static int CKNMPIManager::GetRootRank ( )

inlinestatic

Definition at line 60 of file KNMPIManager.h.

Referenced by BroadcastLanczosResult().

{ return 0;}; 

Here is the caller graph for this function:

static int CKNMPIManager::GetTotalNodeCount ( )

inlinestatic

Definition at line 44 of file KNMPIManager.h.

References m_nTotalNode.

Referenced by CKNMatrixOperation::AllocateLocalCSR(), CKNGeometricShape::ArrangeUnitCell(), CKNGeometricShape::CalculateUnitcellCount(), CKNGeometricShape::ConstructMapInfo(), CKNMatrixDebug::DumpCSR(), CKNIPCCUtility::DumpCSR(), CKNMatrixDebug::DumpCSRBinary(), CKNIPCCUtility::DumpCSRBinary(), CKNGeometricShape::ExchangeAtomInfoBetweenNode(), CKNGeometricShape::GetPeriodicDirection(), InitCommunicationBufferMetric(), InitLevel(), CKNGeometricShape::IsInBoundaryCondition(), CKNLanczosMethod::LanczosIterationLoop(), CKNTBMS_Solver::Launching_TBMS_Solver(), CKNLanczosLaunching::LaunchingLanczos(), CKNLanczosMethod::MergeDegeneratedEigenvalues(), CKNMatrixOperation::MVMulEx_AsyncCommWithLocalBlocks(), CKNMatrixOperation::MVMulEx_Optimal(), CKNGeometricShape::PeriodicUnitCellNumbering(), CKNLanczosMethod::SaveLanczosResult(), CKNLanczosTest::SaveResultCVS(), CKNGeometricConstructionLaunch::SetShapeInformation(), CKNGeometricShape::SetShapeInformation(), CKNLanczosMethod::ShowLanczosResult(), and SplitVector().

{ return m_nTotalNode; }; 

Here is the caller graph for this function:

void CKNMPIManager::InitCommunicationBufferMetric ( )

static

Initializing MPI Communication buffer for MVMul.

Parameters

nMatrixSize

Matrix size that want to solving

Definition at line 718 of file KNMPIManager.cpp.

References GetLoadBalanceCount(), GetTotalNodeCount(), m_pDispls, m_pRecvCount, m_pSendCount, CKNTimeMeasurement::MALLOC, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNTBMS_Solver::AllocateCSR(), CKNLanczosTest::COmpareWIthMatLabSeOrthMPI(), CKNLanczosTest::LargeSizeMatrixMPI(), CKNGeometricConstructionLaunch::LaunchingGeometricConstructionMPI(), and CKNLanczosLaunching::LaunchingLanczos().

{
    unsigned int                i;

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);
    m_pRecvCount = (int*)malloc(sizeof(int)*GetTotalNodeCount());
    m_pSendCount = (int*)malloc(sizeof(int)*GetTotalNodeCount());
    m_pDispls = (int*)malloc(sizeof(int)*GetTotalNodeCount());
    m_pDispls[0] = 0;
    m_pRecvCount[0] = GetLoadBalanceCount(0);
    for (i = 1; i < (unsigned int)GetTotalNodeCount(); i++)
    {
        m_pRecvCount[i] = GetLoadBalanceCount(i);
        m_pDispls[i] = m_pDispls[i - 1] + GetLoadBalanceCount(i - 1);
    }
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool CKNMPIManager::InitLevel ( int  nMPILevel, int  nFindingDegeneratedEVCount  )

static

Init MPI Level, most low level is for multi node cacluation for Lanczos.

Parameters

nMPILevel	MPI level count
nFindingDegeneratedEVCount	Deflation group count

Returns

Spliting MPI Group success or not

< First make group for lanczos method

< Second make group for deflation lanczos - vertical connected group

Definition at line 51 of file KNMPIManager.cpp.

References CheckDeflationNodeCount(), FREE_MEM, GetCurrentRank(), GetTotalNodeCount(), IsDeflationRoot(), IsLanczosComputeRoot(), m_bMultiLevel, m_bNeedPostOperation, m_deflationComm, m_deflationGroup, m_lanczosGroup, m_mpiCommIndex, m_nCommWorldRank, m_nLanczosGroupIndex, SetMPIEnviroment(), and CKNIPCCUtility::SetShow().

Referenced by CKNTBMS_Solver::InitMPIEnv(), and CKNLanczosLaunching::LaunchingLanczos().

{
    bool                bRtn = true;
    int                 nNeedNodeCount = 1;
    int                 world_size, rank;
    int                 nPerGroupNode;
    int                 nLanczosGroupIndex;
    int                 *pNewGroupRank = NULL;
    unsigned int        i;
    MPI_Group           commWorldGroup;


    if( 1 == nMPILevel )
    {
        m_mpiCommIndex = MPI_COMM_WORLD;
        return bRtn;
    }

    if( nFindingDegeneratedEVCount > 1 )
    {
        m_bNeedPostOperation[1] = true;
        nNeedNodeCount *= nFindingDegeneratedEVCount;
        nMPILevel--;
    }

    if( nMPILevel == 1 )
    {
        bRtn = CheckDeflationNodeCount(nNeedNodeCount);
        if( !bRtn )
            return bRtn;
    }
    
    m_nCommWorldRank = GetCurrentRank();
    nPerGroupNode = GetTotalNodeCount() / nNeedNodeCount;
    nLanczosGroupIndex = GetCurrentRank() / nPerGroupNode;
    pNewGroupRank  = (int*)malloc(sizeof(int)*nPerGroupNode);

    for( i = 0; i < nPerGroupNode ; ++i)
        pNewGroupRank [i] = nLanczosGroupIndex * nPerGroupNode + i;

    MPI_Comm_group(MPI_COMM_WORLD,&commWorldGroup);
    MPI_Group_incl(commWorldGroup,nPerGroupNode,pNewGroupRank ,&m_lanczosGroup);
    MPI_Comm_create(MPI_COMM_WORLD,m_lanczosGroup,&m_mpiCommIndex);
    MPI_Comm_size(m_mpiCommIndex, &world_size);
    MPI_Comm_rank(m_mpiCommIndex, &rank);
    SetMPIEnviroment(rank, world_size);
    m_nLanczosGroupIndex = nLanczosGroupIndex;

    pNewGroupRank  = (int*)realloc(pNewGroupRank , sizeof(int)*nFindingDegeneratedEVCount);
    for( i = 0; i < nFindingDegeneratedEVCount ; ++i)
        pNewGroupRank [i] = i * nPerGroupNode + GetCurrentRank();

    MPI_Comm_group(MPI_COMM_WORLD,&commWorldGroup);
    MPI_Group_incl(commWorldGroup,nFindingDegeneratedEVCount,pNewGroupRank ,&m_deflationGroup);
    MPI_Comm_create(MPI_COMM_WORLD,m_deflationGroup,&m_deflationComm);
    MPI_Comm_rank(m_deflationComm, &rank);
    MPI_Comm_size(m_mpiCommIndex, &world_size);

    m_bMultiLevel = true;

    FREE_MEM(pNewGroupRank );

    if( false == CKNMPIManager::IsDeflationRoot() || false == CKNMPIManager::IsLanczosComputeRoot() )
        CKNIPCCUtility::SetShow(false);

    bRtn = true;
    return bRtn;
}

Here is the call graph for this function:

Here is the caller graph for this function:

static bool CKNMPIManager::IsDeflationRoot ( )

inlinestatic

Checking is root rank of Lanczos computation.

Definition at line 72 of file KNMPIManager.h.

References IsRootRank(), and m_deflationComm.

Referenced by CKNGeometricShape::ConstructMapInfo(), CKNIPCCUtility::DumpCSR(), GatherVDouble(), GatherVInt(), InitLevel(), CKNTBMS_Solver::Launching_TBMS_Solver(), CKNLanczosMethod::MergeDegeneratedEigenvalues(), CKNLanczosMethod::RecalcuWaveFunction(), CKNLanczosMethod::SaveLanczosResult(), and CKNLanczosMethod::ShowLanczosResult().

{ return IsRootRank(m_deflationComm); }; 

Here is the call graph for this function:

Here is the caller graph for this function:

static bool CKNMPIManager::IsInMPIRoutine ( )

inlinestatic

Definition at line 47 of file KNMPIManager.h.

References m_bStartMPI.

Referenced by BroadcastVector(), and CKNLanczosTest::TestCSRBuildingViaFileLoad().

{ return m_bStartMPI; }; 

Here is the caller graph for this function:

static bool CKNMPIManager::IsLanczosComputeRoot ( )

inlinestatic

Barrier current deflation group.

Definition at line 71 of file KNMPIManager.h.

References IsRootRank(), and m_mpiCommIndex.

Referenced by InitLevel(), and CKNLanczosMethod::MergeDegeneratedEigenvalues().

{ return IsRootRank(m_mpiCommIndex);}; 

Here is the call graph for this function:

Here is the caller graph for this function:

static bool CKNMPIManager::IsMultiLevelMPI ( )

inlinestatic

Get MPI_Comm.

Definition at line 68 of file KNMPIManager.h.

References m_bMultiLevel.

Referenced by CKNLanczosMethod::LanczosIteration(), CKNTBMS_Solver::Launching_TBMS_Solver(), and CKNLanczosLaunching::LaunchingLanczos().

{ return m_bMultiLevel; }; 

Here is the caller graph for this function:

bool CKNMPIManager::IsRootRank ( )

static

Get Total node count.

Check this node is root rank?

Returns

Root rank or not

Definition at line 217 of file KNMPIManager.cpp.

References GetCurrentRank().

Referenced by CKNTBMS_Solver::ApplyPhPotential(), BroadcastLanczosResult(), CKNGeometricShape::CalculateUnitcellCount(), CKNLanczosTest::COmpareWIthMatLabSeOrthMPI(), CKNGeometricShape::ConstructMapInfo(), CKNLanczosMethod::DoEigenValueSolving(), CKNLanczosMethod::DoResidualCheck(), CKNIPCCUtility::DumpCSR(), CKNMatrixDebug::DumpCSR(), CKNIPCCUtility::DumpCSRBinary(), CKNMatrixDebug::DumpCSRBinary(), CKNGeometricShape::ExchangeAtomInfoBetweenNode(), CKNLanczosMethod::FinalizeLanczosInterationVariable(), GetEigenvalueCountFromDeflationGroup(), CKNGeometricShape::GetPeriodicDirection(), CKNLanczosMethod::InitLanczosIterationVariables(), IsDeflationRoot(), CKNGeometricShape::IsInBoundaryCondition(), IsLanczosComputeRoot(), CKNLanczosMethod::LanczosIteration(), CKNLanczosMethod::LanczosIterationLoop(), CKNLanczosTest::LargeSizeMatrixMPI(), CKNTBMS_Solver::Launching_TBMS_Solver(), CKNGeometricConstructionLaunch::LaunchingGeometricConstructionMPI(), CKNLanczosLaunching::LaunchingLanczos(), CKNLanczosMethod::MergeDegeneratedEigenvalues(), CKNGeometricShape::PeriodicUnitCellNumbering(), CKNLanczosMethod::SaveLanczosResult(), CKNGeometricConstructionLaunch::SetShapeInformation(), CKNGeometricShape::SetShapeInformation(), CKNLanczosMethod::ShowLanczosResult(), and CKNLanczosMethod::SortSolution().

{
    bool            bRtn = true;

#ifdef DISABLE_MPI_ROUTINE
    return bRtn; 
#endif

    if (0 == GetCurrentRank())
        return bRtn;

    bRtn = false;
    return bRtn;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool CKNMPIManager::IsRootRank ( MPI_Comm  comm )

static

Check this node is root rank in 'comm' MPI_Comm.

Parameters

comm

MPI_Comm

Returns

Root rank or not in 'comm' MPI_Comm

Definition at line 236 of file KNMPIManager.cpp.

References GetCurrentRank().

{
    if (MPI_COMM_NULL == comm)
        return true;

    if( 0 == GetCurrentRank(comm) )
        return true;
    else
        return false;
}

Here is the call graph for this function:

void CKNMPIManager::LoadBlancing ( int  nElementCount )

static

Load blancing for MPI, This function for lanczos solving with geometric constrcution.

Parameters

nElementCount

Load balancing count

Remarks

This function for lanczos solving with geometric constrcution

Definition at line 189 of file KNMPIManager.cpp.

References m_mpiCommIndex, m_nTotalNode, and m_pLoadBalance.

Referenced by CKNTBMS_Solver::AllocateCSR().

{
    m_pLoadBalance = (int *)malloc(sizeof(int)*(m_nTotalNode)); // For communication size

#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    MPI_Allgather(&nElementCount, 1, MPI_INTEGER, m_pLoadBalance, 1, MPI_INTEGER, m_mpiCommIndex);
}

Here is the caller graph for this function:

void CKNMPIManager::LoadBlancingForLanczos ( int  nRowCount )

static

Load blancing for MPI, this function only for lanczos solving without geometric constrcution.

Parameters

nRowCount

Matrix row count

Remarks

This function only for lanczos solving without geometric construction, Written by Sungkun Shin

Definition at line 150 of file KNMPIManager.cpp.

References m_nTotalNode, m_pLoadBalance, and CKNIPCCUtility::ShowMsg().

Referenced by CKNLanczosTest::COmpareWIthMatLabSeOrthMPI(), CKNLanczosTest::LargeSizeMatrixMPI(), and CKNGeometricConstructionLaunch::LaunchingGeometricConstructionMPI().

{
    int                 i, j, temp;
    m_pLoadBalance = (int *)malloc(sizeof(int)*(m_nTotalNode)); // For communication size
    m_pLoadBalance[0] = nRowCount;

#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    if (nRowCount < 10 * m_nTotalNode){
        CKNIPCCUtility::ShowMsg("ERROR :: matrix dimension < 10 * number of CPU for Lanczos "); // Error out
    }

    if (0 == (nRowCount / 10) % m_nTotalNode){
        for (i = 0; i<m_nTotalNode; i++){
            m_pLoadBalance[i] = nRowCount / m_nTotalNode;
        }
    }
    else{
        temp = nRowCount;
        for (i = m_nTotalNode - 1; i>-1; i--){
            m_pLoadBalance[i] = ((int)(nRowCount / m_nTotalNode / 10 + 1)) * 10;
            temp -= m_pLoadBalance[i];
            if ((temp / 10) % i == 0){
                for (j = 0; j < i; j++){
                    m_pLoadBalance[j] = temp / i;
                }
                temp = 0;
                break;
            }
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::MergeVector ( CKNMatrixOperation::CKNVector *  pVector, CKNMatrixOperation::CKNVector *  pResultVector, unsigned int  nMergeSize  )

static

Merge vector to sub rank.

Parameters

	pVector	Vector for sharing
[out]	pResultVector	Vector for saving merging result
	nMergeSize	Vector size that after mergsing

Definition at line 276 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, FREE_MEM, GetCurrentLoadBalanceCount(), m_mpiCommIndex, m_pDispls, m_pRecvCount, CKNMatrixOperation::CKNVector::m_vectValueImaginaryBuffer, CKNMatrixOperation::CKNVector::m_vectValueRealBuffer, CKNTimeMeasurement::MeasurementEnd(), and CKNTimeMeasurement::MeasurementStart().

Referenced by CKNMatrixOperation::MVMul().

{
    LPCOMPLEX_NUMBER    lpSendBuffer = NULL;
    unsigned int        i;

#ifdef DISABLE_MPI_ROUTINE
    return;
#endif


    double                  *pBuffer = (double*)malloc(sizeof(double)*nMergeSize);
    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    MPI_Allgatherv(pVector->m_vectValueRealBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,
        pBuffer, m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);
    
    for (i = 0; i < nMergeSize; i++)
        pResultVector->m_vectValueRealBuffer[i] = pBuffer[i];

    MPI_Allgatherv(pVector->m_vectValueImaginaryBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,
        pBuffer, m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);
    
    for (i = 0; i < nMergeSize; i++)
        pResultVector->m_vectValueImaginaryBuffer[i] = pBuffer[i];

    FREE_MEM(pBuffer);

    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::MergeVectorEx_Optimal ( CKNMatrixOperation::CKNVector *  pSrcVector, CKNMatrixOperation::CKNVector *  pResultVector, unsigned int  nMergeSize, double  fFirstIndex, unsigned int  nSizeFromPrevRank, unsigned int  nSizeFromNextRank, unsigned int  nSizetoPrevRank, unsigned int  nSizetoNextRank, unsigned int *  mPos  )

static

Merge vector for 1 layer exchanging.

Parameters

	pSrcVector	Vector for sharing
[out]	pResultVector	Vector for saving merging result
	nMergeSize	Vector size that after mergsing
	fFirstIndex	First index for local vector index
	nSizeFromPrevRank	Exchanging size from previous node
	nSizeFromNextRank	Exchanging size from next node
	nSizetoPrevRank	Exchanging size to previous node
	nSizetoNextRank	Exchanging size to next node
	mPos	Previous, local, next node start index

Definition at line 317 of file KNMPIManager.cpp.

References GetCurrentLoadBalanceCount(), GetLoadBalanceCount(), m_mpiCommIndex, m_nCurrentRank, m_nTotalNode, m_pDispls, m_pRecvCount, CKNMatrixOperation::CKNVector::m_vectValueImaginaryBuffer, CKNMatrixOperation::CKNVector::m_vectValueRealBuffer, CKNTimeMeasurement::MeasurementEnd(), CKNTimeMeasurement::MeasurementStart(), and CKNTimeMeasurement::MV_COMM.

Referenced by CKNMatrixOperation::MVMulEx_Optimal().

{
    if (m_nTotalNode <= 3)
    {
        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);
        MPI_Allgatherv(pSrcVector->m_vectValueRealBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE, pResultVector->m_vectValueRealBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);
        MPI_Allgatherv(pSrcVector->m_vectValueImaginaryBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE, pResultVector->m_vectValueImaginaryBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);
        mPos[0] = -1; mPos[1] = -1; mPos[2] = -1;
    }
    else
    {
        long long       fCurrentRankPos = fFirstIndex;
        int         nPrevRank = (m_nCurrentRank - 1 + m_nTotalNode) % m_nTotalNode;
        int         nNextRank = (m_nCurrentRank + 1) % m_nTotalNode;
        long long       fPrevRankPos = -1, fNextRankPos = -1;
        double          *pSendBuffer = NULL, *pRecvBuffer = NULL;
        MPI_Request     req[2];
        MPI_Status      status[2];

        if (0 == m_nCurrentRank)
            fPrevRankPos = nMergeSize - nSizeFromPrevRank;
        else
            fPrevRankPos = fFirstIndex - nSizeFromPrevRank;

        if (m_nCurrentRank == m_nTotalNode - 1)
            fNextRankPos = 0;
        else
            fNextRankPos = fFirstIndex + GetLoadBalanceCount(m_nCurrentRank);

        mPos[0] = (unsigned int)fPrevRankPos; mPos[1] = (unsigned int)fCurrentRankPos; mPos[2] = (unsigned int)fNextRankPos;

        //printf("Rank %d: myload=%d, nSizeFromPrevRank=%d, nSizeFromNextRank=%d, nSizetoPrevRank=%d, nSizetoNextRank=%d, fPrevRankPos=%d, fNextRankPos=%d\n", m_nCurrentRank, GetLoadBalanceCount(m_nCurrentRank), nSizeFromPrevRank, nSizeFromNextRank, nSizetoPrevRank, nSizetoNextRank, fPrevRankPos, fNextRankPos); 

        pSendBuffer = pSrcVector->m_vectValueRealBuffer.data();
        pRecvBuffer = pResultVector->m_vectValueRealBuffer.data();

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);
        MPI_Irecv(pRecvBuffer + fPrevRankPos, nSizeFromPrevRank, MPI_DOUBLE, nPrevRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);
        MPI_Isend(pSendBuffer + GetLoadBalanceCount(m_nCurrentRank) - nSizetoNextRank, nSizetoNextRank, MPI_DOUBLE, nNextRank, nNextRank, m_mpiCommIndex, &req[1]);
        MPI_Waitall(2, req, status);

        MPI_Irecv(pRecvBuffer + fNextRankPos, nSizeFromNextRank, MPI_DOUBLE, nNextRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);
        MPI_Isend(pSendBuffer, nSizetoPrevRank, MPI_DOUBLE, nPrevRank, nPrevRank, m_mpiCommIndex, &req[1]);
        MPI_Waitall(2, req, status);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);

        pSendBuffer = pSrcVector->m_vectValueImaginaryBuffer.data();
        pRecvBuffer = pResultVector->m_vectValueImaginaryBuffer.data();

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);
        MPI_Irecv(pRecvBuffer + fPrevRankPos, nSizeFromPrevRank, MPI_DOUBLE, nPrevRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);
        MPI_Isend(pSendBuffer + GetLoadBalanceCount(m_nCurrentRank) - nSizetoNextRank, nSizetoNextRank, MPI_DOUBLE, nNextRank, nNextRank, m_mpiCommIndex, &req[1]);
        MPI_Waitall(2, req, status);

        MPI_Irecv(pRecvBuffer + fNextRankPos, nSizeFromNextRank, MPI_DOUBLE, nNextRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);
        MPI_Isend(pSendBuffer, nSizetoPrevRank, MPI_DOUBLE, nPrevRank, nPrevRank, m_mpiCommIndex, &req[1]);
        MPI_Waitall(2, req, status);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);

        memcpy(pResultVector->m_vectValueRealBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueRealBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));
        memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueImaginaryBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));

    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::MergeVectorOptimal ( CKNMatrixOperation::CKNVector *  pSrcVector, CKNMatrixOperation::CKNVector *  pResultVector, unsigned int  nMergeSize, double  fFirstIndex  )

static

Merge vector to sub rank, operated without vector class member function call.

Parameters

	pSrcVector	Vector for sharing
[out]	pResultVector	Vector for saving merging result
	nMergeSize	Vector size that after mergsing
	fFirstIndex	First index for local vector index

Definition at line 389 of file KNMPIManager.cpp.

References FREE_MEM, GetCurrentLoadBalanceCount(), GetLoadBalanceCount(), m_mpiCommIndex, m_nCurrentRank, m_nTotalNode, m_pDispls, m_pRecvCount, CKNMatrixOperation::CKNVector::m_vectValueImaginaryBuffer, CKNMatrixOperation::CKNVector::m_vectValueRealBuffer, CKNTimeMeasurement::MeasurementEnd(), CKNTimeMeasurement::MeasurementStart(), CKNTimeMeasurement::MV_COMM, CKNTimeMeasurement::MV_FREE_MEM, and CKNTimeMeasurement::MV_MALLOC.

Referenced by CKNMatrixOperation::MVMulOptimal().

{
    if( m_nTotalNode <= 3 )
    {
    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);
    MPI_Allgatherv(pSrcVector->m_vectValueRealBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,
        pResultVector->m_vectValueRealBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);
    MPI_Allgatherv(pSrcVector->m_vectValueImaginaryBuffer.data(), GetCurrentLoadBalanceCount(), MPI_DOUBLE,
        pResultVector->m_vectValueImaginaryBuffer.data(), m_pRecvCount, m_pDispls, MPI_DOUBLE, m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);
    }
    else
    {
        double          fCurrentRankPos = fFirstIndex;
        long long       fPrevRankPos = -1, fNextRankPos = -1;
        int             nPrevRank = (m_nCurrentRank-1+m_nTotalNode)%m_nTotalNode;
        int             nNextRank = (m_nCurrentRank+1)%m_nTotalNode;
        long long       nMax;
        double          *pSendBuffer = NULL, *pRecvBuffer = NULL;
        MPI_Request     req[2];
        MPI_Status      status[2];

        if( 0 == m_nCurrentRank )
            fPrevRankPos = nMergeSize - GetLoadBalanceCount(nPrevRank);
        else
            fPrevRankPos = fFirstIndex - GetLoadBalanceCount(nPrevRank);

        if (m_nCurrentRank == m_nTotalNode - 1)
            fNextRankPos = 0;
        else
            fNextRankPos = fCurrentRankPos + GetCurrentLoadBalanceCount();

#ifdef _WIN32
            nMax = max(GetLoadBalanceCount(nPrevRank), GetLoadBalanceCount(nNextRank));
#else //_WIN32
            nMax = std::max(GetLoadBalanceCount(nPrevRank), GetLoadBalanceCount(nNextRank));
#endif//

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_MALLOC);
        pRecvBuffer = (double*)malloc(sizeof(double)*nMax*2);
        pSendBuffer = (double*)malloc(sizeof(double)*GetLoadBalanceCount(m_nCurrentRank)*2);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_MALLOC);


        memcpy(pSendBuffer, pSrcVector->m_vectValueRealBuffer.data(), sizeof(double)*GetLoadBalanceCount(m_nCurrentRank));
        memcpy(pSendBuffer+ GetLoadBalanceCount(m_nCurrentRank), pSrcVector->m_vectValueImaginaryBuffer.data() , sizeof(double)*GetLoadBalanceCount(m_nCurrentRank));

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);
        MPI_Irecv(pRecvBuffer, 2 * GetLoadBalanceCount(nPrevRank), MPI_DOUBLE, nPrevRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);
        MPI_Isend(pSendBuffer, 2 * GetLoadBalanceCount(m_nCurrentRank), MPI_DOUBLE, nNextRank, nNextRank, m_mpiCommIndex, &req[1]);
        MPI_Waitall(2, req, status);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);
        
        memcpy(pResultVector->m_vectValueRealBuffer.data() + fPrevRankPos, pRecvBuffer, GetLoadBalanceCount(nPrevRank) * sizeof(double));
        memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + fPrevRankPos, pRecvBuffer + GetLoadBalanceCount(nPrevRank), GetLoadBalanceCount(nPrevRank) * sizeof(double));

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_COMM);
        MPI_Irecv(pRecvBuffer, 2 * GetLoadBalanceCount(nNextRank), MPI_DOUBLE, nNextRank, m_nCurrentRank, m_mpiCommIndex, &req[0]);
        MPI_Isend(pSendBuffer, 2 * GetLoadBalanceCount(m_nCurrentRank), MPI_DOUBLE, nPrevRank, nPrevRank, m_mpiCommIndex, &req[1]);
        MPI_Waitall(2, req, status);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_COMM);
        memcpy(pResultVector->m_vectValueRealBuffer.data() + fNextRankPos, pRecvBuffer, GetLoadBalanceCount(nNextRank) * sizeof(double));
        memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + fNextRankPos, pRecvBuffer + GetLoadBalanceCount(nNextRank), GetLoadBalanceCount(nNextRank) * sizeof(double));

        CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MV_FREE_MEM);
        FREE_MEM(pRecvBuffer);
        FREE_MEM(pSendBuffer);
        CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MV_FREE_MEM);
        
        memcpy(pResultVector->m_vectValueRealBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueRealBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));
        memcpy(pResultVector->m_vectValueImaginaryBuffer.data() + (long long)fFirstIndex, pSrcVector->m_vectValueImaginaryBuffer.data(), GetLoadBalanceCount(m_nCurrentRank) * sizeof(double));
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::ReceiveDoubleBufferSync ( int  nSourceRank, double *  pBuffer, int  nSize, MPI_Request *  req, MPI_Comm  commWorld = MPI_COMM_NULL  )

static

Receivinging buffer for double data array with sync.

Parameters

nSourceRank	Source rank index
pBuffer	Data buffer that want to receive
nSize	Data buffer size
req	MPI request parameter

Definition at line 770 of file KNMPIManager.cpp.

References GetCurrentRank(), and m_mpiCommIndex.

Referenced by CKNGeometricShape::ConstructMapInfo(), CKNIPCCUtility::DumpCSR(), CKNMatrixDebug::DumpCSR(), CKNMatrixDebug::DumpCSRBinary(), CKNIPCCUtility::DumpCSRBinary(), CKNGeometricShape::ExchangeAtomInfoBetweenNode(), and ReceiveVectorSync().

{
    MPI_Status status;
    if( MPI_COMM_NULL == commWorld)
        //MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, 0, m_mpiCommIndex, &status);
        MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, CKNMPIManager::GetCurrentRank(), m_mpiCommIndex, &status);
    else
        //MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, 0, commWorld, &status);
        MPI_Recv(pBuffer, nSize, MPI_DOUBLE, nSourceRank, CKNMPIManager::GetCurrentRank(commWorld), commWorld, &status);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::ReceiveVectorSync ( int  nSourceRank, CKNMatrixOperation::CKNVector *  pVector, int  nSize, MPI_Request *  req, MPI_Comm  commWorld = MPI_COMM_NULL  )

static

Receiving Vector with sync.

Parameters

nSourceRank	Source rank for receiving data
pVector	Receiving buffer
nSize	Receiving size
req	MPI_Request for MPI_Recv
commWorld	MPI_Comm for Receiving data

Definition at line 910 of file KNMPIManager.cpp.

References FREE_MEM, ReceiveDoubleBufferSync(), and CKNMatrixOperation::CKNVector::Serialize().

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
    double              *pBuffer = NULL;

    pBuffer = (double*)malloc(sizeof(double)*nSize*2);

    ReceiveDoubleBufferSync(nSourceRank, pBuffer, nSize * 2, req, commWorld);

    pVector->Serialize(pBuffer, true);

    FREE_MEM(pBuffer);
}

void CKNMPIManager::SendDoubleBufferSync ( int  nTargetRank, double *  pBuffer, int  nSize, MPI_Request *  req, MPI_Comm  commWorld = MPI_COMM_NULL  )

static

Sending buffer for double data array with sync.

Parameters

nTargetRank	Target rank index
pBuffer	Data buffer that want to send
nSize	Data buffer size
req	MPI request parameter

Definition at line 742 of file KNMPIManager.cpp.

References GetCurrentRank(), and m_mpiCommIndex.

Referenced by CKNGeometricShape::ConstructMapInfo(), CKNIPCCUtility::DumpCSR(), CKNMatrixDebug::DumpCSR(), CKNMatrixDebug::DumpCSRBinary(), CKNIPCCUtility::DumpCSRBinary(), CKNGeometricShape::ExchangeAtomInfoBetweenNode(), and SendVectorSync().

{
    int nRank = CKNMPIManager::GetCurrentRank();
    if( MPI_COMM_NULL == commWorld)
        //MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, 0, m_mpiCommIndex);
        MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, nTargetRank, m_mpiCommIndex);
    else
        if( MPI_SUCCESS != MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, nTargetRank, commWorld) )
        //if( MPI_SUCCESS != MPI_Send(pBuffer, nSize, MPI_DOUBLE, nTargetRank, 0, commWorld) )
            printf("Oh my god!\n");
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::SendVectorSync ( int  nTargetRank, CKNMatrixOperation::CKNVector *  pVector, int  nSize, MPI_Request *  req, MPI_Comm  commWorld = MPI_COMM_NULL  )

static

Getting Deflation computing group MPI_Comm.

Sending Vector with sync

Parameters

nTargetRank	Sending target rank
pVector	Sending buffer
nSize	Sending size
req	MPI_Request for MPI_Send
commWorld	MPI_Comm for sending data

Definition at line 890 of file KNMPIManager.cpp.

References FREE_MEM, SendDoubleBufferSync(), and CKNMatrixOperation::CKNVector::Serialize().

Referenced by CKNLanczosMethod::MergeDegeneratedEigenvalues().

{
    double              *pBuffer = NULL;

    pBuffer = (double*)malloc(sizeof(double)*nSize*2);
    pVector->Serialize(pBuffer, false);

    SendDoubleBufferSync(nTargetRank, pBuffer, nSize * 2, req, commWorld);


    FREE_MEM(pBuffer);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void CKNMPIManager::SetMPIEnviroment ( int  nRank, int  nTotalNode  )

static

Set MPI Enviroment.

Parameters

nRank	Current rank index
nTotalNode	Total rank count

Definition at line 139 of file KNMPIManager.cpp.

References m_bStartMPI, m_nCurrentRank, and m_nTotalNode.

Referenced by CKNLanczosTest::COmpareWIthMatLabSeOrthMPI(), InitLevel(), CKNTBMS_Solver::InitMPIEnv(), CKNLanczosTest::LargeSizeMatrixMPI(), CKNGeometricConstructionLaunch::LaunchingGeometricConstructionMPI(), and CKNLanczosLaunching::LaunchingLanczos().

{
    m_nCurrentRank = nRank;
    m_nTotalNode = nTotalNode;
    m_bStartMPI = true;
}

Here is the caller graph for this function:

void CKNMPIManager::SplitVector ( CKNMatrixOperation::CKNVector *  pVector, int  nRootRank  )

static

Split vector to sub rank.

Parameters

pVector	Vector for want to share
nRootRank	Root rank index

Definition at line 467 of file KNMPIManager.cpp.

References CKNTimeMeasurement::COMM, ConvertVectorToMPIComplexBuffer(), ERROR_MALLOC, CKNMPIManager::COMPLEX_NUMBER::fImginary, CKNMatrixOperation::CKNVector::Finalize(), CKNMPIManager::COMPLEX_NUMBER::fReal, FREE_MEM, CKNTimeMeasurement::FREE_MEM, GetCurrentLoadBalanceCount(), GetCurrentRank(), GetLoadBalanceCount(), GetTotalNodeCount(), m_mpiCommIndex, CKNTimeMeasurement::MALLOC, CKNTimeMeasurement::MeasurementEnd(), CKNTimeMeasurement::MeasurementStart(), CKNMatrixOperation::CKNVector::SetAt(), and CKNMatrixOperation::CKNVector::SetSize().

{
    LPCOMPLEX_NUMBER    lpSendBuffer = NULL;
    LPCOMPLEX_NUMBER    lpRecvBuffer = NULL;
    unsigned int        i;
    int                 *pSendCounts = NULL, *pDispls = NULL;

#ifdef DISABLE_MPI_ROUTINE
    return;
#endif

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);
    pSendCounts = (int*)malloc(sizeof(int)*GetTotalNodeCount());
    pDispls = (int*)malloc(sizeof(int)*GetTotalNodeCount());
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);

    pDispls[0] = 0;
    pSendCounts[0] = 2 * GetLoadBalanceCount(0);
    for (i = 1; i < (unsigned int)GetTotalNodeCount(); i++)
    {
        pSendCounts[i] = 2 * GetLoadBalanceCount(i);
        pDispls[i] = pDispls[i - 1] + 2 * GetLoadBalanceCount(i - 1);
    }

    if (nRootRank == GetCurrentRank())
    {
        lpSendBuffer = ConvertVectorToMPIComplexBuffer(pVector);
        pVector->Finalize();
    }

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::MALLOC);
    lpRecvBuffer = (LPCOMPLEX_NUMBER)malloc(sizeof(COMPLEX_NUMBER)*GetCurrentLoadBalanceCount() * 2);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::MALLOC);

    if (NULL == lpRecvBuffer)
        throw ERROR_MALLOC;

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::COMM);
    MPI_Scatterv(lpSendBuffer, pSendCounts, pDispls, MPI_DOUBLE,
        lpRecvBuffer, 2 * GetCurrentLoadBalanceCount(), MPI_DOUBLE, nRootRank, m_mpiCommIndex);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::COMM);

    pVector->SetSize(GetCurrentLoadBalanceCount());
    for (i = 0; i < (unsigned int)GetCurrentLoadBalanceCount(); i++)
        pVector->SetAt(i, lpRecvBuffer[i].fReal, lpRecvBuffer[i].fImginary);

    CKNTimeMeasurement::MeasurementStart(CKNTimeMeasurement::FREE_MEM);
    FREE_MEM(pSendCounts);
    FREE_MEM(pDispls);
    FREE_MEM(lpRecvBuffer);
    CKNTimeMeasurement::MeasurementEnd(CKNTimeMeasurement::FREE_MEM);
}

Here is the call graph for this function:

void CKNMPIManager::WaitReceiveDoubleBufferAsync ( MPI_Request *  req )

static

Waiting recevinging double buffer sync function.

Parameters

req	MPI request parameter

Definition at line 784 of file KNMPIManager.cpp.

References m_ReceiveDoubleAsyncRequest.

{
    MPI_Status      status;

    MPI_Wait(&m_ReceiveDoubleAsyncRequest, &status);
}

void CKNMPIManager::WaitSendDoubleBufferSync ( MPI_Request *  req )

static

Waiting sending double buffer sync function.

Parameters

req	MPI request parameter

Definition at line 757 of file KNMPIManager.cpp.

References m_SendDoubleAsyncRequest.

{
    MPI_Status      status;

    MPI_Wait(&m_SendDoubleAsyncRequest, &status);
}

Member Data Documentation

bool CKNMPIManager::m_bMultiLevel = false

staticprivate

Flag for Multilevel MPI group.

Definition at line 106 of file KNMPIManager.h.

Referenced by InitLevel(), and IsMultiLevelMPI().

bool CKNMPIManager::m_bNeedPostOperation = { false, false, false, false, false, false, false, false, false, false }

staticprivate

MPI Level.

Definition at line 100 of file KNMPIManager.h.

Referenced by InitLevel().

bool CKNMPIManager::m_bStartMPI = false

staticprivate

MPI_Init call or not.

Definition at line 89 of file KNMPIManager.h.

Referenced by FinalizeManager(), IsInMPIRoutine(), and SetMPIEnviroment().

MPI_Comm CKNMPIManager::m_deflationComm = MPI_COMM_NULL

staticprivate

Deflation computing MPI_Comm.

Definition at line 102 of file KNMPIManager.h.

Referenced by FinalizeManager(), GatherEVFromDeflationGroup(), GatherEVIterationFromDeflationGroup(), GetDeflationComm(), GetEigenvalueCountFromDeflationGroup(), InitLevel(), and IsDeflationRoot().

MPI_Group CKNMPIManager::m_deflationGroup = MPI_GROUP_EMPTY

staticprivate

MPI Group for Deflation computation.

Definition at line 104 of file KNMPIManager.h.

Referenced by FinalizeManager(), and InitLevel().

MPI_Group CKNMPIManager::m_lanczosGroup = MPI_GROUP_EMPTY

staticprivate

MPI Group for Lanczos computation.

Definition at line 103 of file KNMPIManager.h.

Referenced by FinalizeManager(), and InitLevel().

MPI_Comm CKNMPIManager::m_mpiCommIndex = MPI_COMM_WORLD

staticprivate

Lanczos Method MPI_Comm.

Definition at line 101 of file KNMPIManager.h.

Referenced by AllReduceComlex(), AllReduceDouble(), Barrier(), BroadcastBool(), BroadcastDouble(), BroadcastInt(), BroadcastLanczosResult(), FinalizeManager(), GatherVDouble(), GatherVInt(), GetLanczosComputComm(), GetMPIComm(), InitLevel(), IsLanczosComputeRoot(), LoadBlancing(), MergeVector(), MergeVectorEx_Optimal(), MergeVectorOptimal(), ReceiveDoubleBufferSync(), SendDoubleBufferSync(), and SplitVector().

int CKNMPIManager::m_nCommWorldRank = 0

staticprivate

MPI Rank before split.

Definition at line 87 of file KNMPIManager.h.

Referenced by InitLevel().

int CKNMPIManager::m_nCurrentRank = 0

staticprivate

Getting Lanczos group index.

MPI Rank.

MPI Rank

Definition at line 83 of file KNMPIManager.h.

Referenced by FinalizeManager(), GetCurrentLoadBalanceCount(), GetCurrentRank(), MergeVectorEx_Optimal(), MergeVectorOptimal(), and SetMPIEnviroment().

unsigned int CKNMPIManager::m_nLanczosGroupIndex = 0

staticprivate

MPI Group index for Lanczos group.

Definition at line 105 of file KNMPIManager.h.

Referenced by InitLevel().

unsigned int CKNMPIManager::m_nMPILevel = 1

staticprivate

MPI Level.

Definition at line 99 of file KNMPIManager.h.

int CKNMPIManager::m_nTotalNode = 1

staticprivate

Total node count.

Definition at line 88 of file KNMPIManager.h.

Referenced by CheckDeflationNodeCount(), FinalizeManager(), GetLoadBalanceCount(), GetTotalNodeCount(), LoadBlancing(), LoadBlancingForLanczos(), MergeVectorEx_Optimal(), MergeVectorOptimal(), and SetMPIEnviroment().

int * CKNMPIManager::m_pBankInfo = NULL

staticprivate

After MPI Split bank infomation.

Definition at line 95 of file KNMPIManager.h.

Referenced by FinalizeManager().

CKNMPIManager::LPCOMPLEX_NUMBER CKNMPIManager::m_pCommBuffer = NULL

staticprivate

Data buffer for MPI Communication.

Load blancing for MPI Communication.

Data buffer for MPI Communication

Definition at line 91 of file KNMPIManager.h.

CKNMPIManager::LPCOMPLEX_NUMBER CKNMPIManager::m_pConvertingBuffer = NULL

staticprivate

Data buffer for Vector converting.

Definition at line 92 of file KNMPIManager.h.

Referenced by ConvertVectorToMPIComplexBuffer().

int * CKNMPIManager::m_pDispls = NULL

staticprivate

Displ for MPI comminication.

Definition at line 96 of file KNMPIManager.h.

Referenced by FinalizeManager(), InitCommunicationBufferMetric(), MergeVector(), MergeVectorEx_Optimal(), and MergeVectorOptimal().

int * CKNMPIManager::m_pLoadBalance = NULL

staticprivate

Load blancing for MPI Communication.

Definition at line 90 of file KNMPIManager.h.

Referenced by FinalizeManager(), GetCurrentLoadBalanceCount(), GetLoadBalanceCount(), LoadBlancing(), and LoadBlancingForLanczos().

int * CKNMPIManager::m_pRecvCount = NULL

staticprivate

Reciving count variable for MPI comminication.

Definition at line 93 of file KNMPIManager.h.

Referenced by FinalizeManager(), InitCommunicationBufferMetric(), MergeVector(), MergeVectorEx_Optimal(), and MergeVectorOptimal().

int * CKNMPIManager::m_pSendCount = NULL

staticprivate

Sending count variable for MPI comminication.

Definition at line 94 of file KNMPIManager.h.

Referenced by FinalizeManager(), and InitCommunicationBufferMetric().

MPI_Request CKNMPIManager::m_ReceiveDoubleAsyncRequest = MPI_REQUEST_NULL

staticprivate

Request for receving double.

Definition at line 98 of file KNMPIManager.h.

Referenced by WaitReceiveDoubleBufferAsync().

MPI_Request CKNMPIManager::m_SendDoubleAsyncRequest = MPI_REQUEST_NULL

staticprivate

Request for sending double.

Definition at line 97 of file KNMPIManager.h.

Referenced by WaitSendDoubleBufferSync().

---

The documentation for this class was generated from the following files:

• D:/For Doxygen/KNMPIManager.h
• D:/For Doxygen/KNMPIManager.cpp