KNLanczosLaunching.cpp

Go to the documentation of this file.

#include "stdafx.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "CKNGlobal.h"
#include "KNLanczosLaunching.h"
#include "KNLanczosMethod.h"
#include "KNIPCCUtility.h"
#include "KNMPIManager.h"
#include "KNTimeMeasurement.h"
#include "KNLanczosResultAudit.h"
#include <algorithm>
#include <mpi.h>
#include "KNCommandFileParser.h"
#include <omp.h>
#include "XeonPhi_header.h"

extern int phi_tid;

CKNLanczosLaunching::CKNLanczosLaunching()
{
}

CKNLanczosLaunching::~CKNLanczosLaunching()
{
}

CKNLanczosMethod::LPEIGENVALUE_RESULT CKNLanczosLaunching::LaunchingLanczos(CKNMatrixOperation::CKNCSR *pA, bool bShowMsg, LPLANCZOS_PARAM lpParam, bool bMPI)
{
    CKNMatrixOperation::CKNCSR                  *pHamiltonian = pA;
    CKNMatrixOperation::CKNCSR                  *plocalH = NULL;
    CKNMatrixOperation::CKNCSR                  *plocalHm1 = NULL;
    CKNMatrixOperation::CKNCSR                  *plocalHp1 = NULL;
    unsigned int                                nRtnCode = CALCULATION_SUCCESS;
    unsigned int                                i, nRepeatCount = 1;
    CKNLanczosMethod::LPEIGENVALUE_RESULT       lpResult = NULL;
    CKNLanczosMethod                            lanczos;
    char                                        szMsg[1024];
    double                                      *pKValue[3] = {NULL, NULL, NULL};

    if (bShowMsg)
        CKNIPCCUtility::SetShow(true);

    int                         rank;
    int                         world_size;

    if (bMPI)
    {
        MPI_Comm_size(MPI_COMM_WORLD, &world_size);
        MPI_Comm_rank(MPI_COMM_WORLD, &rank);

        CKNMPIManager::SetMPIEnviroment(rank, world_size);
        if(!CKNMPIManager::InitLevel(lpParam->nMPILevel, lpParam->nFindingDegeneratedEVCount))
        {
            SHOW_SIMPLE_MSG("Adjusting MPI level and node count first!\n");
            return lpResult;
        }
        CKNMPIManager::InitCommunicationBufferMetric();
    }

        
    if( CKNMPIManager::IsRootRank() )
    {
#ifdef _WIN32 
            system("rmdir /s /q result");
#else //WIN32
            system("rm -rf result");
#endif //WIN32

#pragma omp parallel
{
        if(omp_get_num_threads() > 1)
        {
            if(omp_get_thread_num() == 0)
            {
                sprintf(szMsg, "-Run with MPI (%d cores) and OpenMP (%d threads)\n", CKNMPIManager::GetTotalNodeCount(), omp_get_num_threads());
                CKNIPCCUtility::ShowMsg(szMsg);
            }
        }
        else
        {
            sprintf(szMsg, "-Run with MPI only (%d cores)\n", CKNMPIManager::GetTotalNodeCount());
            CKNIPCCUtility::ShowMsg(szMsg);
        }
}
    }
            
    if(CKNMPIManager::GetTotalNodeCount() > 2)
        CKNMatrixOperation::AllocateLocalCSR(&plocalH, &plocalHm1, &plocalHp1);
    

#ifndef DISABLE_MPI_ROUTINE
    phi_tid = CKNMPIManager::GetCurrentRank() / 2;
// FIXME test code
#ifndef _WIN32
    char host_name[256];
    gethostname(host_name, 256);
    //printf("[%s] MPI rank: %d | phi tid: %d\n", host_name, CKNMPIManager::GetCurrentRank(), phi_tid);
#endif //_WIN32
#else
    phi_tid = 0;
#endif

    double *local_real      = NULL;
    double *local_imaginary = NULL;
    unsigned int *local_row = NULL;
    unsigned int *local_col = NULL;
    unsigned int local_size     = 0;
    unsigned int local_row_size = 0;
    unsigned int local_col_size = 0;

    double *left_real       = NULL;
    double *left_imaginary  = NULL;
    unsigned int *left_row  = NULL;
    unsigned int *left_col  = NULL;
    unsigned int left_size     = 0;
    unsigned int left_row_size = 0;
    unsigned int left_col_size = 0;

    double *right_real      = NULL;
    double *right_imaginary = NULL;
    unsigned int *right_row = NULL;
    unsigned int *right_col = NULL;
    unsigned int right_size     = 0;
    unsigned int right_row_size = 0;
    unsigned int right_col_size = 0;
    if(CKNMPIManager::GetTotalNodeCount() > 2)
    {
        CKNMatrixOperation::BuildLocalCSR(pHamiltonian, plocalH, plocalHm1, plocalHp1);
        SHOW_SIMPLE_MSG("-Local matrices built\n");

        local_real      = plocalH->m_vectValueRealBuffer.data();
        local_imaginary = plocalH->m_vectValueImaginaryBuffer.data();
        local_row       = plocalH->m_vectRow.data();
        local_col       = plocalH->m_vectColumn.data();
        local_size     = plocalH->GetNoneZeroCount();
        local_row_size = plocalH->m_vectRow.size();
        local_col_size = plocalH->m_vectColumn.size();
#pragma offload_transfer target(mic:phi_tid) \
    nocopy(local_real[0:local_size]      : ALLOC) \
    nocopy(local_imaginary[0:local_size] : ALLOC) \
    nocopy(local_row[0:local_row_size]   : ALLOC) \
    nocopy(local_col[0:local_col_size]   : ALLOC)

        left_real      = plocalHm1->m_vectValueRealBuffer.data();
        left_imaginary = plocalHm1->m_vectValueImaginaryBuffer.data();
        left_row       = plocalHm1->m_vectRow.data();
        left_col       = plocalHm1->m_vectColumn.data();
        left_size = plocalHm1->GetNoneZeroCount();
        left_row_size = plocalHm1->m_vectRow.size();
        left_col_size = plocalHm1->m_vectColumn.size();
#pragma offload_transfer target(mic:phi_tid) \
    nocopy(left_real[0:left_size]      : ALLOC) \
    nocopy(left_imaginary[0:left_size] : ALLOC) \
    nocopy(left_row[0:left_row_size]   : ALLOC) \
    nocopy(left_col[0:left_col_size]   : ALLOC)

// FIXME jinpil: error occurred without the following statement
// error message: "offload error: unexpected variable type"
// any statements with side effects can also avoid the error, e.g. MPI_Barrier(MPI_COMM_WORLD)
        SHOW_SIMPLE_MSG("-Xeon Phi: mem alloc\n");
// the following statement cannot avoid the error
// compiler bug?
//      int dummy = 0;
// -----

        right_real      = plocalHp1->m_vectValueRealBuffer.data();
        right_imaginary = plocalHp1->m_vectValueImaginaryBuffer.data();
        right_row       = plocalHp1->m_vectRow.data();
        right_col       = plocalHp1->m_vectColumn.data();
        right_size = plocalHp1->GetNoneZeroCount();
        right_row_size = plocalHp1->m_vectRow.size();
        right_col_size = plocalHp1->m_vectColumn.size();
#pragma offload_transfer target(mic:phi_tid) \
    nocopy(right_real[0:right_size]      : ALLOC) \
    nocopy(right_imaginary[0:right_size] : ALLOC) \
    nocopy(right_row[0:right_row_size]   : ALLOC) \
    nocopy(right_col[0:right_col_size]   : ALLOC)
    }
    else
    {
        local_real      = pHamiltonian->m_vectValueRealBuffer.data();
        local_imaginary = pHamiltonian->m_vectValueImaginaryBuffer.data();
        local_row       = pHamiltonian->m_vectRow.data();
        local_col       = pHamiltonian->m_vectColumn.data();
        local_size     = pHamiltonian->GetNoneZeroCount();
        local_row_size = pHamiltonian->m_vectRow.size();
        local_col_size = pHamiltonian->m_vectColumn.size();
#pragma offload_transfer target(mic:phi_tid) \
    nocopy(local_real[0:local_size]      : ALLOC) \
    nocopy(local_imaginary[0:local_size] : ALLOC) \
    nocopy(local_row[0:local_row_size]   : ALLOC) \
    nocopy(local_col[0:local_col_size]   : ALLOC)
    }
    
    if(CKNMPIManager::GetTotalNodeCount() > 2)
    {
        CKNMatrixOperation::UpdateLocalCSR(pHamiltonian, plocalH, plocalHm1, plocalHp1);
        SHOW_SIMPLE_MSG("-Local matrices updated\n");

#pragma offload_transfer target(mic:phi_tid) \
    in(local_real[0:local_size]      : REUSE) \
    in(local_imaginary[0:local_size] : REUSE) \
    in(local_row[0:local_row_size]   : REUSE) \
    in(local_col[0:local_col_size]   : REUSE)

#pragma offload_transfer target(mic:phi_tid) \
    in(left_real[0:left_size]      : REUSE) \
    in(left_imaginary[0:left_size] : REUSE) \
    in(left_row[0:left_row_size]   : REUSE) \
    in(left_col[0:left_col_size]   : REUSE)

#pragma offload_transfer target(mic:phi_tid) \
    in(right_real[0:right_size]      : REUSE) \
    in(right_imaginary[0:right_size] : REUSE) \
    in(right_row[0:right_row_size]   : REUSE) \
    in(right_col[0:right_col_size]   : REUSE)
    }
    else
    {
#pragma offload_transfer target(mic:phi_tid) \
    in(local_real[0:local_size]      : REUSE) \
    in(local_imaginary[0:local_size] : REUSE) \
    in(local_row[0:local_row_size]   : REUSE) \
    in(local_col[0:local_col_size]   : REUSE)
    }

    CKNTimeMeasurement::InitTimer();
    CKNTimeMeasurement::TotalMeasurementStart();
    lpResult = lanczos.DoLanczosMethod(pHamiltonian,
        lpParam->nLanczosIterationCount,
        lpParam->nCheckEigenvalueInterval,
        lpParam->nFindingEigenValueCount,
        lpParam->fevMin,
        lpParam->fevMax,
        lpParam->fConvergeceCriteria,
        lpParam->bDoSelectiveReorthogonalization,
        lpParam->bCalculateEigenVectors,
        lpParam->bCalculateWaveFunction,
        lpParam->load_in_MIC,
        plocalH,   // ADDED
        plocalHm1, // ADDED
        plocalHp1);// ADDED

    if( CKNMPIManager::IsMultiLevelMPI() )
    {
        CKNLanczosMethod::MergeDegeneratedEigenvalues(lpResult, lpParam->nFindingDegeneratedEVCount, pHamiltonian, plocalH, plocalHm1, plocalHp1);
        if( lpParam->bCalculateWaveFunction )
            CKNLanczosMethod::RecalcuWaveFunction(lpResult);
    }
        
    if( lpParam->bSaveResultToFile )
    {
        if(CKNMPIManager::IsRootRank())
            CKNIPCCUtility::ShowMsg("\n-Save result into file\n");
        
        CKNLanczosMethod::SaveLanczosResult(lpResult, lpParam->bCalculateEigenVectors, lpParam->bCalculateWaveFunction, NULL, 0);
    }

    CKNTimeMeasurement::TotalMeasurementEnd();

    CKNLanczosMethod::ShowLanczosResult(lpResult, lpParam->bCalculateEigenVectors, lpParam->bCalculateWaveFunction, NULL, 0);

    if(CKNMPIManager::IsRootRank())
        SHOW_SIMPLE_MSG("\n-Finished Lanczos eigenvalue solver.\n\n");

    if(CKNMPIManager::GetTotalNodeCount() > 2)
    {
        CKNMatrixOperation::FreeLocalCSR(plocalH, plocalHm1, plocalHp1);

#pragma offload_transfer target(mic:phi_tid) \
    nocopy(local_real      : FREE) \
    nocopy(local_imaginary : FREE) \
    nocopy(local_row       : FREE) \
    nocopy(local_col       : FREE)

#pragma offload_transfer target(mic:phi_tid) \
    nocopy(left_real       : FREE) \
    nocopy(left_imaginary  : FREE) \
    nocopy(left_row        : FREE) \
    nocopy(left_col        : FREE)

#pragma offload_transfer target(mic:phi_tid) \
    nocopy(right_real      : FREE) \
    nocopy(right_imaginary : FREE) \
    nocopy(right_row       : FREE) \
    nocopy(right_col       : FREE)
    }
    else
    {
#pragma offload_transfer target(mic:phi_tid) \
    nocopy(local_real      : FREE) \
    nocopy(local_imaginary : FREE) \
    nocopy(local_row       : FREE) \
    nocopy(local_col       : FREE)
    }

    return lpResult;
}