From a91839723a2ef63c343371c57d9a59e5e4b438f1 Mon Sep 17 00:00:00 2001 From: Gerald Paul Bowen Collom Date: Tue, 27 Jun 2023 23:06:19 -0700 Subject: [PATCH] Use config flag for locality-aware mpi --- src/parcsr_mv/_hypre_parcsr_mv.h | 15 ++ src/parcsr_mv/new_commpkg.c | 25 ++++ src/parcsr_mv/par_csr_communication.c | 191 +++++++++++++++++++++++++- src/parcsr_mv/par_csr_communication.h | 15 ++ 4 files changed, 244 insertions(+), 2 deletions(-) diff --git a/src/parcsr_mv/_hypre_parcsr_mv.h b/src/parcsr_mv/_hypre_parcsr_mv.h index c492ec71ee..d799ec1dcc 100644 --- a/src/parcsr_mv/_hypre_parcsr_mv.h +++ b/src/parcsr_mv/_hypre_parcsr_mv.h @@ -23,6 +23,10 @@ extern "C" { #ifndef HYPRE_PAR_CSR_COMMUNICATION_HEADER #define HYPRE_PAR_CSR_COMMUNICATION_HEADER +#ifdef HYPRE_USING_NODE_AWARE_MPI +#include "mpi_advance.h" +#endif + /*-------------------------------------------------------------------------- * hypre_ParCSRCommPkg: * Structure containing information for doing communications @@ -59,6 +63,9 @@ typedef struct void *recv_data_buffer; HYPRE_Int num_requests; hypre_MPI_Request *requests; +#ifdef HYPRE_USING_NODE_AWARE_MPI + MPIX_Request *Xrequest; +#endif } hypre_ParCSRCommHandle; typedef hypre_ParCSRCommHandle hypre_ParCSRPersistentCommHandle; @@ -66,6 +73,10 @@ typedef hypre_ParCSRCommHandle hypre_ParCSRPersistentCommHandle; typedef struct _hypre_ParCSRCommPkg { MPI_Comm comm; +#ifdef HYPRE_USING_NODE_AWARE_MPI + MPIX_Comm *neighbor_comm; + MPIX_Comm *neighborT_comm; +#endif HYPRE_Int num_components; HYPRE_Int num_sends; HYPRE_Int *send_procs; @@ -75,6 +86,10 @@ typedef struct _hypre_ParCSRCommPkg HYPRE_Int num_recvs; HYPRE_Int *recv_procs; HYPRE_Int *recv_vec_starts; +#ifdef HYPRE_USING_NODE_AWARE_MPI + long *global_send_indices; + long *global_recv_indices; +#endif /* remote communication information */ hypre_MPI_Datatype *send_mpi_types; hypre_MPI_Datatype *recv_mpi_types; diff --git a/src/parcsr_mv/new_commpkg.c b/src/parcsr_mv/new_commpkg.c index b12c2112ec..a6ccf9f9ce 100644 --- a/src/parcsr_mv/new_commpkg.c +++ b/src/parcsr_mv/new_commpkg.c @@ -11,6 +11,9 @@ *-----------------------------------------------------*/ #include "_hypre_parcsr_mv.h" +#ifdef HYPRE_USING_NODE_AWARE_MPI +#include +#endif /* some debugging tools*/ #define mydebug 0 @@ -546,6 +549,28 @@ hypre_ParCSRCommPkgCreateApart num_sends, send_procs, send_map_starts, send_map_elmts, &comm_pkg); +#ifdef HYPRE_USING_NODE_AWARE_MPI + MPIX_Dist_graph_create_adjacent( comm, num_recvs, hypre_ParCSRCommPkgRecvProcs(comm_pkg), + MPI_UNWEIGHTED, num_sends, hypre_ParCSRCommPkgSendProcs(comm_pkg), + MPI_UNWEIGHTED, MPI_INFO_NULL, 0, &(comm_pkg->neighbor_comm)); + MPIX_Dist_graph_create_adjacent( comm, num_sends, hypre_ParCSRCommPkgSendProcs(comm_pkg), + MPI_UNWEIGHTED, num_recvs, hypre_ParCSRCommPkgRecvProcs(comm_pkg), + MPI_UNWEIGHTED, 0, 0, &(comm_pkg->neighborT_comm)); + + HYPRE_Int num_send_elmts = send_map_starts[num_sends]; + comm_pkg->global_send_indices = hypre_CTAlloc(long, num_send_elmts, HYPRE_MEMORY_HOST); + for (HYPRE_Int i = 0; i < num_send_elmts; ++i) + { + comm_pkg->global_send_indices[i] = send_map_elmts[i] + first_col_diag; + } + + HYPRE_Int num_recv_elmts = recv_vec_starts[num_recvs]; + comm_pkg->global_recv_indices = hypre_CTAlloc(long, num_recv_elmts, HYPRE_MEMORY_HOST); + for (HYPRE_Int i = 0; i < num_recv_elmts; ++i) + { + comm_pkg->global_recv_indices[i] = col_map_off_d[i]; + } +#endif return hypre_error_flag; } diff --git a/src/parcsr_mv/par_csr_communication.c b/src/parcsr_mv/par_csr_communication.c index 38468f2bce..c85cbb09a6 100644 --- a/src/parcsr_mv/par_csr_communication.c +++ b/src/parcsr_mv/par_csr_communication.c @@ -7,6 +7,10 @@ #include "_hypre_parcsr_mv.h" +#ifdef HYPRE_USING_NODE_AWARE_MPI +#include "mpi_advance.h" +#endif + /*==========================================================================*/ #ifdef HYPRE_USING_PERSISTENT_COMM @@ -60,11 +64,22 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_Int num_recvs = hypre_ParCSRCommPkgNumRecvs(comm_pkg); MPI_Comm comm = hypre_ParCSRCommPkgComm(comm_pkg); +#ifndef HYPRE_USING_NODE_AWARE_MPI HYPRE_Int num_requests = num_sends + num_recvs; hypre_MPI_Request *requests = hypre_CTAlloc(hypre_MPI_Request, num_requests, HYPRE_MEMORY_HOST); hypre_ParCSRCommHandleNumRequests(comm_handle) = num_requests; hypre_ParCSRCommHandleRequests(comm_handle) = requests; +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + MPIX_Request *Xrequest; + + HYPRE_Int *send_sizes = hypre_TAlloc(HYPRE_Int, num_sends, HYPRE_MEMORY_HOST); + HYPRE_Int *recv_sizes = hypre_TAlloc(HYPRE_Int, num_recvs, HYPRE_MEMORY_HOST); + + HYPRE_BigInt num_send_elmts = hypre_ParCSRCommPkgSendMapStart(comm_pkg, num_sends); + HYPRE_BigInt num_recv_elmts = hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs); +#endif void *send_buff = NULL, *recv_buff = NULL; @@ -77,6 +92,7 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MEMORY_HOST); recv_buff = hypre_TAlloc(HYPRE_Complex, hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs), HYPRE_MEMORY_HOST); +#ifndef HYPRE_USING_NODE_AWARE_MPI for (i = 0; i < num_recvs; ++i) { HYPRE_Int ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i); @@ -93,6 +109,28 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm hypre_MPI_Send_init( (HYPRE_Complex *)send_buff + vec_start, vec_len, HYPRE_MPI_COMPLEX, ip, 0, comm, requests + num_recvs + i ); } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + for (i = 0; i < num_recvs; ++i) + { + recv_sizes[i] = (hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i)); + } + for (i = 0; i < num_sends; ++i) + { + send_sizes[i] = (hypre_ParCSRCommPkgSendMapStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgSendMapStart(comm_pkg, i)); + } + MPIX_Neighbor_locality_alltoallv_init( (HYPRE_Complex *)send_buff, send_sizes, + hypre_ParCSRCommPkgSendMapStarts(comm_pkg), + comm_pkg->global_send_indices, + HYPRE_MPI_COMPLEX, + (HYPRE_Complex *)recv_buff, recv_sizes, + hypre_ParCSRCommPkgRecvVecStarts(comm_pkg), + comm_pkg->global_recv_indices, + HYPRE_MPI_COMPLEX, comm_pkg->neighbor_comm, + 0, &Xrequest); +#endif break; case HYPRE_COMM_PKG_JOB_COMPLEX_TRANSPOSE: @@ -102,6 +140,7 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MEMORY_HOST); send_buff = hypre_TAlloc(HYPRE_Complex, hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs), HYPRE_MEMORY_HOST); +#ifndef HYPRE_USING_NODE_AWARE_MPI for (i = 0; i < num_sends; ++i) { HYPRE_Int ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i); @@ -118,6 +157,28 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm hypre_MPI_Send_init( (HYPRE_Complex *)send_buff + vec_start, vec_len, HYPRE_MPI_COMPLEX, ip, 0, comm, requests + num_sends + i ); } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + for (i = 0; i < num_recvs; ++i) + { + recv_sizes[i] = (hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i)); + } + for (i = 0; i < num_sends; ++i) + { + send_sizes[i] = (hypre_ParCSRCommPkgSendMapStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgSendMapStart(comm_pkg, i)); + } + MPIX_Neighbor_locality_alltoallv_init( (HYPRE_Complex *)send_buff, recv_sizes, + hypre_ParCSRCommPkgRecvVecStarts(comm_pkg), + comm_pkg->global_recv_indices, + HYPRE_MPI_COMPLEX, + (HYPRE_Complex *)recv_buff, send_sizes, + hypre_ParCSRCommPkgSendMapStarts(comm_pkg), + comm_pkg->global_send_indices, + HYPRE_MPI_COMPLEX, comm_pkg->neighborT_comm, + 0, &Xrequest); +#endif break; case HYPRE_COMM_PKG_JOB_INT: @@ -127,6 +188,7 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MEMORY_HOST); recv_buff = hypre_TAlloc(HYPRE_Int, hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs), HYPRE_MEMORY_HOST); +#ifndef HYPRE_USING_NODE_AWARE_MPI for (i = 0; i < num_recvs; ++i) { HYPRE_Int ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i); @@ -143,6 +205,28 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm hypre_MPI_Send_init( (HYPRE_Int *)send_buff + vec_start, vec_len, HYPRE_MPI_INT, ip, 0, comm, requests + num_recvs + i ); } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + for (i = 0; i < num_recvs; ++i) + { + recv_sizes[i] = (hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i)); + } + for (i = 0; i < num_sends; ++i) + { + send_sizes[i] = (hypre_ParCSRCommPkgSendMapStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgSendMapStart(comm_pkg, i)); + } + MPIX_Neighbor_locality_alltoallv_init( (HYPRE_Int *)send_buff, send_sizes, + hypre_ParCSRCommPkgSendMapStarts(comm_pkg), + comm_pkg->global_send_indices, + HYPRE_MPI_INT, + (HYPRE_Int *)recv_buff, recv_sizes, + hypre_ParCSRCommPkgRecvVecStarts(comm_pkg), + comm_pkg->global_recv_indices, + HYPRE_MPI_INT, comm_pkg->neighbor_comm, + 0, &Xrequest); +#endif break; case HYPRE_COMM_PKG_JOB_INT_TRANSPOSE: @@ -152,6 +236,7 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MEMORY_HOST); send_buff = hypre_TAlloc(HYPRE_Int, hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs), HYPRE_MEMORY_HOST); +#ifndef HYPRE_USING_NODE_AWARE_MPI for (i = 0; i < num_sends; ++i) { HYPRE_Int ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i); @@ -168,6 +253,28 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm hypre_MPI_Send_init( (HYPRE_Int *)send_buff + vec_start, vec_len, HYPRE_MPI_INT, ip, 0, comm, requests + num_sends + i ); } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + for (i = 0; i < num_recvs; ++i) + { + recv_sizes[i] = (hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i)); + } + for (i = 0; i < num_sends; ++i) + { + send_sizes[i] = (hypre_ParCSRCommPkgSendMapStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgSendMapStart(comm_pkg, i)); + } + MPIX_Neighbor_locality_alltoallv_init( (HYPRE_Int *)send_buff, recv_sizes, + hypre_ParCSRCommPkgRecvVecStarts(comm_pkg), + comm_pkg->global_recv_indices, + HYPRE_MPI_INT, + (HYPRE_Int *)recv_buff, send_sizes, + hypre_ParCSRCommPkgSendMapStarts(comm_pkg), + comm_pkg->global_send_indices, + HYPRE_MPI_INT, comm_pkg->neighborT_comm, + 0, &Xrequest); +#endif break; case HYPRE_COMM_PKG_JOB_BIGINT: @@ -177,6 +284,7 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MEMORY_HOST); recv_buff = hypre_TAlloc(HYPRE_BigInt, hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs), HYPRE_MEMORY_HOST); +#ifndef HYPRE_USING_NODE_AWARE_MPI for (i = 0; i < num_recvs; ++i) { HYPRE_Int ip = hypre_ParCSRCommPkgRecvProc(comm_pkg, i); @@ -195,6 +303,28 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MPI_BIG_INT, ip, 0, comm, requests + num_recvs + i); } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + for (i = 0; i < num_recvs; ++i) + { + recv_sizes[i] = (hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i)); + } + for (i = 0; i < num_sends; ++i) + { + send_sizes[i] = (hypre_ParCSRCommPkgSendMapStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgSendMapStart(comm_pkg, i)); + } + MPIX_Neighbor_locality_alltoallv_init( (HYPRE_BigInt *)send_buff, send_sizes, + hypre_ParCSRCommPkgSendMapStarts(comm_pkg), + comm_pkg->global_send_indices, + HYPRE_MPI_BIG_INT, + (HYPRE_BigInt *)recv_buff, recv_sizes, + hypre_ParCSRCommPkgRecvVecStarts(comm_pkg), + comm_pkg->global_recv_indices, + HYPRE_MPI_BIG_INT, comm_pkg->neighbor_comm, + 0, &Xrequest); +#endif break; case HYPRE_COMM_PKG_JOB_BIGINT_TRANSPOSE: @@ -204,6 +334,7 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MEMORY_HOST); send_buff = hypre_TAlloc(HYPRE_BigInt, hypre_ParCSRCommPkgRecvVecStart(comm_pkg, num_recvs), HYPRE_MEMORY_HOST); +#ifndef HYPRE_USING_NODE_AWARE_MPI for (i = 0; i < num_sends; ++i) { HYPRE_Int ip = hypre_ParCSRCommPkgSendProc(comm_pkg, i); @@ -223,6 +354,28 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm HYPRE_MPI_BIG_INT, ip, 0, comm, requests + num_sends + i); } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + for (i = 0; i < num_recvs; ++i) + { + recv_sizes[i] = (hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgRecvVecStart(comm_pkg, i)); + } + for (i = 0; i < num_sends; ++i) + { + send_sizes[i] = (hypre_ParCSRCommPkgSendMapStart(comm_pkg, i + 1) - + hypre_ParCSRCommPkgSendMapStart(comm_pkg, i)); + } + MPIX_Neighbor_locality_alltoallv_init( (HYPRE_BigInt *)send_buff, recv_sizes, + hypre_ParCSRCommPkgRecvVecStarts(comm_pkg), + comm_pkg->global_recv_indices, + HYPRE_MPI_BIG_INT, + (HYPRE_BigInt *)recv_buff, send_sizes, + hypre_ParCSRCommPkgSendMapStarts(comm_pkg), + comm_pkg->global_send_indices, + HYPRE_MPI_BIG_INT, comm_pkg->neighborT_comm, + 0, &Xrequest); +#endif break; default: hypre_assert(1 == 0); @@ -234,6 +387,14 @@ hypre_ParCSRPersistentCommHandleCreate( HYPRE_Int job, hypre_ParCSRCommPkg *comm hypre_ParCSRCommHandleNumSendBytes(comm_handle) = num_bytes_send; hypre_ParCSRCommHandleNumRecvBytes(comm_handle) = num_bytes_recv; + +#ifdef HYPRE_USING_NODE_AWARE_MPI + comm_handle->Xrequest = Xrequest; + + hypre_TFree(send_sizes, HYPRE_MEMORY_HOST); + hypre_TFree(recv_sizes, HYPRE_MEMORY_HOST); +#endif + return ( comm_handle ); } @@ -266,8 +427,16 @@ hypre_ParCSRPersistentCommHandleDestroy( hypre_ParCSRPersistentCommHandle *comm_ { hypre_TFree(hypre_ParCSRCommHandleSendDataBuffer(comm_handle), HYPRE_MEMORY_HOST); hypre_TFree(hypre_ParCSRCommHandleRecvDataBuffer(comm_handle), HYPRE_MEMORY_HOST); - hypre_TFree(comm_handle->requests, HYPRE_MEMORY_HOST); - +#ifndef HYPRE_USING_NODE_AWARE_MPI + if (comm_handle->requests) { + hypre_TFree(comm_handle->requests, HYPRE_MEMORY_HOST); + } +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + if (comm_handle->Xrequest) { + MPIX_Request_free(comm_handle->Xrequest); + } +#endif hypre_TFree(comm_handle, HYPRE_MEMORY_HOST); } } @@ -293,8 +462,14 @@ hypre_ParCSRPersistentCommHandleStart( hypre_ParCSRPersistentCommHandle *comm_ha HYPRE_MEMORY_HOST, send_memory_location ); +#ifndef HYPRE_USING_NODE_AWARE_MPI HYPRE_Int ret = hypre_MPI_Startall(hypre_ParCSRCommHandleNumRequests(comm_handle), hypre_ParCSRCommHandleRequests(comm_handle)); +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + HYPRE_Int ret = (HYPRE_Int) MPIX_Start(comm_handle->Xrequest); +#endif + if (hypre_MPI_SUCCESS != ret) { hypre_error_w_msg(HYPRE_ERROR_GENERIC, "MPI error\n"); @@ -317,9 +492,17 @@ hypre_ParCSRPersistentCommHandleWait( hypre_ParCSRPersistentCommHandle *comm_han if (hypre_ParCSRCommHandleNumRequests(comm_handle) > 0) { + +#ifndef HYPRE_USING_NODE_AWARE_MPI HYPRE_Int ret = hypre_MPI_Waitall(hypre_ParCSRCommHandleNumRequests(comm_handle), hypre_ParCSRCommHandleRequests(comm_handle), hypre_MPI_STATUSES_IGNORE); +#endif +#ifdef HYPRE_USING_NODE_AWARE_MPI + HYPRE_Int ret = (HYPRE_Int) MPIX_Wait( comm_handle->Xrequest, + MPI_STATUS_IGNORE); +#endif + if (hypre_MPI_SUCCESS != ret) { hypre_error_w_msg(HYPRE_ERROR_GENERIC, "MPI error\n"); @@ -1214,6 +1397,10 @@ hypre_MatvecCommPkgCreate ( hypre_ParCSRMatrix *A ) num_cols_offd, global_num_cols, apart, comm_pkg ); + //int rank; + //MPI_Comm_rank(MPI_COMM_WORLD, &rank); + //if (rank == 0) {hypre_printf("test: %d, %d\n", A->global_num_rows, A->global_num_cols);} + HYPRE_ANNOTATE_FUNC_END; diff --git a/src/parcsr_mv/par_csr_communication.h b/src/parcsr_mv/par_csr_communication.h index d0d5b846a7..79898ebca6 100644 --- a/src/parcsr_mv/par_csr_communication.h +++ b/src/parcsr_mv/par_csr_communication.h @@ -8,6 +8,10 @@ #ifndef HYPRE_PAR_CSR_COMMUNICATION_HEADER #define HYPRE_PAR_CSR_COMMUNICATION_HEADER +#ifdef HYPRE_USING_NODE_AWARE_MPI +#include "mpi_advance.h" +#endif + /*-------------------------------------------------------------------------- * hypre_ParCSRCommPkg: * Structure containing information for doing communications @@ -44,6 +48,9 @@ typedef struct void *recv_data_buffer; HYPRE_Int num_requests; hypre_MPI_Request *requests; +#ifdef HYPRE_USING_NODE_AWARE_MPI + MPIX_Request *Xrequest; +#endif } hypre_ParCSRCommHandle; typedef hypre_ParCSRCommHandle hypre_ParCSRPersistentCommHandle; @@ -51,6 +58,10 @@ typedef hypre_ParCSRCommHandle hypre_ParCSRPersistentCommHandle; typedef struct _hypre_ParCSRCommPkg { MPI_Comm comm; +#ifdef HYPRE_USING_NODE_AWARE_MPI + MPIX_Comm *neighbor_comm; + MPIX_Comm *neighborT_comm; +#endif HYPRE_Int num_components; HYPRE_Int num_sends; HYPRE_Int *send_procs; @@ -60,6 +71,10 @@ typedef struct _hypre_ParCSRCommPkg HYPRE_Int num_recvs; HYPRE_Int *recv_procs; HYPRE_Int *recv_vec_starts; +#ifdef HYPRE_USING_NODE_AWARE_MPI + long *global_send_indices; + long *global_recv_indices; +#endif /* remote communication information */ hypre_MPI_Datatype *send_mpi_types; hypre_MPI_Datatype *recv_mpi_types;