cwpearson/astaroth
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed the UCX warning from the last commit. Indexing of MPI_Waitall was wrong and also UCX required that MPI_Isend is also "waited" even though it should implicitly complete at the same time with MPI_Irecv

Browse Source
This commit is contained in:
jpekkila
2019-12-05 16:40:30 +02:00
parent e47cfad6b5
commit 9f4742bafe
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/core/device.cc
View File

@@ -775,6 +775,7 @@ acDeviceBoundStepMPI(const Device device)
} }
// MPI // MPI
MPI_Request send_requests[2 * NUM_VTXBUF_HANDLES];
MPI_Request recv_requests[2 * NUM_VTXBUF_HANDLES]; MPI_Request recv_requests[2 * NUM_VTXBUF_HANDLES];
MPI_Datatype datatype = MPI_FLOAT; MPI_Datatype datatype = MPI_FLOAT;
if (sizeof(AcReal) == 8) if (sizeof(AcReal) == 8)
@@ -812,21 +813,21 @@ acDeviceBoundStepMPI(const Device device)
const size_t src_idx = acVertexBufferIdx(0, 0, mz - 2 * NGHOST, device->local_config); const size_t src_idx = acVertexBufferIdx(0, 0, mz - 2 * NGHOST, device->local_config);
const int send_pid = (pid + 1) % num_processes; const int send_pid = (pid + 1) % num_processes;
MPI_Request request;
MPI_Isend(&device->vba.in[i][src_idx], count, datatype, send_pid, i, MPI_COMM_WORLD, MPI_Isend(&device->vba.in[i][src_idx], count, datatype, send_pid, i, MPI_COMM_WORLD,
&request); &send_requests[i]);
} }
{ // Send back { // Send back
// ...|ooooooo|xxx <- ...|xxxoooo|... // ...|ooooooo|xxx <- ...|xxxoooo|...
const size_t src_idx = acVertexBufferIdx(0, 0, NGHOST, device->local_config); const size_t src_idx = acVertexBufferIdx(0, 0, NGHOST, device->local_config);
const int send_pid = (pid + num_processes - 1) % num_processes; const int send_pid = (pid + num_processes - 1) % num_processes;
MPI_Request request;
MPI_Isend(&device->vba.in[i][src_idx], count, datatype, send_pid, MPI_Isend(&device->vba.in[i][src_idx], count, datatype, send_pid,
i + NUM_VTXBUF_HANDLES, MPI_COMM_WORLD, &request); i + NUM_VTXBUF_HANDLES, MPI_COMM_WORLD,
&send_requests[i + NUM_VTXBUF_HANDLES]);
} }
} }
MPI_Waitall(NUM_VTXBUF_HANDLES, recv_requests, MPI_STATUSES_IGNORE); MPI_Waitall(2 * NUM_VTXBUF_HANDLES, recv_requests, MPI_STATUSES_IGNORE);
MPI_Waitall(2 * NUM_VTXBUF_HANDLES, send_requests, MPI_STATUSES_IGNORE);
return AC_SUCCESS; return AC_SUCCESS;
} }