From 224b91b83a0816f71d151a927e2a33bc98cc1234 Mon Sep 17 00:00:00 2001
From: jpekkila <johannes.pekkila@aalto.fi>
Date: Fri, 5 Jul 2019 15:17:20 +0300
Subject: [PATCH] Added more control for synchronizing streams and halos among
 the GPUs

---
 src/core/astaroth.cu | 39 +++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/src/core/astaroth.cu b/src/core/astaroth.cu
index ff655c0..a141295 100644
--- a/src/core/astaroth.cu
+++ b/src/core/astaroth.cu
@@ -260,8 +260,8 @@ acSwapBuffers(void)
     return AC_SUCCESS;
 }
 
-AcResult
-acExchangeHalos(void)
+static AcResult
+acSynchronizeHalos(void)
 {
     // Exchanges the halos of subgrids
     // After this step, the data within the main grid ranging from
@@ -290,6 +290,26 @@ acExchangeHalos(void)
     return AC_SUCCESS;
 }
 
+static AcResult
+acSynchronizeStream(const StreamType stream)
+{
+    for (int i = 0; i < num_devices; ++i) {
+        synchronize(devices[i], stream);
+    }
+
+    return AC_SUCCESS;
+}
+
+AcResult
+acSynchronize(void)
+{
+    acSynchronizeStream(STREAM_ALL);
+    acSynchronizeHalos();
+    acSynchronizeStream(STREAM_ALL);
+
+    return AC_SUCCESS;
+}
+
 AcResult
 acBoundcondStep(void)
 {
@@ -397,10 +417,7 @@ acBoundcondStep(void)
                         periodic_boundconds(0, tpb, start, end, d_buffer.in[i]);
         <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
         */
-        // Exchange halos
-        acExchangeHalos();
-
-        // With periodic boundary conditions we also exchange the front and back plates of the
+        // With periodic boundary conditions we exchange the front and back plates of the
         // grid. The exchange is done between the first and last device (0 and num_devices - 1).
         const int num_vertices = subgrid.m.x * subgrid.m.y * NGHOST;
         // ...|ooooxxx|... -> xxx|ooooooo|...
@@ -514,16 +531,6 @@ acReduceVec(const ReductionType& rtype, const VertexBufferHandle& a, const Verte
     return simple_final_reduce_scal(rtype, results, num_devices);
 }
 
-AcResult
-acSynchronize(void)
-{
-    for (int i = 0; i < num_devices; ++i) {
-        synchronize(devices[i], STREAM_ALL);
-    }
-
-    return AC_SUCCESS;
-}
-
 AcResult
 acLoadDeviceConstant(const AcRealParam param, const AcReal value)
 {