From d1a93b7d4e1f7be5e13ab48f22009fb7cfb5aadd Mon Sep 17 00:00:00 2001
From: jpekkila <johannes.pekkila@aalto.fi>
Date: Thu, 4 Jul 2019 16:58:24 +0300
Subject: [PATCH] acIntegrateStepWithOffset corrected and confirmed to work on
 1-4 GPUs

---
 src/core/astaroth.cu | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/core/astaroth.cu b/src/core/astaroth.cu
index 9b4afcb..c0f70d5 100644
--- a/src/core/astaroth.cu
+++ b/src/core/astaroth.cu
@@ -254,7 +254,6 @@ acStore(AcMesh* host_mesh)
 AcResult
 acIntegrateStepWithOffset(const int& isubstep, const AcReal& dt, const int3& start, const int3& end)
 {
-    WARNING("acIntegrateStepWithOffset called. This function has not been tested for correctness!");
     for (int i = 0; i < num_devices; ++i) {
         // DECOMPOSITION OFFSET HERE
         // Same naming here (d0, d1, da, db) as in acLoadWithOffset
@@ -265,7 +264,9 @@ acIntegrateStepWithOffset(const int& isubstep, const AcReal& dt, const int3& sta
         const int3 db = min(end, d1);
 
         if (db.z >= da.z) {
-            rkStep(devices[i], STREAM_PRIMARY, isubstep, da, db, dt);
+            const int3 da_local = da - (int3){0, 0, i * subgrid.n.z};
+            const int3 db_local = db - (int3){0, 0, i * subgrid.n.z};
+            rkStep(devices[i], STREAM_PRIMARY, isubstep, da_local, db_local, dt);
         }
     }
     return AC_SUCCESS;
@@ -275,12 +276,7 @@ AcResult
 acIntegrateStep(const int& isubstep, const AcReal& dt)
 {
     const int3 start = (int3){NGHOST, NGHOST, NGHOST};
-    const int3 end   = start + subgrid.n;
-    /* Deprecated block, TODO remove when acIntegrateStepWithOffset confirmed to work
-     for (int i = 0; i < num_devices; ++i) {
-        rkStep(devices[i], STREAM_PRIMARY, isubstep, start, end, dt);
-    }
-    */
+    const int3 end   = start + grid.n;
     acIntegrateStepWithOffset(isubstep, dt, start, end);
 
     return AC_SUCCESS;