Merge branch 'master' into sink_20190723

include/astaroth.h

@@ -57,6 +57,9 @@ AcResult acCheckDeviceAvailability(void);
  * parameter*/
 AcResult acSynchronizeStream(const Stream stream);
 
+/** */
+AcResult acSynchronizeMesh(void);
+
 /** Loads a constant to the memories of the devices visible to the caller */
 AcResult acLoadDeviceConstant(const AcRealParam param, const AcReal value);
 

include/astaroth_node.h

@@ -24,10 +24,21 @@ extern "C" {
 
 #include "astaroth_defines.h"
 
+#include "astaroth_device.h" // TODO: Should this really be here?
+
 typedef struct node_s* Node; // Opaque pointer to node_s.
 
 typedef struct {
+    int3 m;
+    int3 n;
+} Grid;
 
+typedef struct {
+    int num_devices;
+    Device* devices;
+
+    Grid grid;
+    Grid subgrid;
 } DeviceConfiguration;
 
 /** */

src/core/astaroth.cu

@@ -16,7 +16,6 @@
     You should have received a copy of the GNU General Public License
     along with Astaroth.  If not, see <http://www.gnu.org/licenses/>.
 */
-// #include "astaroth_defines.h"
 #include "astaroth.h"
 
 #include "errchk.h"
@@ -36,13 +35,10 @@ const char* vtxbuf_names[]     = {AC_FOR_VTXBUF_HANDLES(AC_GEN_STR)};
 
 static const int num_nodes = 1;
 static Node nodes[num_nodes];
-static int3 grid_n;
 
 AcResult
 acInit(const AcMeshInfo mesh_info)
 {
-    grid_n = (int3){mesh_info.int_params[AC_nx], mesh_info.int_params[AC_ny],
-                    mesh_info.int_params[AC_nz]};
     return acNodeCreate(0, mesh_info, &nodes[0]);
 }
 
@@ -96,18 +92,17 @@ acStore(AcMesh* host_mesh)
 AcResult
 acIntegrate(const AcReal dt)
 {
-    /*
-    acNodeIntegrate(nodes[0], dt);
-    return acBoundcondStep();
-    */
     return acNodeIntegrate(nodes[0], dt);
 }
 
 AcResult
 acIntegrateStep(const int isubstep, const AcReal dt)
 {
+    DeviceConfiguration config;
+    acNodeQueryDeviceConfiguration(nodes[0], &config);
+
     const int3 start = (int3){NGHOST, NGHOST, NGHOST};
-    const int3 end   = start + grid_n;
+    const int3 end   = start + config.grid.n;
     return acNodeIntegrateSubstep(nodes[0], STREAM_DEFAULT, isubstep, start, end, dt);
 }
 
@@ -151,3 +146,9 @@ acLoadWithOffset(const AcMesh host_mesh, const int3 src, const int num_vertices)
 {
     return acNodeLoadMeshWithOffset(nodes[0], STREAM_DEFAULT, host_mesh, src, src, num_vertices);
 }
+
+AcResult
+acSynchronizeMesh(void)
+{
+    return acNodeSynchronizeMesh(nodes[0], STREAM_DEFAULT);
+}

src/core/device.cu

@@ -40,14 +40,44 @@ typedef struct {
 } VertexBufferArray;
 
 __constant__ AcMeshInfo d_mesh_info;
-#define DCONST_INT(X) (d_mesh_info.int_params[X])
+static inline int __device__
-#define DCONST_INT3(X) (d_mesh_info.int3_params[X])
+DCONST(const AcIntParam param)
-#define DCONST_REAL(X) (d_mesh_info.real_params[X])
+{
-#define DCONST_REAL3(X) (d_mesh_info.real3_params[X])
+    return d_mesh_info.int_params[param];
+}
+static inline int3 __device__
+DCONST(const AcInt3Param param)
+{
+    return d_mesh_info.int3_params[param];
+}
+static inline AcReal __device__
+DCONST(const AcRealParam param)
+{
+    return d_mesh_info.real_params[param];
+}
+static inline AcReal3 __device__
+DCONST(const AcReal3Param param)
+{
+    return d_mesh_info.real3_params[param];
+}
+#define DCONST_INT(x) DCONST(x)
+#define DCONST_INT3(x) DCONST(x)
+#define DCONST_REAL(x) DCONST(x)
+#define DCONST_REAL3(x) DCONST(x)
+//#define DCONST_INT(X) (d_mesh_info.int_params[X])
+//#define DCONST_INT3(X) (d_mesh_info.int3_params[X])
+//#define DCONST_REAL(X) (d_mesh_info.real_params[X])
+//#define DCONST_REAL3(X) (d_mesh_info.real3_params[X])
 #define DEVICE_VTXBUF_IDX(i, j, k) ((i) + (j)*DCONST_INT(AC_mx) + (k)*DCONST_INT(AC_mxy))
 #define DEVICE_1D_COMPDOMAIN_IDX(i, j, k) ((i) + (j)*DCONST_INT(AC_nx) + (k)*DCONST_INT(AC_nxy))
 #define globalGridN (d_mesh_info.int3_params[AC_global_grid_n])
+//#define globalMeshM // Placeholder
+//#define localMeshN // Placeholder
+//#define localMeshM // Placeholder
+//#define localMeshN_min // Placeholder
+//#define globalMeshN_min // Placeholder
 #define d_multigpu_offset (d_mesh_info.int3_params[AC_multigpu_offset])
+//#define d_multinode_offset (d_mesh_info.int3_params[AC_multinode_offset]) // Placeholder
 #include "kernels/boundconds.cuh"
 #include "kernels/integration.cuh"
 #include "kernels/reductions.cuh"
@@ -92,6 +122,7 @@ acDeviceCreate(const int id, const AcMeshInfo device_config, Device* device_hand
 
     device->id           = id;
     device->local_config = device_config;
+    acDevicePrintInfo(device);
 
     // Check that the code was compiled for the proper GPU architecture
     printf("Trying to run a dummy kernel. If this fails, make sure that your\n"

src/core/node.cu

@@ -131,11 +131,6 @@
 
 static const int MAX_NUM_DEVICES = 32;
 
-typedef struct {
-    int3 m;
-    int3 n;
-} Grid;
-
 struct node_s {
     int id;
 
@@ -334,10 +329,12 @@ acNodePrintInfo(const Node node)
 AcResult
 acNodeQueryDeviceConfiguration(const Node node, DeviceConfiguration* config)
 {
-    (void)node;
+    config->num_devices = node->num_devices;
-    (void)config;
+    config->devices     = node->devices;
-    WARNING("Not implemented");
+    config->grid        = node->grid;
-    return AC_FAILURE;
+    config->subgrid     = node->subgrid;
+
+    return AC_SUCCESS;
 }
 
 AcResult