added SynchronizerMPI test

tests/common.h

+// File       : common.h
+// Created    : Sun Aug 12 2018 04:33:51 PM (+0200)
+// Author     : Fabian Wermelinger
+// Description: Common stuff
+// Copyright 2018 ETH Zurich. All Rights Reserved.
+#ifndef COMMON_H_QLCQRKJP
+#define COMMON_H_QLCQRKJP
+
+#ifdef _DOUBLE_
+using MyReal = double;
+#else
+using MyReal = float;
+#define _FLOAT_PRECISION_
+#endif /* _DOUBLE_ */
+
+#define _BLOCKSIZE_ 16
+#define _BLOCKSIZEX_ _BLOCKSIZE_
+#define _BLOCKSIZEY_ _BLOCKSIZE_
+#define _BLOCKSIZEZ_ _BLOCKSIZE_
+
+#include <cassert>
+#include <mpi.h>
+#include <cstring>
+#include <vector>
+#include <string>
+#include <sstream>
+
+#include "BlockInfo.h"
+
+template <typename TReal, size_t _AOSmembers=1>
+struct Block
+{
+    typedef TReal ElementType;
+    typedef TReal element_type;
+    static const size_t sizeX   = _BLOCKSIZE_;
+    static const size_t sizeY   = _BLOCKSIZE_;
+    static const size_t sizeZ   = _BLOCKSIZE_;
+    static const size_t members = _AOSmembers;
+
+    inline void clear() { memset(&m_data[0][0][0][0], 0, _BLOCKSIZE_*_BLOCKSIZE_*_BLOCKSIZE_*_AOSmembers*sizeof(TReal)); }
+
+    inline const TReal (&operator()(const size_t ix, const size_t iy, const size_t iz) const)[_AOSmembers]
+    {
+        assert(ix<_BLOCKSIZE_);
+        assert(iy<_BLOCKSIZE_);
+        assert(iz<_BLOCKSIZE_);
+        return this->m_data[iz][iy][ix];
+    }
+
+    inline TReal (&operator()(const size_t ix, const size_t iy, const size_t iz))[_AOSmembers]
+    {
+        assert(ix<_BLOCKSIZE_);
+        assert(iy<_BLOCKSIZE_);
+        assert(iz<_BLOCKSIZE_);
+        return this->m_data[iz][iy][ix];
+    }
+
+    TReal m_data[_BLOCKSIZE_][_BLOCKSIZE_][_BLOCKSIZE_][_AOSmembers];
+};
+
+template <size_t _comp=0>
+struct Streamer
+{
+    static const int NCHANNELS = 1;
+
+    template <typename TBlock, typename T>
+    static inline void operate(const TBlock& b, const int ix, const int iy, const int iz, T output[NCHANNELS])
+    {
+        assert(_comp<TBlock::members);
+        output[0] = b(ix,iy,iz)[_comp];
+    }
+
+    static const char * getAttributeName() { return "Scalar"; }
+};
+
+template <typename TGrid>
+void set_grid_ic(TGrid* grid, const int myrank=0)
+{
+    using TBlock = typename TGrid::BlockType;
+    std::vector<BlockInfo> infos = grid->getResidentBlocksInfo();
+
+    const int NX = grid->getResidentBlocksPerDimension(0);
+    const int NY = grid->getResidentBlocksPerDimension(1);
+    const int NZ = grid->getResidentBlocksPerDimension(2);
+
+    const double blocksize = TBlock::members*TBlock::sizeX*TBlock::sizeY*TBlock::sizeZ;
+    const double offset = myrank * NX*NY*NZ * blocksize;
+
+#pragma omp parallel for
+    for (size_t i = 0; i < infos.size(); ++i)
+    {
+        BlockInfo info = infos[i];
+        TBlock& b = *(TBlock*)info.ptrBlock;
+        for (size_t iz = 0; iz < TBlock::sizeZ; ++iz)
+            for (size_t iy = 0; iy < TBlock::sizeY; ++iy)
+                for (size_t ix = 0; ix < TBlock::sizeX; ++ix)
+                    for (size_t m = 0; m < TBlock::members; ++m)
+                        b(ix,iy,iz)[m] = offset + info.blockID*blocksize + m + TBlock::members*(ix + (double)TBlock::sizeX*(iy + TBlock::sizeY*iz));
+    }
+}
+
+#endif /* COMMON_H_QLCQRKJP */

tests/dumpers/Makefile

@@ -8,13 +8,13 @@ nonuniform ?= no
 
 extra ?=
 
-CFLAGS = -fopenmp -I../../source
+CFLAGS = -fopenmp -g -O2 -I../../source
 CFLAGS+= $(extra)
 # CFLAGS += -I${HDF5_ROOT}/include
-# CFLAGS+= -Wall -Wextra -Wfloat-equal -Wundef -Wcast-align
-# CFLAGS+= -Wwrite-strings -Wmissing-declarations -Wredundant-decls
+CFLAGS+= -Wall -Wextra -Wfloat-equal -Wundef -Wcast-align
+CFLAGS+= -Wwrite-strings -Wmissing-declarations -Wredundant-decls
 # CFLAGS+= -Wlogical-op
-# CFLAGS+= -Wshadow -Woverloaded-virtual -Wuninitialized -pedantic
+CFLAGS+= -Wshadow -Woverloaded-virtual -Wuninitialized
 CFLAGS+= -Wpedantic -Wno-unused-parameter # does not make much sense with function overloading
 
 

tests/dumpers/testDumpsMPI.cpp

@@ -3,36 +3,12 @@
 // Author     : Fabian Wermelinger
 // Description: Test Cubism dumping facilities
 // Copyright 2018 ETH Zurich. All Rights Reserved.
-#ifdef _DOUBLE_
-using MyReal = double;
-#else
-using MyReal = float;
-#define _FLOAT_PRECISION_
-#endif /* _DOUBLE_ */
-
-#define _BLOCKSIZE_ 16
-#define _BLOCKSIZEX_ _BLOCKSIZE_
-#define _BLOCKSIZEY_ _BLOCKSIZE_
-#define _BLOCKSIZEZ_ _BLOCKSIZE_
-
-#include <mpi.h>
-#include <cstring>
-#include <vector>
-#include <string>
-#include <sstream>
-using namespace std;
+#include "../common.h"
 
 #include "ArgumentParser.h"
 #include "Grid.h"
 #include "GridMPI.h"
-#include "BlockInfo.h"
-
-// dumpers
-#ifndef _HDF5_DOUBLE_PRECISION_
-const string prec_string = "4byte";
-#else
-const string prec_string = "8byte";
-#endif
+#include "MeshMap.h"
 
 #define _USE_HDF_
 #include "HDF5Dumper.h"
@@ -49,34 +25,17 @@ const string prec_string = "8byte";
 
 #include "PlainBinDumper_MPI.h"
 
+using namespace std;
 
-template <typename TReal>
-struct Block
-{
-    static const size_t sizeX = _BLOCKSIZE_;
-    static const size_t sizeY = _BLOCKSIZE_;
-    static const size_t sizeZ = _BLOCKSIZE_;
-
-    typedef TReal ElementType;
-
-    TReal data[_BLOCKSIZE_][_BLOCKSIZE_][_BLOCKSIZE_];
-    inline void clear() { memset(&data[0][0][0], 0, _BLOCKSIZE_*_BLOCKSIZE_*_BLOCKSIZE_*sizeof(TReal)); }
-};
-
-struct MyStreamer
-{
-    static const int NCHANNELS = 1;
-
-    template <typename TBlock, typename T>
-    static inline void operate(const TBlock& b, const int ix, const int iy, const int iz, T output[NCHANNELS])
-    {
-        output[0] = b.data[iz][iy][ix];
-    }
-
-    static const char * getAttributeName() { return "Scalar"; }
-};
+// dumpers
+#ifndef _HDF5_DOUBLE_PRECISION_
+const string prec_string = "4byte";
+#else
+const string prec_string = "8byte";
+#endif
 
-using MyBlock        = Block<MyReal>;
+using MyBlock        = Block<MyReal,1>;
+using MyStreamer     = Streamer<0>;
 using MyGrid         = Grid<MyBlock>;
 using MyGridMPI      = GridMPI<MyGrid>;
 using MySlice        = typename SliceTypes::Slice<MyGrid>;
@@ -89,30 +48,6 @@ using MyDensity = RandomDensity;
 #endif /* _NONUNIFORM_ */
 
 
-static void set_grid_ic(MyGridMPI* grid, const int myrank=0)
-{
-    vector<BlockInfo> infos = grid->getResidentBlocksInfo();
-
-    const int NX = grid->getResidentBlocksPerDimension(0);
-    const int NY = grid->getResidentBlocksPerDimension(1);
-    const int NZ = grid->getResidentBlocksPerDimension(2);
-
-    const double blocksize = MyBlock::sizeX*MyBlock::sizeY*MyBlock::sizeZ;
-    const double offset = myrank * NX*NY*NZ * blocksize;
-
-#pragma omp parallel for
-    for (size_t i = 0; i < infos.size(); ++i)
-    {
-        BlockInfo info = infos[i];
-        MyBlock& b = *(MyBlock*)info.ptrBlock;
-        for (size_t iz = 0; iz < MyBlock::sizeZ; ++iz)
-            for (size_t iy = 0; iy < MyBlock::sizeY; ++iy)
-                for (size_t ix = 0; ix < MyBlock::sizeX; ++ix)
-                    b.data[iz][iy][ix] = offset + info.blockID*blocksize + ix + (double)MyBlock::sizeX*(iy + MyBlock::sizeY*iz);
-    }
-}
-
-
 int main(int argc, char* argv[])
 {
     MPI_Init(&argc, &argv);

tests/synchronizer/Makefile

+SHELL := /bin/bash
+
+CC = mpic++
+
+precision ?= single
+nonuniform ?= no
+
+extra ?=
+
+CFLAGS = -fopenmp -g -O2 -I../../source
+CFLAGS+= $(extra)
+# CFLAGS += -I${HDF5_ROOT}/include
+CFLAGS+= -Wall -Wextra -Wfloat-equal -Wundef -Wcast-align
+CFLAGS+= -Wwrite-strings -Wmissing-declarations -Wredundant-decls
+# CFLAGS+= -Wlogical-op
+CFLAGS+= -Wshadow -Woverloaded-virtual -Wuninitialized
+CFLAGS+= -Wpedantic -Wno-unused-parameter # does not make much sense with function overloading
+
+
+.PHONY: all clean
+
+$(warning using precision=$(precision))
+$(warning using nonuniform=$(nonuniform))
+ifeq ("$(precision)","double")
+	CFLAGS += -D_DOUBLE_
+endif
+
+ifeq ("$(nonuniform)","yes")
+	CFLAGS += -D_NONUNIFORM_
+endif
+
+all: testSynchronizerMPI.cpp
+	$(CC) $(CFLAGS) -o testSynchronizerMPI testSynchronizerMPI.cpp
+
+clean:
+	rm -f testSynchronizerMPI *.o

tests/synchronizer/testSynchronizerMPI.cpp

+// File       : testSynchronizerMPI.cpp
+// Created    : Sun Aug 12 2018 04:25:45 PM (+0200)
+// Author     : Fabian Wermelinger
+// Description: Test program for MPI stencil synchronizer
+// Copyright 2018 ETH Zurich. All Rights Reserved.
+#include <iostream>
+
+#include "../common.h"
+#include "ArgumentParser.h"
+#include "Grid.h"
+#include "GridMPI.h"
+#include "MeshMap.h"
+#include "SynchronizerMPI.h"
+#include "StencilInfo.h"
+#include "Profiler.h"
+
+using namespace std;
+
+using MyBlock   = Block<MyReal,32>;
+using MyGrid    = Grid<MyBlock>;
+using MyGridMPI = GridMPI<MyGrid>;
+#ifdef _NONUNIFORM_
+using MyMeshMap = MeshMap<MyBlock>;
+using MyDensity = UniformDensity;
+#endif /* _NONUNIFORM_ */
+
+
+template<typename TKernel, typename TGrid>
+static void process(TKernel& kernel, TGrid& grid, Profiler& prof, const int rank)
+{
+    static size_t pass = 0;
+    SynchronizerMPI& Synch = grid.sync(kernel);
+
+    prof.push_start("Inner");
+    const vector<BlockInfo> avail0 = Synch.avail_inner();
+    prof.pop_stop();
+
+    prof.push_start("Halo");
+    const vector<BlockInfo> avail1 = Synch.avail_halo();
+    prof.pop_stop();
+
+    ++pass;
+    if (pass%20 == 0)
+        cout << "[rank=" << rank << "] Pass " << pass << ": nInner=" << avail0.size() << "; nHalo=" << avail1.size() << endl;
+}
+
+template <typename TBlock>
+struct Kernel
+{
+    StencilInfo stencil;
+
+    Kernel(ArgumentParser& parser)
+    {
+        stencil.sx = parser("sx").asInt(-1);
+        stencil.sy = parser("sy").asInt(-1);
+        stencil.sz = parser("sz").asInt(-1);
+        stencil.ex = parser("ex").asInt(1);
+        stencil.ey = parser("ey").asInt(1);
+        stencil.ez = parser("ez").asInt(1);
+        stencil.tensorial = parser("tensorial").asBool(false);
+        for (int i = 0; i < static_cast<int>(TBlock::members); ++i)
+            stencil.selcomponents.push_back(i);
+    }
+
+    Kernel(const Kernel& c) = default;
+    Kernel& operator=(const Kernel& c) = default;
+};
+
+using MyKernel = Kernel<MyBlock>;
+
+
+int main(int argc, char* argv[])
+{
+    MPI_Init(&argc, &argv);
+
+    ArgumentParser parser(argc, argv);
+
+    // blocks
+    const int bpdx = parser("bpdx").asInt(1);
+    const int bpdy = parser("bpdy").asInt(bpdx);
+    const int bpdz = parser("bpdz").asInt(bpdx);
+
+    // processes
+    const int ppdx = parser("ppdx").asInt(1);
+    const int ppdy = parser("ppdy").asInt(ppdx);
+    const int ppdz = parser("ppdz").asInt(ppdx);
+
+#ifdef _NONUNIFORM_
+    MyDensity mesh_density;
+    MyMeshMap* xmap = new MyMeshMap(0, 1, ppdx * bpdx);
+    MyMeshMap* ymap = new MyMeshMap(0, 1, ppdy * bpdy);
+    MyMeshMap* zmap = new MyMeshMap(0, 1, ppdz * bpdz);
+    xmap->init(&mesh_density);
+    ymap->init(&mesh_density);
+    zmap->init(&mesh_density);
+    MyGridMPI* grid = new MyGridMPI(xmap, ymap, zmap, ppdx, ppdy, ppdz, bpdx, bpdy, bpdz);
+#else
+    MyGridMPI* grid = new MyGridMPI(ppdx, ppdy, ppdz, bpdx, bpdy, bpdz);
+#endif /* _NONUNIFORM_ */
+
+    int myrank;
+    const MPI_Comm comm = grid->getCartComm();
+    MPI_Comm_rank(comm, &myrank);
+
+    set_grid_ic(grid, myrank);
+
+    const int nPasses = parser("passes").asInt(500);
+    MyKernel kernel(parser);
+    Profiler prof;
+    for (int i = 0; i < nPasses; ++i)
+        process(kernel, *grid, prof, myrank);
+
+    cout << std::flush;
+    MPI_Barrier(comm);
+    if (0==myrank)
+        prof.printSummary();
+
+    MPI_Finalize();
+    return 0;
+}