Commit 8e077ef6 authored by fabianw's avatar fabianw

added SynchronizerMPI test

parent 421c1658
// File : common.h
// Created : Sun Aug 12 2018 04:33:51 PM (+0200)
// Author : Fabian Wermelinger
// Description: Common stuff
// Copyright 2018 ETH Zurich. All Rights Reserved.
#ifndef COMMON_H_QLCQRKJP
#define COMMON_H_QLCQRKJP
#ifdef _DOUBLE_
using MyReal = double;
#else
using MyReal = float;
#define _FLOAT_PRECISION_
#endif /* _DOUBLE_ */
#define _BLOCKSIZE_ 16
#define _BLOCKSIZEX_ _BLOCKSIZE_
#define _BLOCKSIZEY_ _BLOCKSIZE_
#define _BLOCKSIZEZ_ _BLOCKSIZE_
#include <cassert>
#include <mpi.h>
#include <cstring>
#include <vector>
#include <string>
#include <sstream>
#include "BlockInfo.h"
template <typename TReal, size_t _AOSmembers=1>
struct Block
{
typedef TReal ElementType;
typedef TReal element_type;
static const size_t sizeX = _BLOCKSIZE_;
static const size_t sizeY = _BLOCKSIZE_;
static const size_t sizeZ = _BLOCKSIZE_;
static const size_t members = _AOSmembers;
inline void clear() { memset(&m_data[0][0][0][0], 0, _BLOCKSIZE_*_BLOCKSIZE_*_BLOCKSIZE_*_AOSmembers*sizeof(TReal)); }
inline const TReal (&operator()(const size_t ix, const size_t iy, const size_t iz) const)[_AOSmembers]
{
assert(ix<_BLOCKSIZE_);
assert(iy<_BLOCKSIZE_);
assert(iz<_BLOCKSIZE_);
return this->m_data[iz][iy][ix];
}
inline TReal (&operator()(const size_t ix, const size_t iy, const size_t iz))[_AOSmembers]
{
assert(ix<_BLOCKSIZE_);
assert(iy<_BLOCKSIZE_);
assert(iz<_BLOCKSIZE_);
return this->m_data[iz][iy][ix];
}
TReal m_data[_BLOCKSIZE_][_BLOCKSIZE_][_BLOCKSIZE_][_AOSmembers];
};
template <size_t _comp=0>
struct Streamer
{
static const int NCHANNELS = 1;
template <typename TBlock, typename T>
static inline void operate(const TBlock& b, const int ix, const int iy, const int iz, T output[NCHANNELS])
{
assert(_comp<TBlock::members);
output[0] = b(ix,iy,iz)[_comp];
}
static const char * getAttributeName() { return "Scalar"; }
};
template <typename TGrid>
void set_grid_ic(TGrid* grid, const int myrank=0)
{
using TBlock = typename TGrid::BlockType;
std::vector<BlockInfo> infos = grid->getResidentBlocksInfo();
const int NX = grid->getResidentBlocksPerDimension(0);
const int NY = grid->getResidentBlocksPerDimension(1);
const int NZ = grid->getResidentBlocksPerDimension(2);
const double blocksize = TBlock::members*TBlock::sizeX*TBlock::sizeY*TBlock::sizeZ;
const double offset = myrank * NX*NY*NZ * blocksize;
#pragma omp parallel for
for (size_t i = 0; i < infos.size(); ++i)
{
BlockInfo info = infos[i];
TBlock& b = *(TBlock*)info.ptrBlock;
for (size_t iz = 0; iz < TBlock::sizeZ; ++iz)
for (size_t iy = 0; iy < TBlock::sizeY; ++iy)
for (size_t ix = 0; ix < TBlock::sizeX; ++ix)
for (size_t m = 0; m < TBlock::members; ++m)
b(ix,iy,iz)[m] = offset + info.blockID*blocksize + m + TBlock::members*(ix + (double)TBlock::sizeX*(iy + TBlock::sizeY*iz));
}
}
#endif /* COMMON_H_QLCQRKJP */
......@@ -8,13 +8,13 @@ nonuniform ?= no
extra ?=
CFLAGS = -fopenmp -I../../source
CFLAGS = -fopenmp -g -O2 -I../../source
CFLAGS+= $(extra)
# CFLAGS += -I${HDF5_ROOT}/include
# CFLAGS+= -Wall -Wextra -Wfloat-equal -Wundef -Wcast-align
# CFLAGS+= -Wwrite-strings -Wmissing-declarations -Wredundant-decls
CFLAGS+= -Wall -Wextra -Wfloat-equal -Wundef -Wcast-align
CFLAGS+= -Wwrite-strings -Wmissing-declarations -Wredundant-decls
# CFLAGS+= -Wlogical-op
# CFLAGS+= -Wshadow -Woverloaded-virtual -Wuninitialized -pedantic
CFLAGS+= -Wshadow -Woverloaded-virtual -Wuninitialized
CFLAGS+= -Wpedantic -Wno-unused-parameter # does not make much sense with function overloading
......
......@@ -3,36 +3,12 @@
// Author : Fabian Wermelinger
// Description: Test Cubism dumping facilities
// Copyright 2018 ETH Zurich. All Rights Reserved.
#ifdef _DOUBLE_
using MyReal = double;
#else
using MyReal = float;
#define _FLOAT_PRECISION_
#endif /* _DOUBLE_ */
#define _BLOCKSIZE_ 16
#define _BLOCKSIZEX_ _BLOCKSIZE_
#define _BLOCKSIZEY_ _BLOCKSIZE_
#define _BLOCKSIZEZ_ _BLOCKSIZE_
#include <mpi.h>
#include <cstring>
#include <vector>
#include <string>
#include <sstream>
using namespace std;
#include "../common.h"
#include "ArgumentParser.h"
#include "Grid.h"
#include "GridMPI.h"
#include "BlockInfo.h"
// dumpers
#ifndef _HDF5_DOUBLE_PRECISION_
const string prec_string = "4byte";
#else
const string prec_string = "8byte";
#endif
#include "MeshMap.h"
#define _USE_HDF_
#include "HDF5Dumper.h"
......@@ -49,34 +25,17 @@ const string prec_string = "8byte";
#include "PlainBinDumper_MPI.h"
using namespace std;
template <typename TReal>
struct Block
{
static const size_t sizeX = _BLOCKSIZE_;
static const size_t sizeY = _BLOCKSIZE_;
static const size_t sizeZ = _BLOCKSIZE_;
typedef TReal ElementType;
TReal data[_BLOCKSIZE_][_BLOCKSIZE_][_BLOCKSIZE_];
inline void clear() { memset(&data[0][0][0], 0, _BLOCKSIZE_*_BLOCKSIZE_*_BLOCKSIZE_*sizeof(TReal)); }
};
struct MyStreamer
{
static const int NCHANNELS = 1;
template <typename TBlock, typename T>
static inline void operate(const TBlock& b, const int ix, const int iy, const int iz, T output[NCHANNELS])
{
output[0] = b.data[iz][iy][ix];
}
static const char * getAttributeName() { return "Scalar"; }
};
// dumpers
#ifndef _HDF5_DOUBLE_PRECISION_
const string prec_string = "4byte";
#else
const string prec_string = "8byte";
#endif
using MyBlock = Block<MyReal>;
using MyBlock = Block<MyReal,1>;
using MyStreamer = Streamer<0>;
using MyGrid = Grid<MyBlock>;
using MyGridMPI = GridMPI<MyGrid>;
using MySlice = typename SliceTypes::Slice<MyGrid>;
......@@ -89,30 +48,6 @@ using MyDensity = RandomDensity;
#endif /* _NONUNIFORM_ */
static void set_grid_ic(MyGridMPI* grid, const int myrank=0)
{
vector<BlockInfo> infos = grid->getResidentBlocksInfo();
const int NX = grid->getResidentBlocksPerDimension(0);
const int NY = grid->getResidentBlocksPerDimension(1);
const int NZ = grid->getResidentBlocksPerDimension(2);
const double blocksize = MyBlock::sizeX*MyBlock::sizeY*MyBlock::sizeZ;
const double offset = myrank * NX*NY*NZ * blocksize;
#pragma omp parallel for
for (size_t i = 0; i < infos.size(); ++i)
{
BlockInfo info = infos[i];
MyBlock& b = *(MyBlock*)info.ptrBlock;
for (size_t iz = 0; iz < MyBlock::sizeZ; ++iz)
for (size_t iy = 0; iy < MyBlock::sizeY; ++iy)
for (size_t ix = 0; ix < MyBlock::sizeX; ++ix)
b.data[iz][iy][ix] = offset + info.blockID*blocksize + ix + (double)MyBlock::sizeX*(iy + MyBlock::sizeY*iz);
}
}
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
......
SHELL := /bin/bash
CC = mpic++
precision ?= single
nonuniform ?= no
extra ?=
CFLAGS = -fopenmp -g -O2 -I../../source
CFLAGS+= $(extra)
# CFLAGS += -I${HDF5_ROOT}/include
CFLAGS+= -Wall -Wextra -Wfloat-equal -Wundef -Wcast-align
CFLAGS+= -Wwrite-strings -Wmissing-declarations -Wredundant-decls
# CFLAGS+= -Wlogical-op
CFLAGS+= -Wshadow -Woverloaded-virtual -Wuninitialized
CFLAGS+= -Wpedantic -Wno-unused-parameter # does not make much sense with function overloading
.PHONY: all clean
$(warning using precision=$(precision))
$(warning using nonuniform=$(nonuniform))
ifeq ("$(precision)","double")
CFLAGS += -D_DOUBLE_
endif
ifeq ("$(nonuniform)","yes")
CFLAGS += -D_NONUNIFORM_
endif
all: testSynchronizerMPI.cpp
$(CC) $(CFLAGS) -o testSynchronizerMPI testSynchronizerMPI.cpp
clean:
rm -f testSynchronizerMPI *.o
// File : testSynchronizerMPI.cpp
// Created : Sun Aug 12 2018 04:25:45 PM (+0200)
// Author : Fabian Wermelinger
// Description: Test program for MPI stencil synchronizer
// Copyright 2018 ETH Zurich. All Rights Reserved.
#include <iostream>
#include "../common.h"
#include "ArgumentParser.h"
#include "Grid.h"
#include "GridMPI.h"
#include "MeshMap.h"
#include "SynchronizerMPI.h"
#include "StencilInfo.h"
#include "Profiler.h"
using namespace std;
using MyBlock = Block<MyReal,32>;
using MyGrid = Grid<MyBlock>;
using MyGridMPI = GridMPI<MyGrid>;
#ifdef _NONUNIFORM_
using MyMeshMap = MeshMap<MyBlock>;
using MyDensity = UniformDensity;
#endif /* _NONUNIFORM_ */
template<typename TKernel, typename TGrid>
static void process(TKernel& kernel, TGrid& grid, Profiler& prof, const int rank)
{
static size_t pass = 0;
SynchronizerMPI& Synch = grid.sync(kernel);
prof.push_start("Inner");
const vector<BlockInfo> avail0 = Synch.avail_inner();
prof.pop_stop();
prof.push_start("Halo");
const vector<BlockInfo> avail1 = Synch.avail_halo();
prof.pop_stop();
++pass;
if (pass%20 == 0)
cout << "[rank=" << rank << "] Pass " << pass << ": nInner=" << avail0.size() << "; nHalo=" << avail1.size() << endl;
}
template <typename TBlock>
struct Kernel
{
StencilInfo stencil;
Kernel(ArgumentParser& parser)
{
stencil.sx = parser("sx").asInt(-1);
stencil.sy = parser("sy").asInt(-1);
stencil.sz = parser("sz").asInt(-1);
stencil.ex = parser("ex").asInt(1);
stencil.ey = parser("ey").asInt(1);
stencil.ez = parser("ez").asInt(1);
stencil.tensorial = parser("tensorial").asBool(false);
for (int i = 0; i < static_cast<int>(TBlock::members); ++i)
stencil.selcomponents.push_back(i);
}
Kernel(const Kernel& c) = default;
Kernel& operator=(const Kernel& c) = default;
};
using MyKernel = Kernel<MyBlock>;
int main(int argc, char* argv[])
{
MPI_Init(&argc, &argv);
ArgumentParser parser(argc, argv);
// blocks
const int bpdx = parser("bpdx").asInt(1);
const int bpdy = parser("bpdy").asInt(bpdx);
const int bpdz = parser("bpdz").asInt(bpdx);
// processes
const int ppdx = parser("ppdx").asInt(1);
const int ppdy = parser("ppdy").asInt(ppdx);
const int ppdz = parser("ppdz").asInt(ppdx);
#ifdef _NONUNIFORM_
MyDensity mesh_density;
MyMeshMap* xmap = new MyMeshMap(0, 1, ppdx * bpdx);
MyMeshMap* ymap = new MyMeshMap(0, 1, ppdy * bpdy);
MyMeshMap* zmap = new MyMeshMap(0, 1, ppdz * bpdz);
xmap->init(&mesh_density);
ymap->init(&mesh_density);
zmap->init(&mesh_density);
MyGridMPI* grid = new MyGridMPI(xmap, ymap, zmap, ppdx, ppdy, ppdz, bpdx, bpdy, bpdz);
#else
MyGridMPI* grid = new MyGridMPI(ppdx, ppdy, ppdz, bpdx, bpdy, bpdz);
#endif /* _NONUNIFORM_ */
int myrank;
const MPI_Comm comm = grid->getCartComm();
MPI_Comm_rank(comm, &myrank);
set_grid_ic(grid, myrank);
const int nPasses = parser("passes").asInt(500);
MyKernel kernel(parser);
Profiler prof;
for (int i = 0; i < nPasses; ++i)
process(kernel, *grid, prof, myrank);
cout << std::flush;
MPI_Barrier(comm);
if (0==myrank)
prof.printSummary();
MPI_Finalize();
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment