Commit 97a49114 authored by fabianw's avatar fabianw
Browse files

added parallel MPI I/O

parent ed7dc0c0
......@@ -18,8 +18,8 @@
struct Triangle
{
Real vertices[3][3];
};
float vertices[3][3];
} __attribute__((packed));
static const Real gVertexOffset[8][3] =
{
......@@ -532,7 +532,7 @@ void IsoExtractor<Tkernel,Tinterpolator>::_extractIso(const Real isoval,
std::chrono::time_point<std::chrono::system_clock> start, end;
start = std::chrono::system_clock::now();
if (verbose)
std::cout << "Extracting isovalue " << isoval << "..." << std::endl;
std::cout << "Extracting isovalue " << isoval << std::endl;
const size_t nCubes = static_cast<size_t>(Nx) * Ny * Nz;
size_t totalSize = 0;
......
......@@ -10,6 +10,12 @@
#include "IsoExtractor.h"
#include "InterpolatorMPI.h"
struct TriangleConnect
{
unsigned char n;
int v[3];
} __attribute__((packed));
template <typename Tkernel, template<typename> class Tinterpolator=InterpolatorMPI>
class IsoExtractorMPI : public IsoExtractor<Tkernel,Tinterpolator>
{
......@@ -109,7 +115,8 @@ private:
// write content
// _writePLY_binary_MPI(comm, tList);
_writePLY_binary_MPI_lazy(tList, basename+".ply");
// _writePLY_binary_MPI_lazy(tList, basename+".ply");
_writePLY_binary_MPI(comm, tList, basename+".ply");
}
void _writePLY_binary_MPI_lazy(const std::vector<Triangle>& tList, const std::string fname)
......@@ -164,28 +171,92 @@ private:
}
}
// void _writePLY_binary_MPI(const MPI_Comm comm, const std::vector<Triangle>& tList)
// {
// for (auto t : tList)
// {
// float tc[3][3];
// for (int j = 0; j < 3; ++j)
// {
// tc[j][0] = t.vertices[j][0];
// tc[j][1] = t.vertices[j][1];
// tc[j][2] = t.vertices[j][2];
// }
// ply.write((char*)&tc[0][0], 9*sizeof(float));
// }
// const char nVert = 3;
// for (int i = 0; i < tList.size()*3; i += 3)
// {
// int tv[3] = {i, i+1, i+2};
// ply.write(&nVert, sizeof(char));
// ply.write((char*)&tv[0], 3*sizeof(int));
// }
// }
void _writePLY_binary_MPI(const MPI_Comm comm, const std::vector<Triangle>& tList, const std::string fname)
{
int rank, size;
MPI_Status status;
MPI_File file_id;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
int rc = MPI_File_open(MPI_COMM_SELF, fname.c_str(), MPI_MODE_APPEND | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_id);
if (rc)
{
if (0 == rank)
std::cerr << "ERROR: IsoExtractorMPI: Can not open file." << std::endl;
abort();
}
std::vector<unsigned long long> allsize(size);
unsigned long long mytri = tList.size();
MPI_Allgather(&mytri, 1, MPI_UNSIGNED_LONG_LONG, allsize.data(), 1, MPI_UNSIGNED_LONG_LONG, comm);
// if (0 == rank)
// {
// for (const auto v: allsize)
// std::cout << v << std::endl;
// }
long long offset = 0;
MPI_File_get_position(file_id, &offset);
assert(sizeof(Triangle) == 9*sizeof(float));
for (int i = 0; i < rank; ++i)
offset = offset + allsize[i]*sizeof(Triangle);
Triangle* tarr = nullptr;
if (allsize[rank] > 0)
{
tarr = new Triangle[allsize[rank]];
// for (size_t i=0; i<allsize[rank]; ++i)
// memcpy(&tarr[0].vertices[0][0] + i*sizeof(Triangle), (char*)&tList[i].vertices[0][0], sizeof(Triangle));
for (size_t i=0; i<allsize[rank]; ++i)
{
for (int j = 0; j < 3; ++j)
{
tarr[i].vertices[j][0] = tList[i].vertices[j][0];
tarr[i].vertices[j][1] = tList[i].vertices[j][1];
tarr[i].vertices[j][2] = tList[i].vertices[j][2];
}
}
MPI_File_write_at(file_id, offset, (char *)&tarr[0].vertices[0][0], allsize[rank]*sizeof(Triangle), MPI_CHAR, &status);
}
offset = offset + allsize[rank]*sizeof(Triangle);
MPI_Barrier(comm);
MPI_Bcast(&offset, 1, MPI_LONG_LONG, size-1, comm);
if (tarr != nullptr)
delete [] tarr;
for (int i = 0; i < rank; ++i)
offset = offset + allsize[i]*sizeof(TriangleConnect);
TriangleConnect* tcarr = nullptr;
if (allsize[rank] > 0)
{
tcarr = new TriangleConnect[allsize[rank]];
int base = 0;
for (int i = 0; i < rank; ++i)
base += 3*allsize[i];
for (size_t i=0; i<allsize[rank]; ++i)
{
tcarr[i].n = 3;
tcarr[i].v[0] = base + 3*i + 0;
tcarr[i].v[1] = base + 3*i + 1;
tcarr[i].v[2] = base + 3*i + 2;
}
MPI_File_write_at(file_id, offset, (char *)&tcarr[0].n, allsize[rank]*sizeof(TriangleConnect), MPI_CHAR, &status);
}
MPI_Barrier(comm);
if (tcarr != nullptr)
delete [] tcarr;
MPI_File_close(&file_id);
}
};
#endif /* ISOEXTRACTORMPI_H_XABTB0WN */
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment