Skip to content

GitLab

C
Cubism
Commit f9e21938 authored by fabianw's avatar fabianw
Browse files
Options

ZBinDumper/MPI: fix wrong func signature -- eliminate narrowing conversions 

Re-indented code
parent 0f9fe82f
Hide whitespace changes
Inline Side-by-side
Showing with 256 additions and 200 deletions
include/Cubism/LosslessCompression.h
View file @ f9e21938
......@@ -44,7 +44,8 @@ inline size_t ZZdecompress(unsigned char * inputbuf, size_t ninputbytes, int lay
}
template <typename Real>
inline size_t ZZcompress(unsigned char *buf, unsigned len, int layout[4], unsigned *max)
inline size_t
ZZcompress(unsigned char *buf, size_t len, int layout[4], size_t *max)
{
#if 1
int zbufsize = 0;
......
include/Cubism/ZBinDumper.h
View file @ f9e21938
......@@ -9,26 +9,25 @@
#pragma once
#include <iostream>
#include <vector>
#include <string>
#include <stdio.h>
#include <sstream>
#include <stdio.h>
#include <string>
#include <vector>
#include "BlockInfo.h"
#include "LosslessCompression.h"
CUBISM_NAMESPACE_BEGIN
typedef struct _header_serial
{
typedef struct _header_serial {
long size[8];
} header_serial;
// The following requirements for the data TStreamer are required:
// TStreamer::NCHANNELS : Number of data elements (1=Scalar, 3=Vector, 9=Tensor)
// TStreamer::operate : Data access methods for read and write
// TStreamer::getAttributeName : Attribute name of the date ("Scalar", "Vector", "Tensor")
// TStreamer::NCHANNELS: Number of data elements (1=Scalar, 3=Vector, 9=Tensor)
// TStreamer::operate: Data access methods for read and write
// TStreamer::getAttributeName: Attribute name of the date ("Scalar", "Vector",
// "Tensor")
template <typename TStreamer, typename TGrid>
void DumpZBin(const TGrid &grid,
const int iCounter,
......@@ -43,76 +42,81 @@ void DumpZBin(const TGrid &grid,
std::ostringstream filename;
filename << dump_path << "/" << f_name;
FILE *file_id;
int status;
FILE *file_id;
int status;
static const unsigned int NCHANNELS = TStreamer::NCHANNELS;
const unsigned int NX = grid.getBlocksPerDimension(0)*B::sizeX;
const unsigned int NY = grid.getBlocksPerDimension(1)*B::sizeY;
const unsigned int NZ = grid.getBlocksPerDimension(2)*B::sizeZ;
static const int NCHANNELS = TStreamer::NCHANNELS;
const int NX = grid.getBlocksPerDimension(0) * B::sizeX;
const int NY = grid.getBlocksPerDimension(1) * B::sizeY;
const int NZ = grid.getBlocksPerDimension(2) * B::sizeZ;
Real memsize = (NX * NY * NZ * sizeof(Real))/(1024.*1024.*1024.);
Real memsize = (NX * NY * NZ * sizeof(Real)) / (1024. * 1024. * 1024.);
std::cout << "Allocating " << memsize << " GB of BIN data" << std::endl;
Real * array_all = new Real[NX * NY * NZ];
Real *array_all = new Real[NX * NY * NZ];
std::vector<BlockInfo> vInfo_local = grid.getBlocksInfo();
static const unsigned int sX = 0;
static const unsigned int sY = 0;
static const unsigned int sZ = 0;
static const int sX = 0;
static const int sY = 0;
static const int sZ = 0;
static const unsigned int eX = B::sizeX;
static const unsigned int eY = B::sizeY;
static const unsigned int eZ = B::sizeZ;
static const int eX = B::sizeX;
static const int eY = B::sizeY;
static const int eZ = B::sizeZ;
file_id = fopen((filename.str()+".zbin").c_str(), "w");
file_id = fopen((filename.str() + ".zbin").c_str(), "w");
header_serial tag;
fseek(file_id, sizeof(tag), SEEK_SET);
for (unsigned int ichannel = 0; ichannel < NCHANNELS; ichannel++)
{
for (int ichannel = 0; ichannel < NCHANNELS; ichannel++) {
#pragma omp parallel for
for(unsigned int i=0; i<vInfo_local.size(); i++)
{
BlockInfo& info = vInfo_local[i];
const unsigned int idx[3] = {info.index[0], info.index[1], info.index[2]};
B & b = *(B*)info.ptrBlock;
for(unsigned int ix=sX; ix<eX; ix++)
{
const unsigned int gx = idx[0]*B::sizeX + ix;
for(unsigned int iy=sY; iy<eY; iy++)
{
const unsigned int gy = idx[1]*B::sizeY + iy;
for(unsigned int iz=sZ; iz<eZ; iz++)
{
const unsigned int gz = idx[2]*B::sizeZ + iz;
for (int i = 0; i < static_cast<int>(vInfo_local.size()); i++) {
BlockInfo &info = vInfo_local[i];
const int idx[3] = {info.index[0], info.index[1], info.index[2]};
B &b = *(B *)info.ptrBlock;
for (int ix = sX; ix < eX; ix++) {
const int gx = idx[0] * B::sizeX + ix;
for (int iy = sY; iy < eY; iy++) {
const int gy = idx[1] * B::sizeY + iy;
for (int iz = sZ; iz < eZ; iz++) {
const int gz = idx[2] * B::sizeZ + iz;
assert((gz + NZ * (gy + NY * gx)) < NX * NY * NZ);
Real * const ptr = array_all + (gz + NZ * (gy + NY * gx));
Real *const ptr =
array_all + (gz + NZ * (gy + NY * gx));
Real output;
TStreamer::operate(b, ix, iy, iz, &output, ichannel); // point -> output,
TStreamer::operate(b,
ix,
iy,
iz,
&output,
ichannel); // point -> output,
ptr[0] = output;
}
}
}
}
// long local_count = NX * NY * NZ * NCHANNELS;
long local_count = NX * NY * NZ * 1;
long local_bytes = local_count * sizeof(Real);
// size_t local_count = NX * NY * NZ * NCHANNELS;
size_t local_count = NX * NY * NZ * 1;
size_t local_bytes = local_count * sizeof(Real);
unsigned int max = local_bytes;
size_t max = local_bytes;
// int layout[4] = {NCHANNELS, NX, NY, NZ};
int layout[4] = {NX, NY, NZ, 1};
long compressed_bytes = ZZcompress<typename TGrid::Real>((unsigned char *)array_all, local_bytes, layout, &max); // "in place"
long compressed_bytes =
ZZcompress<typename TGrid::Real>((unsigned char *)array_all,
local_bytes,
layout,
&max); // "in place"
printf("Writing %ld bytes of Compressed data (cr = %.2f)\n", compressed_bytes, NX*NY*NZ*sizeof(Real)*NCHANNELS*1.0/compressed_bytes);
printf("Writing %ld bytes of Compressed data (cr = %.2f)\n",
compressed_bytes,
NX * NY * NZ * sizeof(Real) * NCHANNELS * 1.0 /
compressed_bytes);
tag.size[ichannel] = compressed_bytes;
size_t wb_data = fwrite(array_all, 1, compressed_bytes, file_id);
......@@ -123,15 +127,16 @@ void DumpZBin(const TGrid &grid,
status = fclose(file_id);
delete [] array_all;
delete[] array_all;
}
template <typename TStreamer, typename TGrid>
void ReadZBin(TGrid &grid, const std::string& f_name, const std::string& read_path=".")
void ReadZBin(TGrid &grid,
const std::string &f_name,
const std::string &read_path = ".")
{
typedef typename TGrid::BlockType B;
typedef typename TGrid::Real Real;
typedef typename TGrid::Real Real;
// f_name is the base filename without file type extension
std::ostringstream filename;
......@@ -140,12 +145,12 @@ void ReadZBin(TGrid &grid, const std::string& f_name, const std::string& read_pa
int status;
FILE *file_id;
const int NX = grid.getBlocksPerDimension(0)*B::sizeX;
const int NY = grid.getBlocksPerDimension(1)*B::sizeY;
const int NZ = grid.getBlocksPerDimension(2)*B::sizeZ;
const int NX = grid.getBlocksPerDimension(0) * B::sizeX;
const int NY = grid.getBlocksPerDimension(1) * B::sizeY;
const int NZ = grid.getBlocksPerDimension(2) * B::sizeZ;
static const int NCHANNELS = TStreamer::NCHANNELS;
Real * array_all = new Real[NX * NY * NZ * NCHANNELS];
Real *array_all = new Real[NX * NY * NZ * NCHANNELS];
std::vector<BlockInfo> vInfo_local = grid.getBlocksInfo();
......@@ -157,10 +162,10 @@ void ReadZBin(TGrid &grid, const std::string& f_name, const std::string& read_pa
const int eY = B::sizeY;
const int eZ = B::sizeZ;
file_id = fopen((filename.str()+".zbin").c_str(), "rb");
file_id = fopen((filename.str() + ".zbin").c_str(), "rb");
long local_count = NX * NY * NZ * 1;
long local_bytes = local_count * sizeof(Real);
size_t local_count = NX * NY * NZ * 1;
size_t local_bytes = local_count * sizeof(Real);
header_serial tag;
size_t rb_header = fread(&tag.size[0], 1, sizeof(tag), file_id);
......@@ -172,33 +177,40 @@ void ReadZBin(TGrid &grid, const std::string& f_name, const std::string& read_pa
}
#endif
for (unsigned int ichannel = 0; ichannel < NCHANNELS; ichannel++)
{
for (unsigned int ichannel = 0; ichannel < NCHANNELS; ichannel++) {
#if DBG
printf("compr. size = %ld\n", tag.size[ichannel]); fflush(0);
printf("compr. size = %ld\n", tag.size[ichannel]);
fflush(0);
#endif
long compressed_bytes = tag.size[ichannel];
size_t compressed_bytes = tag.size[ichannel];
#if DBG
printf("Reading %ld bytes of Compressed data (cr = %.2f)\n", compressed_bytes, local_bytes*1.0/compressed_bytes);
printf("Reading %ld bytes of Compressed data (cr = %.2f)\n",
compressed_bytes,
local_bytes * 1.0 / compressed_bytes);
#endif
unsigned char *tmp = (unsigned char *) malloc(compressed_bytes+4096);
unsigned char *tmp = (unsigned char *)malloc(compressed_bytes + 4096);
size_t rb_data = fread(tmp, 1, compressed_bytes, file_id);
int layout[4] = {NX, NY, NZ, 1};
size_t decompressed_bytes = ZZdecompress<typename TGrid::Real>(tmp, compressed_bytes, layout, (unsigned char *)array_all, local_bytes);
size_t decompressed_bytes =
ZZdecompress<typename TGrid::Real>(tmp,
compressed_bytes,
layout,
(unsigned char *)array_all,
local_bytes);
free(tmp);
#if DBG
printf("size = %ld (%ld)\n", decompressed_bytes, local_bytes); fflush(0);
printf("size = %ld (%ld)\n", decompressed_bytes, local_bytes);
fflush(0);
#endif
#pragma omp parallel for
for(int i=0; i<vInfo_local.size(); i++)
{
BlockInfo& info = vInfo_local[i];
for (int i = 0; i < static_cast<int>(vInfo_local.size()); i++) {
BlockInfo &info = vInfo_local[i];
const int idx[3] = {info.index[0], info.index[1], info.index[2]};
B & b = *(B*)info.ptrBlock;
B &b = *(B *)info.ptrBlock;
for(int ix=sX; ix<eX; ix++)
for(int iy=sY; iy<eY; iy++)
......
include/Cubism/ZBinDumper_MPI.h
View file @ f9e21938
......@@ -9,39 +9,37 @@
#pragma once
#include <iostream>
#include <cstdio>
#include <vector>
#include <string>
#include <cassert>
#include <cstdio>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "BlockInfo.h"
#include "LosslessCompression.h"
CUBISM_NAMESPACE_BEGIN
#define MAX_MPI_PROCS (16*1024) // header: 0.25MB* 8
#define MAX_MPI_PROCS (16 * 1024) // header: 0.25MB* 8
typedef struct _header
{
long offset[8]; // 1 for single compression, NCHANNELS otherwise
typedef struct _header {
long offset[8]; // 1 for single compression, NCHANNELS otherwise
long size[8];
} header;
// The following requirements for the data TStreamer are required:
// TStreamer::NCHANNELS : Number of data elements (1=Scalar, 3=Vector, 9=Tensor)
// TStreamer::operate : Data access methods for read and write
// TStreamer::getAttributeName : Attribute name of the date ("Scalar", "Vector", "Tensor")
template<typename TStreamer, typename TGrid>
void DumpZBin_MPI(
const TGrid &grid,
const int iCounter,
const typename TGrid::Real t,
const std::string &f_name,
const std::string &dump_path = ".",
const bool bDummy = false)
// TStreamer::NCHANNELS: Number of data elements (1=Scalar, 3=Vector, 9=Tensor)
// TStreamer::operate: Data access methods for read and write
// TStreamer::getAttributeName: Attribute name of the date ("Scalar", "Vector",
// "Tensor")
template <typename TStreamer, typename TGrid>
void DumpZBin_MPI(const TGrid &grid,
const int iCounter,
const typename TGrid::Real t,
const std::string &f_name,
const std::string &dump_path = ".",
const bool bDummy = false)
{
typedef typename TGrid::BlockType B;
......@@ -61,116 +59,143 @@ void DumpZBin_MPI(
int coords[3];
grid.peindex(coords);
const unsigned int NX = grid.getResidentBlocksPerDimension(0)*B::sizeX;
const unsigned int NY = grid.getResidentBlocksPerDimension(1)*B::sizeY;
const unsigned int NZ = grid.getResidentBlocksPerDimension(2)*B::sizeZ;
static const unsigned int NCHANNELS = TStreamer::NCHANNELS;
const int NX = grid.getResidentBlocksPerDimension(0) * B::sizeX;
const int NY = grid.getResidentBlocksPerDimension(1) * B::sizeY;
const int NZ = grid.getResidentBlocksPerDimension(2) * B::sizeZ;
static const int NCHANNELS = TStreamer::NCHANNELS;
if (rank==0)
{
//Real memsize = (NX * NY * NZ * NCHANNELS * sizeof(Real))/(1024.*1024.*1024.);
Real memsize = (NX * NY * NZ * sizeof(Real))/(1024.*1024.*1024.);
std::cout << "Allocating " << memsize << " GB of BIN data per rank (" << memsize*nranks << " GB in total)" << std::endl;
if (rank == 0) {
// Real memsize = (NX * NY * NZ * NCHANNELS *
// sizeof(Real))/(1024.*1024.*1024.);
Real memsize = (NX * NY * NZ * sizeof(Real)) / (1024. * 1024. * 1024.);
std::cout << "Allocating " << memsize << " GB of BIN data per rank ("
<< memsize * nranks << " GB in total)" << std::endl;
}
// Real * array_all = new Real[NX * NY * NZ * NCHANNELS];
Real * array_all = new Real[NX * NY * NZ];
// Real * array_all = new Real[NX * NY * NZ * NCHANNELS];
Real *array_all = new Real[NX * NY * NZ];
std::vector<BlockInfo> vInfo_local = grid.getResidentBlocksInfo();
static const unsigned int sX = 0;
static const unsigned int sY = 0;
static const unsigned int sZ = 0;
static const int sX = 0;
static const int sY = 0;
static const int sZ = 0;
static const unsigned int eX = B::sizeX;
static const unsigned int eY = B::sizeY;
static const unsigned int eZ = B::sizeZ;
static const int eX = B::sizeX;
static const int eY = B::sizeY;
static const int eZ = B::sizeZ;
int rc = MPI_File_open( MPI_COMM_SELF, const_cast<char*>( (filename.str()+".zbin").c_str() ), MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_id );
int rc =
MPI_File_open(MPI_COMM_SELF,
const_cast<char *>((filename.str() + ".zbin").c_str()),
MPI_MODE_CREATE | MPI_MODE_WRONLY,
MPI_INFO_NULL,
&file_id);
if (rc) {
printf("Unable to create ZBIN file\n");
exit(1);
}
long previous_offset = 0;
header tag; // moved here
for (unsigned int ichannel = 0; ichannel < NCHANNELS; ichannel++)
{
header tag; // moved here
for (int ichannel = 0; ichannel < NCHANNELS; ichannel++) {
#pragma omp parallel for
for(unsigned int i=0; i<vInfo_local.size(); i++)
{
BlockInfo& info = vInfo_local[i];
const unsigned int idx[3] = {info.index[0], info.index[1], info.index[2]};
B & b = *(B*)info.ptrBlock;
for(unsigned int ix=sX; ix<eX; ix++)
{
const unsigned int gx = idx[0]*B::sizeX + ix;
for(unsigned int iy=sY; iy<eY; iy++)
{
const unsigned int gy = idx[1]*B::sizeY + iy;
for(unsigned int iz=sZ; iz<eZ; iz++)
{
const unsigned int gz = idx[2]*B::sizeZ + iz;
assert((gz + NZ * (gy + NY * gx)) < NX * NY * NZ);
Real * const ptr = array_all + (gz + NZ * (gy + NY * gx));
Real output;
TStreamer::operate(b, ix, iy, iz, &output, ichannel); // point -> output,
ptr[0] = output;
for (int i = 0; i < static_cast<int>(vInfo_local.size()); i++) {
BlockInfo &info = vInfo_local[i];
const int idx[3] = {info.index[0], info.index[1], info.index[2]};
B &b = *(B *)info.ptrBlock;
for (int ix = sX; ix < eX; ix++) {
const int gx = idx[0] * B::sizeX + ix;
for (int iy = sY; iy < eY; iy++) {
const int gy = idx[1] * B::sizeY + iy;
for (int iz = sZ; iz < eZ; iz++) {
const int gz = idx[2] * B::sizeZ + iz;
assert((gz + NZ * (gy + NY * gx)) < NX * NY * NZ);
Real *const ptr =
array_all + (gz + NZ * (gy + NY * gx));
Real output;
TStreamer::operate(b,
ix,
iy,
iz,
&output,
ichannel); // point -> output,
ptr[0] = output;
}
}
}
}
}
// long local_count = NX * NY * NZ * NCHANNELS;
long local_count = NX * NY * NZ * 1;
long local_bytes = local_count * sizeof(Real);
long offset; // global offset
unsigned int max = local_bytes;
// int layout[4] = {NCHANNELS, NX, NY, NZ};
int layout[4] = {NX, NY, NZ, 1};
long compressed_bytes = ZZcompress<typename TGrid::Real>((unsigned char *)array_all, local_bytes, layout, &max); // "in place"
// size_t local_count = NX * NY * NZ * NCHANNELS;
size_t local_count = NX * NY * NZ * 1;
size_t local_bytes = local_count * sizeof(Real);
long offset; // global offset
size_t max = local_bytes;
// int layout[4] = {NCHANNELS, NX, NY, NZ};
int layout[4] = {NX, NY, NZ, 1};
long compressed_bytes =
ZZcompress<typename TGrid::Real>((unsigned char *)array_all,
local_bytes,
layout,
&max); // "in place"
#if DBG
printf("Writing %ld bytes of Compressed data (cr = %.2f)\n", compressed_bytes, local_bytes*1.0/compressed_bytes);
printf("Writing %ld bytes of Compressed data (cr = %.2f)\n",
compressed_bytes,
local_bytes * 1.0 / compressed_bytes);
#endif
MPI_Exscan( &compressed_bytes, &offset, 1, MPI_LONG, MPI_SUM, comm);
MPI_Exscan(&compressed_bytes, &offset, 1, MPI_LONG, MPI_SUM, comm);
if (rank == 0) offset = 0;
if (rank == 0)
offset = 0;
#if DBG
printf("rank %d, offset = %ld, size = %ld\n", rank, offset, compressed_bytes); fflush(0);
printf("rank %d, offset = %ld, size = %ld\n",
rank,
offset,
compressed_bytes);
fflush(0);
#endif
// header tag;
tag.offset[ichannel] = offset + previous_offset;
tag.size[ichannel] = compressed_bytes;
// header tag;
tag.offset[ichannel] = offset + previous_offset;
tag.size[ichannel] = compressed_bytes;
#if DBG
printf("rank %d, offset = %ld, size = %ld\n", rank, tag.offset[ichannel], tag.size[ichannel]); fflush(0);
printf("rank %d, offset = %ld, size = %ld\n",
rank,
tag.offset[ichannel],
tag.size[ichannel]);
fflush(0);
#endif
previous_offset = (tag.offset[ichannel] + tag.size[ichannel]);
MPI_Bcast(&previous_offset, 1, MPI_LONG, nranks-1, comm);
previous_offset = (tag.offset[ichannel] + tag.size[ichannel]);
MPI_Bcast(&previous_offset, 1, MPI_LONG, nranks - 1, comm);
long base = MAX_MPI_PROCS*sizeof(tag); // full Header
long base = MAX_MPI_PROCS * sizeof(tag); // full Header
MPI_File_write_at(file_id, base + tag.offset[ichannel], (char *)array_all, tag.size[ichannel], MPI_CHAR, &status);
MPI_File_write_at(file_id,
base + tag.offset[ichannel],
(char *)array_all,
tag.size[ichannel],
MPI_CHAR,
&status);
} /* ichannel */
} /* ichannel */
MPI_File_write_at(file_id, rank*sizeof(tag), &tag, 2*8, MPI_LONG, &status);
MPI_File_write_at(
file_id, rank * sizeof(tag), &tag, 2 * 8, MPI_LONG, &status);
MPI_File_close(&file_id);
delete [] array_all;
delete[] array_all;
}
template<typename TStreamer, typename TGrid>
void ReadZBin_MPI(TGrid &grid, const std::string& f_name, const std::string& read_path=".")
template <typename TStreamer, typename TGrid>
void ReadZBin_MPI(TGrid &grid,
const std::string &f_name,
const std::string &read_path = ".")
{
typedef typename TGrid::BlockType B;
......@@ -190,12 +215,12 @@ void ReadZBin_MPI(TGrid &grid, const std::string& f_name, const std::string& rea
int coords[3];
grid.peindex(coords);
const int NX = grid.getResidentBlocksPerDimension(0)*B::sizeX;
const int NY = grid.getResidentBlocksPerDimension(1)*B::sizeY;
const int NZ = grid.getResidentBlocksPerDimension(2)*B::sizeZ;
const int NX = grid.getResidentBlocksPerDimension(0) * B::sizeX;
const int NY = grid.getResidentBlocksPerDimension(1) * B::sizeY;
const int NZ = grid.getResidentBlocksPerDimension(2) * B::sizeZ;
static const int NCHANNELS = TStreamer::NCHANNELS;
Real * array_all = new Real[NX * NY * NZ * NCHANNELS];
Real *array_all = new Real[NX * NY * NZ * NCHANNELS];
std::vector<BlockInfo> vInfo_local = grid.getResidentBlocksInfo();
......@@ -207,20 +232,25 @@ void ReadZBin_MPI(TGrid &grid, const std::string& f_name, const std::string& rea
const int eY = B::sizeY;
const int eZ = B::sizeZ;
int rc = MPI_File_open( MPI_COMM_SELF, const_cast<char*>( (filename.str()+".zbin").c_str() ), MPI_MODE_RDONLY, MPI_INFO_NULL, &file_id );
int rc =
MPI_File_open(MPI_COMM_SELF,
const_cast<char *>((filename.str() + ".zbin").c_str()),
MPI_MODE_RDONLY,
MPI_INFO_NULL,
&file_id);
if (rc) {
printf("Unable to read ZBIN file\n");
exit(1);
}
// long local_count = NX * NY * NZ * NCHANNELS;
long local_count = NX * NY * NZ * 1;
long local_bytes = local_count * sizeof(Real);
// size_t local_count = NX * NY * NZ * NCHANNELS;
size_t local_count = NX * NY * NZ * 1;
size_t local_bytes = local_count * sizeof(Real);
long offset;
header tag;
MPI_File_read_at(file_id, rank*sizeof(tag), &tag, 2*8, MPI_LONG, &status);
MPI_File_read_at(
file_id, rank * sizeof(tag), &tag, 2 * 8, MPI_LONG, &status);
#if DBG
printf("HEADER(%d):\n", rank);
......@@ -229,35 +259,49 @@ void ReadZBin_MPI(TGrid &grid, const std::string& f_name, const std::string& rea
}
#endif
for (unsigned int ichannel = 0; ichannel < NCHANNELS; ichannel++)
{
//MPI_File_read_at(file_id, (rank*2+0)*sizeof(long), &offset , 1, MPI_LONG, &status);
//MPI_File_read_at(file_id, (rank*2+1)*sizeof(long), &compressed_bytes, 1, MPI_LONG, &status);
for (unsigned int ichannel = 0; ichannel < NCHANNELS; ichannel++) {
// MPI_File_read_at(file_id, (rank*2+0)*sizeof(long), &offset , 1,
// MPI_LONG, &status); MPI_File_read_at(file_id,
// (rank*2+1)*sizeof(long), &compressed_bytes, 1, MPI_LONG, &status);