so I'm trying to copy over vectors of different lengths between MPI processes in C++, namely taking vectors on all of the nodes and concatenating them into a new vector on node 0.
I have the following code, which does not return what I expected, driving me crazy, and causing trouble further down the line.
The code is this (abbreviated):
//previously summed all of numfrags to make _numFrag
//numfrags is a vector of the local sizes of _fragLoc
//_numFrag is the total of numfrags
MPI::COMM_WORLD.Barrier();
cout << _myid << "local numFrag = " << _fragLoc.size() << endl;
MPI::COMM_WORLD.Barrier();
for (unsigned i = 0; i < _fragLoc.size(); ++i) cout << "fragloc(" << i << ") = " << _fragLoc[i] << endl;
MPI::COMM_WORLD.Barrier();
vector<int> outVector (_numFrag);
int displ[_numprocs];
if (_myid == 0) {
double sum = 0;
for (int i = 0; i < _numprocs; ++i) {
displ[i] = sum;
cout << _myid << " : " << i << " : " << sum << endl;
sum += numfrags[i];
}
}
MPI::COMM_WORLD.Barrier(); MPI::COMM_WORLD.Gatherv(&_fragLoc[0], numfrags[_myid], MPI::INT, &outVector[0], &numfrags[0], &displ[0], MPI::INT,0);
MPI::COMM_WORLD.Barrier();
if (_myid == 0) {
cout << "X numFrag = " << _numFrag << endl;
for (unsigned i = 0; i < _numFrag; ++i) cout << "outVector(" << i << ") = " << outVector[i] << endl;
}
Giving a simple example, I have a four-node run. Here are the variable inputs as pseudocode:
int _numprocs = 4;
vector<int> numfrags = {0,1,0,1};
vector<int> _fragLoc <node 0> = {};
vector<int> _fragLoc <node 1> = {12};
vector<int> _fragLoc <node 2> = {};
vector<int> _fragLoc <node 3> = {37};
int _numFrag = 2;
The output is:
2local numFrag = 0
3local numFrag = 1
0local numFrag = 0
1local numFrag = 1
fragloc(0) = 12
fragloc(0) = 37
0 : 0 : 0
0 : 1 : 0
0 : 2 : 1
0 : 3 : 1
0: after stage 2
X numFrag = 2
outVector(0) = 0
outVector(1) = 0
But I expected the individual fragLoc's to be put together into outVector and this isn't happening. Any advice? I'll clean up the barriers when I'm done debugging.