diff --git a/src/libs/ck-libs/collide/collide_util.C b/src/libs/ck-libs/collide/collide_util.C index 3148fdbf5c..5a533f3a59 100644 --- a/src/libs/ck-libs/collide/collide_util.C +++ b/src/libs/ck-libs/collide/collide_util.C @@ -49,7 +49,7 @@ void CollideGrid3d::init(const vector3d &Norigin,//Grid voxel corner 0,0,0 sizes[i]=desiredSize[i]; #endif ((double *)scales)[i]=1.0/((double *)sizes)[i]; - testMapping(*this,i,((double *)origin)[i],((double *)sizes)[i]); + //testMapping(*this,i,((double *)origin)[i],((double *)sizes)[i]); } } diff --git a/src/libs/ck-libs/collide/collidecharm.C b/src/libs/ck-libs/collide/collidecharm.C index 529f39e0c5..6b949d4170 100644 --- a/src/libs/ck-libs/collide/collidecharm.C +++ b/src/libs/ck-libs/collide/collidecharm.C @@ -4,6 +4,7 @@ */ #include "charm++.h" #include "collidecharm_impl.h" +#include #define COLLIDE_TRACE 0 #if COLLIDE_TRACE @@ -45,6 +46,17 @@ CkGroupID CollideSerialClient(CkCallback clientCb) return cl; } +/// Call this on all procesors at restart, to reinitialize the Collision client +/// with correct clientFn and clientParam +void CollideSerialClientRestart( + CollideHandle h, + CollisionClientFn clientFn, + void *clientParam ) +{ + CProxy_collideMgr mgr(h); + mgr.ckLocalBranch()->reinitClient(clientFn, clientParam); +} + /// Create a collider group to contribute objects to. /// Should be called on processor 0. CollideHandle CollideCreate(const CollideGrid3d &gridMap, @@ -74,6 +86,7 @@ void CollideBoxesPrio(CollideHandle h,int chunkNo, int nBox,const bbox3d *boxes,const int *prio) { CProxy_collideMgr mgr(h); + std::cout << "CollideBoxesPrio init'ed collideMgr from nBoxes " << nBox << std::endl; mgr.ckLocalBranch()->contribute(chunkNo,nBox,boxes,prio); } @@ -301,7 +314,6 @@ void CollisionAggregator::compact(void) /********************* syncReductionMgr ******************/ syncReductionMgr::syncReductionMgr() - :thisproxy(thisgroup) { stepCount=-1; stepFinished=true; @@ -326,16 +338,19 @@ void syncReductionMgr::startStep(int stepNo,bool withProd) stepFinished=false; localFinished=false; childrenCount=0; - if (nChildren>0) + if (nChildren>0) { + CProxy_syncReductionMgr proxy(ckGetGroupID()); for (int i=0;iCkNumPes()) treeChildStart=CkNumPes(); + if (treeChildEnd>CkNumPes()) treeChildEnd=CkNumPes(); + nChildren=treeChildEnd-treeChildStart; +} /*********************** collideMgr ***********************/ //Extract the (signed) low 23 bits of src-- @@ -402,7 +438,7 @@ static const char * voxName(const CkIndex3D &idx,char *buf) { collideMgr::collideMgr(const CollideGrid3d &gridMap_, const CProxy_collideClient &client_, const CProxy_collideVoxel &voxels) - :thisproxy(thisgroup), voxelProxy(voxels), + :voxelProxy(voxels), gridMap(gridMap_), client(client_), aggregator(gridMap,this) { steps=0; @@ -411,6 +447,26 @@ collideMgr::collideMgr(const CollideGrid3d &gridMap_, msgsSent=msgsRecvd=0; } +collideMgr::collideMgr(CkMigrateMessage* m) : + CBase_collideMgr(m), + gridMap(CollideGrid3d(CkVector3d(0, 0, 0),CkVector3d(2, 100, 2))), + aggregator(gridMap,this) +{ + steps=0; + nContrib=0; + contribCount=0; + msgsSent=msgsRecvd=0; +} + +// Reinitialize collideClient's state on restart +void collideMgr::reinitClient( + CollisionClientFn clientFn, + void *clientParam ) +{ + std::cout << "collideMgr::reinitClient from " << thisIndex << std::endl; + client.ckLocalBranch()->setClient(clientFn, clientParam); +} + //Maintain contributor registration count void collideMgr::registerContributor(int chunkNo) { @@ -428,6 +484,7 @@ void collideMgr::contribute(int chunkNo, int n,const bbox3d *boxes,const int *prio) { //printf("[%d] Receiving contribution from %d\n",CkMyPe(), chunkNo); + std::cout << "began contribute() of nBoxes " << n << std::endl; CM_STATUS("collideMgr::contribute "<collisions(this,step,colls); emptyMessages(); CC_STATUS("} startCollision"); } +collideClient::collideClient() {} + +collideClient::collideClient(CkMigrateMessage* m) : Group(m) {} + collideClient::~collideClient() {} /********************** serialCollideClient *****************/ @@ -629,6 +695,13 @@ serialCollideClient::serialCollideClient(void) { useCb = false; } +serialCollideClient::serialCollideClient(CkMigrateMessage* m) : collideClient(m) +{ + clientFn = NULL; + clientParam = NULL; + clientCb = CkCallback(CkCallback::ignore); +} + serialCollideClient::serialCollideClient(CkCallback clientCb_) { clientFn=NULL; clientParam=NULL; @@ -638,6 +711,7 @@ serialCollideClient::serialCollideClient(CkCallback clientCb_) { /// Call this client function on processor 0: void serialCollideClient::setClient(CollisionClientFn clientFn_,void *clientParam_) { + std::cout << "serialCollideClient::setClient from " << thisIndex << std::endl; clientFn=clientFn_; clientParam=clientParam_; } @@ -646,22 +720,28 @@ void serialCollideClient::collisions(ArrayElement *src, int step,CollisionList &colls) { if(useCb) { // Use user passed callback as reduction target + std::cout << "in serialCollideClient::collisions(), useCB" << std::endl; src->contribute(colls.length()*sizeof(Collision),colls.getData(), CkReduction::concat, clientCb); } else { // Use client fn with reduction targetted at serialCollideClient::reductionDone CkCallback cb(CkIndex_serialCollideClient::reductionDone(0),0,thisgroup); + std::cout << "in serialCollideClient::collisions(), NOT useCB " << thisIndex << std::endl; src->contribute(colls.length()*sizeof(Collision),colls.getData(), CkReduction::concat,cb); + std::cout << "in serialCollideClient::collisions(), after contribute " << thisIndex << std::endl; } } //This is called after the collideVoxel Collision reduction completes void serialCollideClient::reductionDone(CkReductionMsg *msg) { + std::cout << "in serialCollideClient::reductionDone()" << std::endl; int nColl=msg->getSize()/sizeof(Collision); CM_STATUS("serialCollideClient with "<getData()); + } else //FIXME: never registered a client CkAbort("Forgot to call serialCollideClient::setClient!\n"); delete msg; diff --git a/src/libs/ck-libs/collide/collidecharm.ci b/src/libs/ck-libs/collide/collidecharm.ci index 8f1d507af2..b5f833dc1d 100644 --- a/src/libs/ck-libs/collide/collidecharm.ci +++ b/src/libs/ck-libs/collide/collidecharm.ci @@ -3,18 +3,18 @@ module collidecharm { group collideClient { }; - group serialCollideClient : collideClient { + group [migratable] serialCollideClient : collideClient { entry serialCollideClient(); entry serialCollideClient(CkCallback clientCb_); entry void reductionDone(CkReductionMsg *m); }; - group syncReductionMgr { + group [migratable] syncReductionMgr { entry syncReductionMgr(); entry void childProd(int stepCount); entry void childDone(int stepCount); }; - group collideMgr : syncReductionMgr { + group [migratable] collideMgr : syncReductionMgr { entry collideMgr(CollideGrid3d gridMap, CProxy_collideClient client, CkArrayID cells); diff --git a/src/libs/ck-libs/collide/collidecharm.h b/src/libs/ck-libs/collide/collidecharm.h index aabf7d2684..32908777af 100644 --- a/src/libs/ck-libs/collide/collidecharm.h +++ b/src/libs/ck-libs/collide/collidecharm.h @@ -9,6 +9,10 @@ #include "collide_util.h" #include "collidecharm.decl.h" +/// This client function is called on PE 0 with a Collision list +typedef void (*CollisionClientFn)(void *param,int nColl,Collision *colls); + + /********************** collideClient ***************** A place for the assembled Collision lists to go. Each voxel calls this "Collisions" method with their @@ -22,17 +26,18 @@ */ class collideClient : public Group { public: + collideClient(); + collideClient(CkMigrateMessage* m); virtual ~collideClient(); virtual void collisions(ArrayElement *src, int step,CollisionList &colls) =0; + virtual void setClient(CollisionClientFn clientFn, + void *clientParam) = 0; }; /********************** serialCollideClient ***************** Reduces the Collision list down to processor 0. */ -/// This client function is called on PE 0 with a Collision list -typedef void (*CollisionClientFn)(void *param,int nColl,Collision *colls); - /// Call this on processor 0 to build a Collision client that /// just calls this serial routine on processor 0 with the final, /// complete Collision list. @@ -43,9 +48,17 @@ CkGroupID CollideSerialClient(CollisionClientFn clientFn,void *clientParam); /// complete Collision list. CkGroupID CollideSerialClient(CkCallback clientCb); -/****************** Collision Interface ******************/ typedef CkGroupID CollideHandle; +/// Call this on all procesors at restart, to reinitialize the Collision client +/// with correct clientFn and clientParam +void CollideSerialClientRestart( + CollideHandle h, + CollisionClientFn clientFn, + void *clientParam ); + +/****************** Collision Interface ******************/ + /// Create a collider group to contribute objects to. /// Should be called on processor 0. CollideHandle CollideCreate(const CollideGrid3d &gridMap, diff --git a/src/libs/ck-libs/collide/collidecharm_impl.h b/src/libs/ck-libs/collide/collidecharm_impl.h index 7232caf68c..c087f2952e 100644 --- a/src/libs/ck-libs/collide/collidecharm_impl.h +++ b/src/libs/ck-libs/collide/collidecharm_impl.h @@ -96,7 +96,6 @@ class CollisionAggregator { */ class syncReductionMgr : public CBase_syncReductionMgr { - CProxy_syncReductionMgr thisproxy; void status(const char *msg) { CkPrintf("SRMgr pe %d> %s\n",CkMyPe(),msg); } @@ -125,6 +124,7 @@ class syncReductionMgr : public CBase_syncReductionMgr public: int getStepCount(void) const {return stepCount;} syncReductionMgr(); + syncReductionMgr(CkMigrateMessage* m); //Called by parent-- will you contribute? void childProd(int stepCount); @@ -172,11 +172,34 @@ class collideMgr : public CBase_collideMgr collideMgr(const CollideGrid3d &gridMap, const CProxy_collideClient &client, const CProxy_collideVoxel &voxels); + collideMgr(CkMigrateMessage* m); + + void pup(PUP::er &p) { + // thisgroup is a protected member of IrrGroup. It is not saved/restored + // by the migrate constructor, so we do it here. syncReductionMgr uses + // CProxy_syncReductionMgr(ckGetGroupID()) at call time, which requires + // thisgroup to be valid when startStep()/tryFinish() are invoked. + p | thisgroup; + p | thisproxy; + //p | steps; + p | voxelProxy; + p | gridMap; + p | client; + //p | nContrib; + //p | contribCount; + //p | msgsSent; + //p | msgsRecvd; + }; //Maintain contributor registration count void registerContributor(int chunkNo); void unregisterContributor(int chunkNo); + // Reinitialize collideClient's state on restart + void reinitClient( + CollisionClientFn clientFn, + void *clientParam ); + //Clients call this to contribute their objects void contribute(int chunkNo, int n,const bbox3d *boxes,const int *prio); @@ -225,6 +248,14 @@ class serialCollideClient : public collideClient { public: serialCollideClient(void); serialCollideClient(CkCallback clientCb_); + serialCollideClient(CkMigrateMessage *m); + void pup(PUP::er &p) { + // thisgroup must be saved: collisions() uses it to construct the + // reduction callback CkCallback(..., thisgroup). Without this, thisgroup + // is 0 after restart and the callback targets a non-existent group. + p | thisgroup; + p | useCb; + } /// Call this client function on processor 0: void setClient(CollisionClientFn clientFn,void *clientParam); diff --git a/src/libs/ck-libs/collide/threadCollide.C b/src/libs/ck-libs/collide/threadCollide.C index 340dd1d69e..1b5c15dcc0 100644 --- a/src/libs/ck-libs/collide/threadCollide.C +++ b/src/libs/ck-libs/collide/threadCollide.C @@ -77,6 +77,10 @@ class threadCollideMgr : public CBase_threadCollideMgr contrib[chunkNo]=NULL; } + /// Aditya Pandare: not sure why this is needed + /// Placeholder: Call this client function on processor 0: + void setClient(CollisionClientFn, void *) {} + /// collideClient interface (called by voxels) /// Splits up collisions by destination PE void collisions(ArrayElement *src,int step,CollisionList &colls);