From befb24b32e5c8435c5051247d74032892b0c474c Mon Sep 17 00:00:00 2001 From: adityakpandare Date: Wed, 18 Sep 2024 16:20:30 -0600 Subject: [PATCH 1/2] Changes attempting to make the collide-lib migratable --- src/libs/ck-libs/collide/collide_util.C | 2 +- src/libs/ck-libs/collide/collidecharm.C | 83 +++++++++++++++++++- src/libs/ck-libs/collide/collidecharm.ci | 6 +- src/libs/ck-libs/collide/collidecharm.h | 21 ++++- src/libs/ck-libs/collide/collidecharm_impl.h | 23 ++++++ src/libs/ck-libs/collide/threadCollide.C | 4 + 6 files changed, 129 insertions(+), 10 deletions(-) diff --git a/src/libs/ck-libs/collide/collide_util.C b/src/libs/ck-libs/collide/collide_util.C index 3148fdbf5c..5a533f3a59 100644 --- a/src/libs/ck-libs/collide/collide_util.C +++ b/src/libs/ck-libs/collide/collide_util.C @@ -49,7 +49,7 @@ void CollideGrid3d::init(const vector3d &Norigin,//Grid voxel corner 0,0,0 sizes[i]=desiredSize[i]; #endif ((double *)scales)[i]=1.0/((double *)sizes)[i]; - testMapping(*this,i,((double *)origin)[i],((double *)sizes)[i]); + //testMapping(*this,i,((double *)origin)[i],((double *)sizes)[i]); } } diff --git a/src/libs/ck-libs/collide/collidecharm.C b/src/libs/ck-libs/collide/collidecharm.C index 529f39e0c5..327a2e696f 100644 --- a/src/libs/ck-libs/collide/collidecharm.C +++ b/src/libs/ck-libs/collide/collidecharm.C @@ -4,6 +4,7 @@ */ #include "charm++.h" #include "collidecharm_impl.h" +#include #define COLLIDE_TRACE 0 #if COLLIDE_TRACE @@ -45,6 +46,17 @@ CkGroupID CollideSerialClient(CkCallback clientCb) return cl; } +/// Call this on all procesors at restart, to reinitialize the Collision client +/// with correct clientFn and clientParam +void CollideSerialClientRestart( + CollideHandle h, + CollisionClientFn clientFn, + void *clientParam ) +{ + CProxy_collideMgr mgr(h); + mgr.ckLocalBranch()->reinitClient(clientFn, clientParam); +} + /// Create a collider group to contribute objects to. /// Should be called on processor 0. CollideHandle CollideCreate(const CollideGrid3d &gridMap, @@ -74,6 +86,7 @@ void CollideBoxesPrio(CollideHandle h,int chunkNo, int nBox,const bbox3d *boxes,const int *prio) { CProxy_collideMgr mgr(h); + std::cout << "CollideBoxesPrio init'ed collideMgr from nBoxes " << nBox << std::endl; mgr.ckLocalBranch()->contribute(chunkNo,nBox,boxes,prio); } @@ -336,6 +349,7 @@ void syncReductionMgr::startStep(int stepNo,bool withProd) void syncReductionMgr::advance(void) { + std::cout << "in syncReductionMgr::advance()" << std::endl; SRM_STATUS("syncReductionMgr::advance"); if (stepFinished) startStep(stepCount+1,false); localFinished=true; @@ -347,10 +361,13 @@ void syncReductionMgr::pleaseAdvance(void) //This is called on PE 0 once the reduction is finished void syncReductionMgr::reductionFinished(void) -{ /*Child use only */ } +{ /*Child use only */ + std::cout << "in syncReductionMgr::reductionFinished()" << std::endl; +} void syncReductionMgr::tryFinish(void) //Try to finish reduction { + std::cout << "in syncReductionMgr::tryFinished()" << std::endl; SRM_STATUS("syncReductionMgr::tryFinish"); if (localFinished && (!stepFinished) && childrenCount==nChildren) { @@ -371,12 +388,30 @@ void syncReductionMgr::childProd(int stepCount) //Called by tree children-- me and my children are finished void syncReductionMgr::childDone(int stepCount) { + std::cout << "in syncReductionMgr::childDone()" << std::endl; SRM_STATUS("syncReductionMgr::childDone"); if (stepFinished) startStep(stepCount,true); childrenCount++; tryFinish(); } +syncReductionMgr::syncReductionMgr(CkMigrateMessage* m) : + CBase_syncReductionMgr(m) +{ + stepCount=-1; + stepFinished=true; + localFinished=false; + + //Set up the reduction tree + onPE=CkMyPe(); + if (onPE==0) treeParent=-1; + else treeParent=(onPE-1)/TREE_WID; + treeChildStart=(onPE*TREE_WID)+1; + treeChildEnd=treeChildStart+TREE_WID; + if (treeChildStart>CkNumPes()) treeChildStart=CkNumPes(); + if (treeChildEnd>CkNumPes()) treeChildEnd=CkNumPes(); + nChildren=treeChildEnd-treeChildStart; +} /*********************** collideMgr ***********************/ //Extract the (signed) low 23 bits of src-- @@ -411,6 +446,26 @@ collideMgr::collideMgr(const CollideGrid3d &gridMap_, msgsSent=msgsRecvd=0; } +collideMgr::collideMgr(CkMigrateMessage* m) : + CBase_collideMgr(m), + gridMap(CollideGrid3d(CkVector3d(0, 0, 0),CkVector3d(2, 100, 2))), + aggregator(gridMap,this) +{ + steps=0; + nContrib=0; + contribCount=0; + msgsSent=msgsRecvd=0; +} + +// Reinitialize collideClient's state on restart +void collideMgr::reinitClient( + CollisionClientFn clientFn, + void *clientParam ) +{ + std::cout << "collideMgr::reinitClient from " << thisIndex << std::endl; + client.ckLocalBranch()->setClient(clientFn, clientParam); +} + //Maintain contributor registration count void collideMgr::registerContributor(int chunkNo) { @@ -428,6 +483,7 @@ void collideMgr::contribute(int chunkNo, int n,const bbox3d *boxes,const int *prio) { //printf("[%d] Receiving contribution from %d\n",CkMyPe(), chunkNo); + std::cout << "began contribute() of nBoxes " << n << std::endl; CM_STATUS("collideMgr::contribute "<collisions(this,step,colls); emptyMessages(); CC_STATUS("} startCollision"); } +collideClient::collideClient() {} + +collideClient::collideClient(CkMigrateMessage* m) : Group(m) {} + collideClient::~collideClient() {} /********************** serialCollideClient *****************/ @@ -629,6 +694,13 @@ serialCollideClient::serialCollideClient(void) { useCb = false; } +serialCollideClient::serialCollideClient(CkMigrateMessage* m) : collideClient(m) +{ + clientFn = NULL; + clientParam = NULL; + clientCb = CkCallback(CkCallback::ignore); +} + serialCollideClient::serialCollideClient(CkCallback clientCb_) { clientFn=NULL; clientParam=NULL; @@ -638,6 +710,7 @@ serialCollideClient::serialCollideClient(CkCallback clientCb_) { /// Call this client function on processor 0: void serialCollideClient::setClient(CollisionClientFn clientFn_,void *clientParam_) { + std::cout << "serialCollideClient::setClient from " << thisIndex << std::endl; clientFn=clientFn_; clientParam=clientParam_; } @@ -646,22 +719,28 @@ void serialCollideClient::collisions(ArrayElement *src, int step,CollisionList &colls) { if(useCb) { // Use user passed callback as reduction target + std::cout << "in serialCollideClient::collisions(), useCB" << std::endl; src->contribute(colls.length()*sizeof(Collision),colls.getData(), CkReduction::concat, clientCb); } else { // Use client fn with reduction targetted at serialCollideClient::reductionDone CkCallback cb(CkIndex_serialCollideClient::reductionDone(0),0,thisgroup); + std::cout << "in serialCollideClient::collisions(), NOT useCB " << thisIndex << std::endl; src->contribute(colls.length()*sizeof(Collision),colls.getData(), CkReduction::concat,cb); + std::cout << "in serialCollideClient::collisions(), after contribute " << thisIndex << std::endl; } } //This is called after the collideVoxel Collision reduction completes void serialCollideClient::reductionDone(CkReductionMsg *msg) { + std::cout << "in serialCollideClient::reductionDone()" << std::endl; int nColl=msg->getSize()/sizeof(Collision); CM_STATUS("serialCollideClient with "<getData()); + } else //FIXME: never registered a client CkAbort("Forgot to call serialCollideClient::setClient!\n"); delete msg; diff --git a/src/libs/ck-libs/collide/collidecharm.ci b/src/libs/ck-libs/collide/collidecharm.ci index 8f1d507af2..b5f833dc1d 100644 --- a/src/libs/ck-libs/collide/collidecharm.ci +++ b/src/libs/ck-libs/collide/collidecharm.ci @@ -3,18 +3,18 @@ module collidecharm { group collideClient { }; - group serialCollideClient : collideClient { + group [migratable] serialCollideClient : collideClient { entry serialCollideClient(); entry serialCollideClient(CkCallback clientCb_); entry void reductionDone(CkReductionMsg *m); }; - group syncReductionMgr { + group [migratable] syncReductionMgr { entry syncReductionMgr(); entry void childProd(int stepCount); entry void childDone(int stepCount); }; - group collideMgr : syncReductionMgr { + group [migratable] collideMgr : syncReductionMgr { entry collideMgr(CollideGrid3d gridMap, CProxy_collideClient client, CkArrayID cells); diff --git a/src/libs/ck-libs/collide/collidecharm.h b/src/libs/ck-libs/collide/collidecharm.h index aabf7d2684..32908777af 100644 --- a/src/libs/ck-libs/collide/collidecharm.h +++ b/src/libs/ck-libs/collide/collidecharm.h @@ -9,6 +9,10 @@ #include "collide_util.h" #include "collidecharm.decl.h" +/// This client function is called on PE 0 with a Collision list +typedef void (*CollisionClientFn)(void *param,int nColl,Collision *colls); + + /********************** collideClient ***************** A place for the assembled Collision lists to go. Each voxel calls this "Collisions" method with their @@ -22,17 +26,18 @@ */ class collideClient : public Group { public: + collideClient(); + collideClient(CkMigrateMessage* m); virtual ~collideClient(); virtual void collisions(ArrayElement *src, int step,CollisionList &colls) =0; + virtual void setClient(CollisionClientFn clientFn, + void *clientParam) = 0; }; /********************** serialCollideClient ***************** Reduces the Collision list down to processor 0. */ -/// This client function is called on PE 0 with a Collision list -typedef void (*CollisionClientFn)(void *param,int nColl,Collision *colls); - /// Call this on processor 0 to build a Collision client that /// just calls this serial routine on processor 0 with the final, /// complete Collision list. @@ -43,9 +48,17 @@ CkGroupID CollideSerialClient(CollisionClientFn clientFn,void *clientParam); /// complete Collision list. CkGroupID CollideSerialClient(CkCallback clientCb); -/****************** Collision Interface ******************/ typedef CkGroupID CollideHandle; +/// Call this on all procesors at restart, to reinitialize the Collision client +/// with correct clientFn and clientParam +void CollideSerialClientRestart( + CollideHandle h, + CollisionClientFn clientFn, + void *clientParam ); + +/****************** Collision Interface ******************/ + /// Create a collider group to contribute objects to. /// Should be called on processor 0. CollideHandle CollideCreate(const CollideGrid3d &gridMap, diff --git a/src/libs/ck-libs/collide/collidecharm_impl.h b/src/libs/ck-libs/collide/collidecharm_impl.h index 7232caf68c..5cb6b1d8e8 100644 --- a/src/libs/ck-libs/collide/collidecharm_impl.h +++ b/src/libs/ck-libs/collide/collidecharm_impl.h @@ -125,6 +125,7 @@ class syncReductionMgr : public CBase_syncReductionMgr public: int getStepCount(void) const {return stepCount;} syncReductionMgr(); + syncReductionMgr(CkMigrateMessage* m); //Called by parent-- will you contribute? void childProd(int stepCount); @@ -172,11 +173,29 @@ class collideMgr : public CBase_collideMgr collideMgr(const CollideGrid3d &gridMap, const CProxy_collideClient &client, const CProxy_collideVoxel &voxels); + collideMgr(CkMigrateMessage* m); + + void pup(PUP::er &p) { + p | thisproxy; + //p | steps; + p | voxelProxy; + p | gridMap; + p | client; + //p | nContrib; + //p | contribCount; + //p | msgsSent; + //p | msgsRecvd; + }; //Maintain contributor registration count void registerContributor(int chunkNo); void unregisterContributor(int chunkNo); + // Reinitialize collideClient's state on restart + void reinitClient( + CollisionClientFn clientFn, + void *clientParam ); + //Clients call this to contribute their objects void contribute(int chunkNo, int n,const bbox3d *boxes,const int *prio); @@ -225,6 +244,10 @@ class serialCollideClient : public collideClient { public: serialCollideClient(void); serialCollideClient(CkCallback clientCb_); + serialCollideClient(CkMigrateMessage *m); + void pup(PUP::er &p) { + p | useCb; + } /// Call this client function on processor 0: void setClient(CollisionClientFn clientFn,void *clientParam); diff --git a/src/libs/ck-libs/collide/threadCollide.C b/src/libs/ck-libs/collide/threadCollide.C index 340dd1d69e..1b5c15dcc0 100644 --- a/src/libs/ck-libs/collide/threadCollide.C +++ b/src/libs/ck-libs/collide/threadCollide.C @@ -77,6 +77,10 @@ class threadCollideMgr : public CBase_threadCollideMgr contrib[chunkNo]=NULL; } + /// Aditya Pandare: not sure why this is needed + /// Placeholder: Call this client function on processor 0: + void setClient(CollisionClientFn, void *) {} + /// collideClient interface (called by voxels) /// Splits up collisions by destination PE void collisions(ArrayElement *src,int step,CollisionList &colls); From 7401d78ef7e0860e4fede46537d01791dcb7f460 Mon Sep 17 00:00:00 2001 From: Advait Tahilyani Date: Mon, 20 Apr 2026 13:39:13 -0500 Subject: [PATCH 2/2] Changes to fix reset --- src/libs/ck-libs/collide/collidecharm.C | 11 ++++++----- src/libs/ck-libs/collide/collidecharm_impl.h | 10 +++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/libs/ck-libs/collide/collidecharm.C b/src/libs/ck-libs/collide/collidecharm.C index 327a2e696f..6b949d4170 100644 --- a/src/libs/ck-libs/collide/collidecharm.C +++ b/src/libs/ck-libs/collide/collidecharm.C @@ -314,7 +314,6 @@ void CollisionAggregator::compact(void) /********************* syncReductionMgr ******************/ syncReductionMgr::syncReductionMgr() - :thisproxy(thisgroup) { stepCount=-1; stepFinished=true; @@ -339,10 +338,12 @@ void syncReductionMgr::startStep(int stepNo,bool withProd) stepFinished=false; localFinished=false; childrenCount=0; - if (nChildren>0) + if (nChildren>0) { + CProxy_syncReductionMgr proxy(ckGetGroupID()); for (int i=0;i %s\n",CkMyPe(),msg); } @@ -176,6 +175,11 @@ class collideMgr : public CBase_collideMgr collideMgr(CkMigrateMessage* m); void pup(PUP::er &p) { + // thisgroup is a protected member of IrrGroup. It is not saved/restored + // by the migrate constructor, so we do it here. syncReductionMgr uses + // CProxy_syncReductionMgr(ckGetGroupID()) at call time, which requires + // thisgroup to be valid when startStep()/tryFinish() are invoked. + p | thisgroup; p | thisproxy; //p | steps; p | voxelProxy; @@ -246,6 +250,10 @@ class serialCollideClient : public collideClient { serialCollideClient(CkCallback clientCb_); serialCollideClient(CkMigrateMessage *m); void pup(PUP::er &p) { + // thisgroup must be saved: collisions() uses it to construct the + // reduction callback CkCallback(..., thisgroup). Without this, thisgroup + // is 0 after restart and the callback targets a non-existent group. + p | thisgroup; p | useCb; }