[dcmf] [PATCH] Issue 3867: Free storage before composites are
regenerated.
Joe Ratterman
jratt at us.ibm.com
Mon Feb 4 14:00:29 CST 2008
From: Bob Cernohous <bobc at us.ibm.com>
---
sys/collectives/adaptor/dcmf/Geometry.h | 17 +++---
sys/collectives/adaptor/dcmf/api/dcmf.cc | 28 ++++++++--
.../adaptor/dcmf/protocols/allreduce/Composite.h | 9 ++--
.../dcmf/protocols/allreduce/binomial/Composite.h | 6 ++
.../dcmf/protocols/allreduce/rectangle/Composite.h | 6 ++
.../protocols/allreduce/rectanglering/Composite.h | 9 +++-
.../dcmf/protocols/allreduce/tree/Composite.h | 22 +++++---
.../dcmf/protocols/allreduce/tree/Factory.h | 4 +-
.../dcmf/protocols/reduce/rectangle/Composite.h | 6 ++
.../protocols/reduce/rectanglering/Composite.h | 6 ++
sys/collectives/kernel/executor/Allreduce.h | 25 ++++++---
sys/collectives/kernel/executor/AllreduceBase.h | 56 ++++++++++++++------
sys/collectives/kernel/executor/AllreduceState.h | 14 +++++-
sys/collectives/kernel/interfaces/Composite.h | 44 ++++++++++++----
sys/collectives/kernel/interfaces/Executor.h | 45 ++++++++++------
sys/collectives/tests/dcmf/tree/tree_allreduce.cc | 16 +++---
sys/collectives/tests/dcmf/tree/tree_reduce.cc | 16 +++---
17 files changed, 234 insertions(+), 95 deletions(-)
mode change 100644 => 100755 sys/collectives/adaptor/dcmf/Geometry.h
mode change 100644 => 100755 sys/collectives/adaptor/dcmf/api/dcmf.cc
mode change 100644 => 100755 sys/collectives/kernel/executor/Allreduce.h
mode change 100644 => 100755 sys/collectives/kernel/interfaces/Executor.h
mode change 100644 => 100755 sys/collectives/tests/dcmf/tree/tree_allreduce.cc
mode change 100644 => 100755 sys/collectives/tests/dcmf/tree/tree_reduce.cc
diff --git a/sys/collectives/adaptor/dcmf/Geometry.h b/sys/collectives/adaptor/dcmf/Geometry.h
old mode 100644
new mode 100755
index c43a880..1381dd0
--- a/sys/collectives/adaptor/dcmf/Geometry.h
+++ b/sys/collectives/adaptor/dcmf/Geometry.h
@@ -164,19 +164,18 @@ namespace CCMI
inline void freeAllocations ()
{
+// fprintf(stderr,"Geometry::freeAllocations(), _allreduce %#X, _allreduce_storage %#X\n",
+// (int)_allreduce, (int)_allreduce_storage);
if (_allreduce)
{
- unsigned nexec = _allreduce->getNumExecutors();
- for (unsigned count = 0; count < nexec; count ++)
- {
- //Call the destructor on the executor constructor
- delete _allreduce->getExecutor(count);
- }
-
- CCMI_Free (_allreduce_storage);
- _allreduce_storage = NULL;
+ _allreduce->~Composite();
_allreduce = NULL;
}
+ if(_allreduce_storage)
+ {
+ CCMI_Free (_allreduce_storage);
+ _allreduce_storage = NULL;
+ }
}
inline Kernel::Executor::Executor * getCollectiveExecutor (unsigned color=0)
diff --git a/sys/collectives/adaptor/dcmf/api/dcmf.cc b/sys/collectives/adaptor/dcmf/api/dcmf.cc
old mode 100644
new mode 100755
index 04443c4..56485c2
--- a/sys/collectives/adaptor/dcmf/api/dcmf.cc
+++ b/sys/collectives/adaptor/dcmf/api/dcmf.cc
@@ -423,10 +423,16 @@ int DCMF_Allreduce (DCMF_CollectiveProtocol_t * registration,
count,
DCMF_Dt_to_CCMI_Dt[dtype],
DCMF_Op_to_CCMI_Op[op]);
- if (status == DCMF_SUCCESS)
- return DCMF_SUCCESS;
+// if (status != DCMF_SUCCESS) fprintf(stderr, "DCMF_Allreduce::ALERT: restart failed on executor %#X with status %#X\n", (int) allreduce, status);
+ return status;
+ }
+ if ( allreduce != NULL ) // Different factory? Cleanup old executor.
+ {
+ geometry->setAllreduceComposite(NULL);
+ allreduce->~BaseComposite();
}
+// fprintf(stderr, "DCMF_Allreduce::ALERT: generate executor %#X with factory %#X\n",(int) allreduce,(int)factory);
void *ptr =factory->generate((CCMI_CollectiveRequest_t*)request,
*(CCMI_Callback_t *) &cb_done,
(CCMI_Consistency) consistency,
@@ -437,7 +443,10 @@ int DCMF_Allreduce (DCMF_CollectiveProtocol_t * registration,
DCMF_Dt_to_CCMI_Dt[dtype],
DCMF_Op_to_CCMI_Op[op]);
if(ptr == NULL)
+ {
+// fprintf(stderr, "DCMF_Allreduce::ALERT: generate failed\n");
return DCMF_UNIMPL;
+ }
return DCMF_SUCCESS;
}
@@ -598,10 +607,16 @@ int DCMF_Reduce (DCMF_CollectiveProtocol_t * registration,
DCMF_Dt_to_CCMI_Dt[dtype],
DCMF_Op_to_CCMI_Op[op],
root);
- if (status == DCMF_SUCCESS)
- return DCMF_SUCCESS;
+// if (status != DCMF_SUCCESS) fprintf(stderr, "DCMF_Reduce::ALERT: restart failed on executor %#X with status %#X\n", (int) allreduce, status);
+ return status;
+ }
+ if ( allreduce != NULL ) // Different factory? Cleanup old executor.
+ {
+ geometry->setAllreduceComposite(NULL);
+ allreduce->~BaseComposite();
}
+// fprintf(stderr, "DCMF_Reduce::ALERT: generate executor %#X with factory %#X\n",(int) allreduce,(int)factory);
void *ptr =factory->generate((CCMI_CollectiveRequest_t*)request,
*(CCMI_Callback_t *) &cb_done,
(CCMI_Consistency) consistency,
@@ -612,8 +627,11 @@ int DCMF_Reduce (DCMF_CollectiveProtocol_t * registration,
DCMF_Dt_to_CCMI_Dt[dtype],
DCMF_Op_to_CCMI_Op[op],
root);
- if(ptr==NULL)
+ if(ptr == NULL)
+ {
+// fprintf(stderr, "DCMF_Reduce::ALERT: generate failed\n");
return DCMF_UNIMPL;
+ }
return DCMF_SUCCESS;
}
diff --git a/sys/collectives/adaptor/dcmf/protocols/allreduce/Composite.h b/sys/collectives/adaptor/dcmf/protocols/allreduce/Composite.h
index ae98703..0f553db 100755
--- a/sys/collectives/adaptor/dcmf/protocols/allreduce/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/allreduce/Composite.h
@@ -99,16 +99,17 @@ namespace CCMI
initializeBarrier ((Kernel::Executor::Barrier *) barrier);
}
+ /// Default Destructor
virtual ~Composite()
{
TRACE_ALERT((stderr,"<%#.8X>Allreduce::Composite::dtor() ALERT:\n",(int)this));
TRACE_ADAPTOR((stderr,"<%#.8X>Allreduce::Composite::dtor()\n",(int)this));
}
- void operator delete (void *p)
- {
- CCMI_assert (0);
- }
+ void operator delete (void *p)
+ {
+ CCMI_assert (0);
+ }
void initializeBarrier (Kernel::Executor::Barrier *barrier)
{
diff --git a/sys/collectives/adaptor/dcmf/protocols/allreduce/binomial/Composite.h b/sys/collectives/adaptor/dcmf/protocols/allreduce/binomial/Composite.h
index 92084a2..a30a19e 100755
--- a/sys/collectives/adaptor/dcmf/protocols/allreduce/binomial/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/allreduce/binomial/Composite.h
@@ -38,6 +38,12 @@ namespace CCMI
Kernel::Schedule::BinomialTreeSchedule _schedule;
public:
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::Binomial::Composite::dtor() ALERT:\n",(int)this));
+ TRACE_ADAPTOR((stderr,"<%#.8X>Allreduce::Binomial::Composite::dtor()\n",(int)this));
+ }
///
/// \brief Constructor
///
diff --git a/sys/collectives/adaptor/dcmf/protocols/allreduce/rectangle/Composite.h b/sys/collectives/adaptor/dcmf/protocols/allreduce/rectangle/Composite.h
index b0f86bd..ef7a010 100755
--- a/sys/collectives/adaptor/dcmf/protocols/allreduce/rectangle/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/allreduce/rectangle/Composite.h
@@ -40,6 +40,12 @@ namespace CCMI
Kernel::Schedule::OneColorRectAllredSched _schedule;
public:
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::Rectangle::Composite::dtor() ALERT:\n",(int)this));
+ TRACE_ADAPTOR((stderr,"<%#.8X>Allreduce::Rectangle::Composite::dtor()\n",(int)this));
+ }
///
/// \brief Constructor
///
diff --git a/sys/collectives/adaptor/dcmf/protocols/allreduce/rectanglering/Composite.h b/sys/collectives/adaptor/dcmf/protocols/allreduce/rectanglering/Composite.h
index ca3621c..87b6e2e 100755
--- a/sys/collectives/adaptor/dcmf/protocols/allreduce/rectanglering/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/allreduce/rectanglering/Composite.h
@@ -21,7 +21,8 @@
namespace CCMI
{
- namespace Adaptor {
+ namespace Adaptor
+ {
namespace Allreduce
{
namespace RectangleRing
@@ -39,6 +40,12 @@ namespace CCMI
Kernel::Schedule::OneColorRectAllredSched _schedule;
public:
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::RectangleRing::Composite::dtor() ALERT:\n",(int)this));
+ TRACE_ADAPTOR((stderr,"<%#.8X>Allreduce::RectangleRing::Composite::dtor()\n",(int)this));
+ }
///
/// \brief Constructor
///
diff --git a/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Composite.h b/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Composite.h
index 40a1643..39d5d05 100755
--- a/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Composite.h
@@ -44,6 +44,12 @@ namespace CCMI
unsigned _tSize; // for convenience
public:
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::Tree::Composite::dtor() ALERT:\n",(int)this));
+ TRACE_ADAPTOR((stderr,"<%#.8X>Allreduce::Tree::Composite::dtor()\n",(int)this));
+ }
///
/// \brief Constructor
///
@@ -91,7 +97,7 @@ namespace CCMI
// Disable pipelining for dual/vn 2 pass operations (double/sum) (and for SMP always)
(((dtype == CCMI_DOUBLE) && (op == CCMI_SUM) && (_tSize > 1) || (_tSize == 1))?
(unsigned) -1 // max pipelineWidth/disabled
- : 0 // none specified/calculate the default
+ : 0 // none specified/calculate the default
),
cb_barrierTreeDone, this);
_executor.setMulticastInterface (mf);
@@ -115,26 +121,28 @@ namespace CCMI
/// \todo what about freeing allocations?
/// \todo what about operator/datatype changes? Will it recover?
TRACE_ADAPTOR((stderr,"<%#.8X>Allreduce::Tree::Composite::restart()\n",(int)this));
- if (checkOp(dtype,op) == -1)
+ CCMI_assert_debug (getNumExecutors() == 1);
+ Kernel::Executor::AllreduceBase * allreduce = (Kernel::Executor::AllreduceBase *) getExecutor(0);
+ if (allreduce && ((op != allreduce->getOp()) || (dtype != allreduce->getDt())) &&
+ (checkOp(dtype,op) == -1))
{
- TRACE_ALERT((stderr,"<%#.8X>Allreduce::Tree::Composite::restart() ALERT: unsupported op\n",(int)this));
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::Tree::Composite::restart() ALERT: unsupported op %#X, type %#X!\n",
+ (int)this, op, dtype));
TRACE_ADAPTOR((stderr, "<%#.8X>Allreduce::Tree::Composite::restart():unsupported op %#X, type %#X!\n",
(int)this, op, dtype));
return DCMF_UNIMPL;
}
+
_myClientFunction = cb_done.function;
_myClientData = cb_done.clientdata;
- CCMI_assert_debug (getNumExecutors() == 1);
- Kernel::Executor::AllreduceBase * allreduce =
- (Kernel::Executor::AllreduceBase *) getExecutor(0);
initialize (allreduce, request, srcbuf, dstbuf,
count, dtype, op, root,
/// \todo re-evaluate pipelining
// Disable pipelining for dual/vn 2 pass operations (double/sum) (and for SMP always)
(((dtype == CCMI_DOUBLE) && (op == CCMI_SUM) && (_tSize > 1) || (_tSize == 1))?
(unsigned) -1 // max pipelineWidth/disabled
- : 0 // none specified/calculate the default
+ : 0 // none specified/calculate the default
),
cb_barrierTreeDone, this);
_executor.setConsistency(consistency);
diff --git a/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Factory.h b/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Factory.h
index 58ebe0c..1258c53 100755
--- a/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Factory.h
+++ b/sys/collectives/adaptor/dcmf/protocols/allreduce/tree/Factory.h
@@ -80,12 +80,14 @@ namespace CCMI
{
if (checkOp(dtype,op) == -1)
{
- TRACE_ALERT((stderr,"<%#.8X>Allreduce::Tree::Factory::generate() ALERT:\n",(int)this));
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::Tree::Factory::generate() ALERT: unsupported op %#X, type %#X!\n",
+ (int)this, op, dtype));
TRACE_ADAPTOR((stderr, "<%#.8X>Allreduce::Tree::Factory::generate():unsupported op %#X, type %#X!\n",
(int)this, op, dtype));
return NULL;
}
+ TRACE_ALERT((stderr,"<%#.8X>Allreduce::Tree::Factory::generate() ALERT:\n",(int)this));
TRACE_ADAPTOR ((stderr, "<%#.8X>Allreduce::Tree::Factory::generate() %#X, geometry %#X comm %#X\n",(int)this,
sizeof(*this),(int) geometry, (int) geometry->comm()));
diff --git a/sys/collectives/adaptor/dcmf/protocols/reduce/rectangle/Composite.h b/sys/collectives/adaptor/dcmf/protocols/reduce/rectangle/Composite.h
index fb8759a..2832837 100755
--- a/sys/collectives/adaptor/dcmf/protocols/reduce/rectangle/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/reduce/rectangle/Composite.h
@@ -40,6 +40,12 @@ namespace CCMI
Kernel::Schedule::OneColorRectRedSched _schedule;
public:
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Reduce::Rectangle::Composite::dtor() ALERT:\n",(int)this));
+ TRACE_ADAPTOR((stderr,"<%#.8X>Reduce::Rectangle::Composite::dtor()\n",(int)this));
+ }
///
/// \brief Constructor
///
diff --git a/sys/collectives/adaptor/dcmf/protocols/reduce/rectanglering/Composite.h b/sys/collectives/adaptor/dcmf/protocols/reduce/rectanglering/Composite.h
index 1e8aa70..1cd5f3c 100755
--- a/sys/collectives/adaptor/dcmf/protocols/reduce/rectanglering/Composite.h
+++ b/sys/collectives/adaptor/dcmf/protocols/reduce/rectanglering/Composite.h
@@ -40,6 +40,12 @@ namespace CCMI
Kernel::Schedule::OneColorRectRedSched _schedule;
public:
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Reduce::RectangleRing::Composite::dtor() ALERT:\n",(int)this));
+ TRACE_ADAPTOR((stderr,"<%#.8X>Reduce::RectangleRing::Composite::dtor()\n",(int)this));
+ }
///
/// \brief Constructor
///
diff --git a/sys/collectives/kernel/executor/Allreduce.h b/sys/collectives/kernel/executor/Allreduce.h
old mode 100644
new mode 100755
index eb59024..d22066d
--- a/sys/collectives/kernel/executor/Allreduce.h
+++ b/sys/collectives/kernel/executor/Allreduce.h
@@ -106,12 +106,6 @@ namespace CCMI
/// \brief Post receives for all expected src pe's
inline void postReceives();
- /// \brief Free all allocations
- inline void freeAllocations()
- {
- return _state->freeAllocations();
- }
-
/// \brief Static function to be passed into the done of multisend send
static void staticNotifySendDone (void *cd)
{
@@ -132,6 +126,21 @@ namespace CCMI
TRACE_FLOW((stderr,"<%#.8X>Executor::Allreduce::staticNotifyReceiveDone() exit\n",(int)cdata->allreduce));
}
+ /// \brief Default Destructor
+ virtual ~Allreduce ()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Executor::Allreduce::dtor() ALERT:\n",(int)this));
+ #ifdef CCMI_DEBUG
+ _nextRecvData=0;
+ _sndInfoRequired=1;
+ _numActiveSends=0;
+ _curRcvPhase=CCMI_KERNEL_EXECUTOR_ALLREDUCE_INITIAL_PHASE;
+ _curRcvChunk=0;
+ _curSrcPeIndex=0;
+ _startRcvPhase=0;
+ _delayAdvance=0;
+ #endif
+ }
/// \brief Default Constructor
inline Allreduce () :
AllreduceBase(),
@@ -509,7 +518,7 @@ inline void CCMI::Kernel::Executor::Allreduce::start()
if (_state->getPhaseNumDstPes(nextSrcPhase+1) != 0)
{
TRACE_INIT((stderr, "<%#.8X>Executor::Allreduce::start() compressing phase %#X, phase %#X\n",
- (int)this,_state->getStartPhase(),nextSrcPhase+1));
+ (int)this,_state->getStartPhase(),nextSrcPhase+1));
// Move this DstPes back to nextActivePhase by manipulating numDstPes.
/// \todo move to state?
_state->compressPhaseNumDstPes(_state->getStartPhase(),nextSrcPhase+1);
@@ -745,7 +754,7 @@ void CCMI::Kernel::Executor::Allreduce::advance()
// Move this DstPes back to nextActivePhase by manipulating numDstPes.
/// \todo move to state?
TRACE_INIT((stderr, "<%#.8X>Executor::Allreduce::advance() compressing phase %#X, phase %#X\n",
- (int)this,nextActivePhase, nextSrcPhase+1));
+ (int)this,nextActivePhase, nextSrcPhase+1));
_state->compressPhaseNumDstPes(nextActivePhase, nextSrcPhase+1);
}
++nextSrcPhase;
diff --git a/sys/collectives/kernel/executor/AllreduceBase.h b/sys/collectives/kernel/executor/AllreduceBase.h
index acc87cf..9fa7bb3 100755
--- a/sys/collectives/kernel/executor/AllreduceBase.h
+++ b/sys/collectives/kernel/executor/AllreduceBase.h
@@ -119,7 +119,7 @@ inline void CCMI_KERNEL_EXECUTOR_ALLREDUCE_trace_data(const char* string, const
// TRACE_ALERT helps (minimally) trace object ctor/resets that help determine whether
// they're being reused appropriately.
-//#define TRACE_ALERT(x) fprintf x
+// #define TRACE_ALERT(x) fprintf x
#ifndef TRACE_ALERT
#define TRACE_ALERT(x)
#endif
@@ -217,6 +217,30 @@ namespace CCMI
unsigned _log_sendmessage;
public:
+ /// Default Destructor
+ virtual ~AllreduceBase ()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Executor::AllreduceBase::dtor() ALERT:\n",(int)this));
+ #ifdef CCMI_DEBUG
+ _curPhase=(unsigned) -1;
+ _startPhase=(unsigned) -1;
+ _endPhase=(unsigned) -1;
+ _nAsyncRcvd=0;
+ _initialized=false;
+ _postReceives=false;
+ _enablePipelining=false;
+ _sState=NULL;
+ _srcbuf=NULL;
+ _dstbuf=NULL;
+ _op=CCMI_UNDEFINED_OP;
+ _dt=CCMI_UNDEFINED_DT;
+ _count=(unsigned) -1;
+ _reduceFunc=NULL;
+ _msendInterface=NULL;
+ _connmgr=NULL;
+ #endif
+ }
+
/// Default Constructor
AllreduceBase () :
Executor (),
@@ -338,17 +362,16 @@ namespace CCMI
/// \param[out] pipeWidth pipeline width
/// \param[out] cb_done receive callback function
///
- virtual
- CCMI_Request_t * notifyRecvHead(const DCQuad * info,
- unsigned count,
- unsigned peer,
- unsigned sndlen,
- unsigned conn_id,
- void * arg,
- unsigned * rcvlen,
- char ** rcvbuf,
- unsigned * pipewidth,
- CCMI_Callback_t * cb_done);
+ virtual CCMI_Request_t * notifyRecvHead(const DCQuad * info,
+ unsigned count,
+ unsigned peer,
+ unsigned sndlen,
+ unsigned conn_id,
+ void * arg,
+ unsigned * rcvlen,
+ char ** rcvbuf,
+ unsigned * pipewidth,
+ CCMI_Callback_t * cb_done);
///
/// \brief Set the buffer info for the allreduce collective
@@ -382,7 +405,7 @@ namespace CCMI
CCMI_Dt dt = CCMI_UNDEFINED_DT)
{
TRACE_INIT((stderr,"<%#.8X>Executor::AllreduceBase::setReduceInfo() "
- "count %#X, pipelineWidth %#X, sizeOfType %#X, func %#X, op %#X, dt %#X\n",
+ "count %#X, pipelineWidth %#X, sizeOfType %#X, func %#X, op %#X, dt %#X\n",
(int)this,count,pipelineWidth,sizeOfType,(int)func,op,dt));
CCMI_assert (pipelineWidth % sizeOfType == 0);
@@ -450,15 +473,15 @@ namespace CCMI
_nAsyncRcvd = 0;
}
- void operator delete(void * p)
+ void operator delete(void * p)
{
- ((AllreduceBase *) p)->_astate.freeAllocations ();
+ CCMI_abort();
}
///
/// \bf Query functions
///
- inline AllreduceState * getAllreduceState ()
+ inline AllreduceState * getAllreduceState ()
{
return &_astate;
}
@@ -799,3 +822,4 @@ CCMI::Kernel::Executor::AllreduceBase::notifyRecvHead
#endif /* __simple_allreduce_executor_h__ */
+
diff --git a/sys/collectives/kernel/executor/AllreduceState.h b/sys/collectives/kernel/executor/AllreduceState.h
index 2245d63..f182d21 100755
--- a/sys/collectives/kernel/executor/AllreduceState.h
+++ b/sys/collectives/kernel/executor/AllreduceState.h
@@ -532,6 +532,18 @@ namespace CCMI
TRACE_ALERT((stderr,"<%#.8X>Executor::AllreduceState::ctor ALERT: Constructor\n",(int)this));
TRACE_STATE((stderr,"<%#.8X>Executor::AllreduceState::ctor(void) exit\n",(int)this));
}
+ /// Default Destructor
+ virtual ~AllreduceState ()
+ {
+ TRACE_ALERT((stderr,"<%#.8X>Executor::AllreduceState::dtor ALERT: Destructor\n",(int)this));
+ freeAllocations();
+ }
+
+ /// NOTE: This is required to make "C" programs link successfully with virtual destructors
+ inline void operator delete(void * p)
+ {
+ CCMI_abort();
+ }
inline void setExecutor (Executor *exe)
{
@@ -580,7 +592,7 @@ namespace CCMI
else
{
unsigned lastChunk = bytes / _pipelineWidth + (bytes % _pipelineWidth != 0) - 1;
- if(lastChunk > _lastChunk)
+ if (lastChunk > _lastChunk)
{
TRACE_ALERT((stderr,"<%#.8X>Executor::AllreduceState::setDataFunc ALERT: Pipelining grew %#X > %#X\n",(int)this, lastChunk, _lastChunk));
_isConfigChanged = true;
diff --git a/sys/collectives/kernel/interfaces/Composite.h b/sys/collectives/kernel/interfaces/Composite.h
index 8326c29..fa174a7 100755
--- a/sys/collectives/kernel/interfaces/Composite.h
+++ b/sys/collectives/kernel/interfaces/Composite.h
@@ -18,11 +18,14 @@
#define MAX_EXECUTORS 6
-namespace CCMI {
- namespace Kernel {
- namespace Executor {
-
- class Composite {
+namespace CCMI
+{
+ namespace Kernel
+ {
+ namespace Executor
+ {
+ class Composite
+ {
protected:
///
/// \brief for synchronous protocols which need a barrier
@@ -39,28 +42,49 @@ namespace CCMI {
public:
- Composite () {
+ Composite ()
+ {
_barrier = NULL;
for (int count = 0; count < MAX_EXECUTORS; count ++)
_executors[count] = NULL;
_numExecutors = 0;
}
- void addExecutor (Executor *exec) {
+ /// Default Destructor
+ virtual ~Composite()
+ {
+ for (unsigned count = 0; count < _numExecutors; count ++)
+ {
+ _executors[count]->~Executor();
+ _executors[count] = NULL;
+ }
+ _numExecutors = 0;
+ }
+ /// NOTE: This is required to make "C" programs link successfully with virtual destructors
+ inline void operator delete(void * p)
+ {
+ CCMI_abort();
+ }
+
+ void addExecutor (Executor *exec)
+ {
CCMI_assert (_numExecutors < MAX_EXECUTORS);
_executors [_numExecutors] = exec;
_numExecutors ++;
}
- void addBarrier (Executor *exec) {
+ void addBarrier (Executor *exec)
+ {
_barrier = exec;
}
- Executor * getExecutor (int idx) {
+ Executor * getExecutor (int idx)
+ {
return _executors [idx];
}
- unsigned getNumExecutors () {
+ unsigned getNumExecutors ()
+ {
return _numExecutors;
}
diff --git a/sys/collectives/kernel/interfaces/Executor.h b/sys/collectives/kernel/interfaces/Executor.h
old mode 100644
new mode 100755
index 34070eb..7f6e03f
--- a/sys/collectives/kernel/interfaces/Executor.h
+++ b/sys/collectives/kernel/interfaces/Executor.h
@@ -11,27 +11,32 @@
* \brief ???
*/
#ifndef __executor_h__
-#define __executor_h__
+ #define __executor_h__
-#include "ccmi_internal.h"
-#include "Util.h"
+ #include "ccmi_internal.h"
+ #include "Util.h"
-namespace CCMI {
- namespace Kernel {
- namespace Executor {
+namespace CCMI
+{
+ namespace Kernel
+ {
+ namespace Executor
+ {
/**
* \brief Base Class for all Executors
**/
- class Executor {
+ class Executor
+ {
public:
/**
* \brief Constructor
*/
- Executor () {
+ Executor ()
+ {
_cb_done = NULL;
_clientdata = NULL;
_consistency = CCMI_UNDEFINED_CONSISTENCY;
@@ -40,13 +45,18 @@ namespace CCMI {
/**
* \brief Destructor
*/
- virtual ~Executor () {}
+ virtual ~Executor ()
+ {
+// _cb_done = NULL;
+// _clientdata = NULL;
+// _consistency = CCMI_UNDEFINED_CONSISTENCY;
+ }
/// NOTE: This is required to make "C" programs link successfully with virtual destructors
inline void operator delete(void * p)
- {
- CCMI_abort();
- }
+ {
+ CCMI_abort();
+ }
/**
* \brief Start method which is called when the collective
@@ -73,16 +83,19 @@ namespace CCMI {
virtual void notifyRecv (unsigned src, const DCQuad &info, char * buf, unsigned bytes) = 0;
- void setDoneCallback (void (*cb_done)(void *), void *cd) {
+ void setDoneCallback (void (*cb_done)(void *), void *cd)
+ {
_cb_done = cb_done;
_clientdata = cd;
}
- void setConsistency (CCMI_Consistency consistency) {
+ void setConsistency (CCMI_Consistency consistency)
+ {
_consistency = consistency;
}
- CCMI_Consistency getConsistency () {
+ CCMI_Consistency getConsistency ()
+ {
return _consistency;
}
@@ -97,8 +110,6 @@ namespace CCMI {
/// \brief Consistency required to perform the collective
///
CCMI_Consistency _consistency;
-
-
}; //-- Executor class
}; //-- Executor Name Space
}; //-- Kernel
diff --git a/sys/collectives/tests/dcmf/tree/tree_allreduce.cc b/sys/collectives/tests/dcmf/tree/tree_allreduce.cc
old mode 100644
new mode 100755
index 7d48af1..b739bc4
--- a/sys/collectives/tests/dcmf/tree/tree_allreduce.cc
+++ b/sys/collectives/tests/dcmf/tree/tree_allreduce.cc
@@ -169,7 +169,7 @@ int main(int argc, char **argv)
if (rank == 0)
- printf("initializing elements\n");
+ fprintf(stderr,"initializing elements\n");
#if 0
#define DTYPE_TEST unsigned
@@ -190,7 +190,7 @@ int main(int argc, char **argv)
if (rank == 0)
{
for (dt = 0; dt<1; dt++)
- printf("%d %d\n", inbuf[dt], outbuf[dt]);
+ fprintf(stderr,"%d %d\n", inbuf[dt], outbuf[dt]);
}
return 0;
#endif
@@ -205,7 +205,7 @@ int main(int argc, char **argv)
// Warmup
if (rank == 0)
- printf("Allreduce Bandwidth Test Warmup %s %s\n",
+ fprintf(stderr,"Allreduce Bandwidth Test Warmup %s %s\n",
dt_array_str[dt],
op_array_str[op]);
@@ -218,7 +218,7 @@ int main(int argc, char **argv)
if (rc)
{
if (rank == 0)
- printf("Skipped Unsupported Test %s %s\n",
+ fprintf(stderr,"Skipped Unsupported Test %s %s\n",
dt_array_str[dt],
op_array_str[op]);
@@ -227,8 +227,8 @@ int main(int argc, char **argv)
if (rank ==0)
{
- printf("Size(bytes) cycles bytes/cycle usec\n");
- printf("----------- ----------- ----------- ---------\n");
+ fprintf(stderr,"Size(bytes) cycles bytes/cycle usec\n");
+ fprintf(stderr,"----------- ----------- ----------- ---------\n");
}
for (i=1; i<=COUNT; i*=2)
{
@@ -249,7 +249,7 @@ int main(int argc, char **argv)
if (rank==0)
{
double usec = ((double)treport)/((double)clockMHZ);
- printf("%11lld %16lld %14f %12.2f\n",
+ fprintf(stderr,"%11lld %16lld %14f %12.2f\n",
dataSent,
treport,
(double)dataSent/(double)treport,
@@ -259,7 +259,7 @@ int main(int argc, char **argv)
}
}
gibarrier();
- if (rank==0) printf("SUCCESS\n");
+ if (rank==0) fprintf(stderr,"SUCCESS\n");
DCMF_Messager_finalize ();
return 0;
diff --git a/sys/collectives/tests/dcmf/tree/tree_reduce.cc b/sys/collectives/tests/dcmf/tree/tree_reduce.cc
old mode 100644
new mode 100755
index 555bc3d..dbaec1c
--- a/sys/collectives/tests/dcmf/tree/tree_reduce.cc
+++ b/sys/collectives/tests/dcmf/tree/tree_reduce.cc
@@ -169,7 +169,7 @@ int main(int argc, char **argv)
if (rank == 0)
- printf("initializing elements\n");
+ fprintf(stderr,"initializing elements\n");
#if 0
#define DTYPE_TEST unsigned
@@ -190,7 +190,7 @@ int main(int argc, char **argv)
if (rank == 0)
{
for (dt = 0; dt<1; dt++)
- printf("%d %d\n", inbuf[dt], outbuf[dt]);
+ fprintf(stderr,"%d %d\n", inbuf[dt], outbuf[dt]);
}
return 0;
#endif
@@ -205,7 +205,7 @@ int main(int argc, char **argv)
// Warmup
if (rank == 0)
- printf("Reduce Bandwidth Test Warmup %s %s\n",
+ fprintf(stderr,"Reduce Bandwidth Test Warmup %s %s\n",
dt_array_str[dt],
op_array_str[op]);
@@ -218,7 +218,7 @@ int main(int argc, char **argv)
if (rc)
{
if (rank == 0)
- printf("Skipped Unsupported Test %s %s\n",
+ fprintf(stderr,"Skipped Unsupported Test %s %s\n",
dt_array_str[dt],
op_array_str[op]);
@@ -227,8 +227,8 @@ int main(int argc, char **argv)
if (rank ==0)
{
- printf("Size(bytes) cycles bytes/cycle usec\n");
- printf("----------- ----------- ----------- ---------\n");
+ fprintf(stderr,"Size(bytes) cycles bytes/cycle usec\n");
+ fprintf(stderr,"----------- ----------- ----------- ---------\n");
}
for (i=1; i<=COUNT; i*=2)
{
@@ -249,7 +249,7 @@ int main(int argc, char **argv)
if (rank==0)
{
double usec = ((double)treport)/((double)clockMHZ);
- printf("%11lld %16lld %14f %12.2f\n",
+ fprintf(stderr,"%11lld %16lld %14f %12.2f\n",
dataSent,
treport,
(double)dataSent/(double)treport,
@@ -259,7 +259,7 @@ int main(int argc, char **argv)
}
}
gibarrier();
- if (rank==0) printf("SUCCESS\n");
+ if (rank==0) fprintf(stderr,"SUCCESS\n");
DCMF_Messager_finalize ();
return 0;
--
1.5.4
More information about the dcmf
mailing list