-:    0:Source:/home/MPI/testing/mpich2/mpich2/src/mpid/ch3/src/ch3u_recvq.c
        -:    0:Graph:ch3u_recvq.gcno
        -:    0:Data:ch3u_recvq.gcda
        -:    0:Runs:4080
        -:    0:Programs:1195
        -:    1:/* -*- Mode: C; c-basic-offset:4 ; -*- */
        -:    2:/*
        -:    3: *  (C) 2001 by Argonne National Laboratory.
        -:    4: *      See COPYRIGHT in top-level directory.
        -:    5: */
        -:    6:
        -:    7:#include "mpidimpl.h"
        -:    8:
        -:    9:/* These are needed for nemesis to know from where to expect a message */
        -:   10:#ifndef MPIDI_POSTED_RECV_ENQUEUE_HOOK
        -:   11:#define MPIDI_POSTED_RECV_ENQUEUE_HOOK(x)
        -:   12:#endif
        -:   13:#ifndef MPIDI_POSTED_RECV_DEQUEUE_HOOK
        -:   14:#define MPIDI_POSTED_RECV_DEQUEUE_HOOK(x)
        -:   15:#endif
        -:   16:
        -:   17:/* FIXME: 
        -:   18: * Recvq_lock/unlock removed because it is not needed for the SINGLE_CS
        -:   19: * approach and we might want a different, non-lock-based approach in 
        -:   20: * a finer-grained thread-sync version.  For example, 
        -:   21: * some routines can be implemented in a lock-free
        -:   22: * fashion (because the user is required to guarantee non-conflicting 
        -:   23: * accesses, such as doing a probe and a receive that matches in different 
        -:   24: * threads).  
        -:   25: *
        -:   26: * There are a lot of routines here.  Do we really need them all?
        -:   27: * 
        -:   28: * The search criteria can be implemented in a single 64 bit compare on
        -:   29: * systems with efficient 64-bit operations.  The rank and contextid can also
        -:   30: * in many cases be combined into a single 32-bit word for the comparison
        -:   31: * (in which case the message info should be stored in the queue in a 
        -:   32: * naturally aligned, 64-bit word.
        -:   33: * 
        -:   34: */
        -:   35:
        -:   36:static MPID_Request * recvq_posted_head = 0;
        -:   37:static MPID_Request * recvq_posted_tail = 0;
        -:   38:static MPID_Request * recvq_unexpected_head = 0;
        -:   39:static MPID_Request * recvq_unexpected_tail = 0;
        -:   40:
        -:   41:/* Export the location of the queue heads if debugger support is enabled.
        -:   42: * This allows the queue code to rely on the local variables for the
        -:   43: * queue heads while also exporting those variables to the debugger.
        -:   44: * See src/mpi/debugger/dll_mpich2.c for how this is used to 
        -:   45: * access the message queues.
        -:   46: */
        -:   47:#ifdef HAVE_DEBUGGER_SUPPORT
        -:   48:MPID_Request ** const MPID_Recvq_posted_head_ptr     = &recvq_posted_head;
        -:   49:MPID_Request ** const MPID_Recvq_unexpected_head_ptr = &recvq_unexpected_head;
        -:   50:#endif
        -:   51:
        -:   52:/* If the MPIDI_Message_match structure fits into a pointer size, we
        -:   53: * can directly work on it */
        -:   54:/* MATCH_WITH_NO_MASK compares the match values without masking
        -:   55: * them. This is useful for the case where there are no ANY_TAG or
        -:   56: * ANY_SOURCE wild cards. */
        -:   57:#define MATCH_WITH_NO_MASK(match1, match2) \
        -:   58:    ((sizeof(MPIDI_Message_match) == SIZEOF_VOID_P && 0) ? ((match1).whole == (match2).whole) : \
        -:   59:     (((match1).parts.rank == (match2).parts.rank) && \
        -:   60:      ((match1).parts.tag == (match2).parts.tag) && \
        -:   61:      ((match1).parts.context_id == (match2).parts.context_id)))
        -:   62:
        -:   63:/* MATCH_WITH_LEFT_MASK compares the match values after masking only
        -:   64: * the left field. This is useful for the case where the right match
        -:   65: * is a part of the unexpected queue and has no ANY_TAG or ANY_SOURCE
        -:   66: * wild cards, but the left match might have them. */
        -:   67:#define MATCH_WITH_LEFT_MASK(match1, match2, mask) \
        -:   68:    ((sizeof(MPIDI_Message_match) == SIZEOF_VOID_P && 0) ? (((match1).whole & (mask).whole) == (match2).whole) : \
        -:   69:     ((((match1).parts.rank & (mask).parts.rank) == (match2).parts.rank) && \
        -:   70:      (((match1).parts.tag & (mask).parts.tag) == (match2).parts.tag) && \
        -:   71:      ((match1).parts.context_id == (match2).parts.context_id)))
        -:   72:
        -:   73:/* This is the most general case where both matches have to be
        -:   74: * masked. Both matches are masked with the same value. There doesn't
        -:   75: * seem to be a need for two different masks at this time. */
        -:   76:#define MATCH_WITH_LEFT_RIGHT_MASK(match1, match2, mask) \
        -:   77:    ((sizeof(MPIDI_Message_match) == SIZEOF_VOID_P && 0) ? (((match1).whole & (mask).whole) == ((match2).whole & (mask).whole)) : \
        -:   78:     ((((match1).parts.rank & (mask).parts.rank) == ((match2).parts.rank & (mask).parts.rank)) && \
        -:   79:      (((match1).parts.tag & (mask).parts.tag) == ((match2).parts.tag & (mask).parts.tag)) && \
        -:   80:      ((match1).parts.context_id == (match2).parts.context_id)))
        -:   81:
        -:   82:
        -:   83:/* FIXME: If this routine is only used by probe/iprobe, then we don't need
        -:   84:   to set the cancelled field in status (only set for nonblocking requests) */
        -:   85:/*
        -:   86: * MPIDI_CH3U_Recvq_FU()
        -:   87: *
        -:   88: * Search for a matching request in the unexpected receive queue.  Return 
        -:   89: * true if one is found, false otherwise.  If the status arguement is
        -:   90: * not MPI_STATUS_IGNORE, return information about the request in that
        -:   91: * parameter.  This routine is used by mpid_probe and mpid_iprobe.
        -:   92: *
        -:   93: * Multithread - As this is a read-only routine, it need not
        -:   94: * require an external critical section (careful organization of the
        -:   95: * queue updates would not even require a critical section within this
        -:   96: * routine).  However, this routine is used both from within the progress
        -:   97: * engine and from without it.  To make that work with the current
        -:   98: * design for MSGQUEUE and the brief-global mode, the critical section 
        -:   99: * is *outside* of this routine.
        -:  100: *
        -:  101: * This routine is used only in mpid_iprobe and mpid_probe
        -:  102: *
        -:  103: */
        -:  104:#undef FUNCNAME
        -:  105:#define FUNCNAME MPIDI_CH3U_Recvq_FU
        -:  106:#undef FCNAME
        -:  107:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  108:int MPIDI_CH3U_Recvq_FU(int source, int tag, int context_id, MPI_Status *s)
  7491014:  109:{
        -:  110:    MPID_Request * rreq;
  7491014:  111:    int found = 0;
        -:  112:    MPIDI_Message_match match, mask;
        -:  113:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
        -:  114:
        -:  115:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FU);
        -:  116:
  7491014:  117:    rreq = recvq_unexpected_head;
        -:  118:
  7491014:  119:    match.parts.context_id = context_id;
  7491014:  120:    match.parts.tag = tag;
  7491014:  121:    match.parts.rank = source;
        -:  122:
  7491014:  123:    if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
 14439694:  124:	while (rreq != NULL) {
  7212632:  125:	    if (MATCH_WITH_NO_MASK(rreq->dev.match, match))
     1093:  126:		break;
  7211539:  127:	    rreq = rreq->dev.next;
        -:  128:	}
        -:  129:    }
        -:  130:    else {
   262859:  131:	mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
   262859:  132:	if (tag == MPI_ANY_TAG)
    45762:  133:	    match.parts.tag = mask.parts.tag = 0;
   262859:  134:	if (source == MPI_ANY_SOURCE)
   241165:  135:	    match.parts.rank = mask.parts.rank = 0;
        -:  136:
   415950:  137:	while (rreq != NULL) {
   154395:  138:	    if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask))
     1304:  139:		break;
   153091:  140:	    rreq = rreq->dev.next;
        -:  141:	}
        -:  142:    }
        -:  143:
        -:  144:    /* Save the information about the request before releasing the 
        -:  145:       queue */
  7491014:  146:    if (rreq) {
     2397:  147:	if (s != MPI_STATUS_IGNORE) {
        -:  148:	    /* Avoid setting "extra" fields like MPI_ERROR */
     2383:  149:	    s->MPI_SOURCE = rreq->status.MPI_SOURCE;
     2383:  150:	    s->MPI_TAG    = rreq->status.MPI_TAG;
     2383:  151:	    s->count      = rreq->status.count;
     2383:  152:	    s->cancelled  = rreq->status.cancelled;
        -:  153:	}
     2397:  154:	found = 1;
        -:  155:    }
        -:  156:

        -:  157:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FU);

  7491014:  158:    return found;
        -:  159:}
        -:  160:
        -:  161:/*
        -:  162: * MPIDI_CH3U_Recvq_FDU()
        -:  163: *
        -:  164: * Find a request in the unexpected queue and dequeue it; otherwise return NULL.
        -:  165: *
        -:  166: * Multithread - This routine must be atomic (since it dequeues a
        -:  167: * request).  However, once the request is dequeued, no other thread can
        -:  168: * see it, so this routine provides its own atomicity.
        -:  169: *
        -:  170: * This routine is used only in the case of send_cancel.  However, it is used both
        -:  171: * within mpid_send_cancel and within a packet handler.
        -:  172: */
        -:  173:#undef FUNCNAME
        -:  174:#define FUNCNAME MPIDI_CH3U_Recvq_FDU
        -:  175:#undef FCNAME
        -:  176:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  177:MPID_Request * MPIDI_CH3U_Recvq_FDU(MPI_Request sreq_id, 
        -:  178:				    MPIDI_Message_match * match)
      475:  179:{
        -:  180:    MPID_Request * rreq;
        -:  181:    MPID_Request * prev_rreq;
        -:  182:    MPID_Request * cur_rreq;
        -:  183:    MPID_Request * matching_prev_rreq;
        -:  184:    MPID_Request * matching_cur_rreq;
        -:  185:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
        -:  186:
        -:  187:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);
        -:  188:
      475:  189:    matching_prev_rreq = NULL;
      475:  190:    matching_cur_rreq = NULL;
      475:  191:    prev_rreq = NULL;
        -:  192:
        -:  193:    /* Note that since this routine is used only in the case of send_cancel,
        -:  194:       there can be only one match if at all. */
        -:  195:    /* FIXME: Why doesn't this exit after it finds the first match? */
      475:  196:    cur_rreq = recvq_unexpected_head;
     1417:  197:    while (cur_rreq != NULL) {
      467:  198:	if (cur_rreq->dev.sender_req_id == sreq_id &&
        -:  199:	    (MATCH_WITH_NO_MASK(cur_rreq->dev.match, *match))) {
      467:  200:	    matching_prev_rreq = prev_rreq;
      467:  201:	    matching_cur_rreq = cur_rreq;
        -:  202:	}
      467:  203:	prev_rreq = cur_rreq;
      467:  204:	cur_rreq = cur_rreq->dev.next;
        -:  205:    }
        -:  206:
      475:  207:    if (matching_cur_rreq != NULL) {
      467:  208:	if (matching_prev_rreq != NULL) {

    #####:  209:	    matching_prev_rreq->dev.next = matching_cur_rreq->dev.next;
        -:  210:	}
        -:  211:	else {

      467:  212:	    recvq_unexpected_head = matching_cur_rreq->dev.next;
        -:  213:	}
        -:  214:	
      467:  215:	if (matching_cur_rreq->dev.next == NULL) {
      467:  216:	    recvq_unexpected_tail = matching_prev_rreq;
        -:  217:	}
        -:  218:
      467:  219:	rreq = matching_cur_rreq;
        -:  220:    }
        -:  221:    else {
        8:  222:	rreq = NULL;
        -:  223:    }
        -:  224:

        -:  225:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU);

      475:  226:    return rreq;
        -:  227:}
        -:  228:
        -:  229:
        -:  230:/*
        -:  231: * MPIDI_CH3U_Recvq_FDU_or_AEP()
        -:  232: *
        -:  233: * Atomically find a request in the unexpected queue and dequeue it, or 
        -:  234: * allocate a new request and enqueue it in the posted queue
        -:  235: *
        -:  236: * Multithread - This routine must be called from within a MSGQUEUE 
        -:  237: * critical section.  If a request is allocated, it must not release
        -:  238: * the MSGQUEUE until the request is completely valid, as another thread
        -:  239: * may then find it and dequeue it.
        -:  240: *
        -:  241: * This routine is used in mpid_irecv and mpid_recv.
        -:  242: *
        -:  243: */
        -:  244:#undef FUNCNAME
        -:  245:#define FUNCNAME MPIDI_CH3U_Recvq_FDU_or_AEP
        -:  246:#undef FCNAME
        -:  247:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  248:MPID_Request * MPIDI_CH3U_Recvq_FDU_or_AEP(int source, int tag, 
        -:  249:                                           int context_id, MPID_Comm *comm, void *user_buf,
        -:  250:                                           int user_count, MPI_Datatype datatype, int * foundp)
 11375436:  251:{
        -:  252:    int found;
        -:  253:    MPID_Request *rreq, *prev_rreq;
        -:  254:    MPIDI_Message_match match;
        -:  255:    MPIDI_Message_match mask;
        -:  256:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
        -:  257:
        -:  258:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);
        -:  259:
        -:  260:    /* Optimize this loop for an empty unexpected receive queue */
 11375436:  261:    rreq = recvq_unexpected_head;
 11375436:  262:    if (rreq) {
  3535472:  263:	prev_rreq = NULL;
        -:  264:
  3535472:  265:	match.parts.context_id = context_id;
  3535472:  266:	match.parts.tag = tag;
  3535472:  267:	match.parts.rank = source;
        -:  268:
  3535472:  269:	if (tag != MPI_ANY_TAG && source != MPI_ANY_SOURCE) {
        -:  270:	    do {
  3498702:  271:		if (MATCH_WITH_NO_MASK(rreq->dev.match, match)) {
  2652071:  272:		    if (prev_rreq != NULL) {
    86558:  273:			prev_rreq->dev.next = rreq->dev.next;
        -:  274:		    }
        -:  275:		    else {
  2565513:  276:			recvq_unexpected_head = rreq->dev.next;
        -:  277:		    }
        -:  278:
  2652071:  279:		    if (rreq->dev.next == NULL) {
  2320349:  280:			recvq_unexpected_tail = prev_rreq;
        -:  281:		    }
        -:  282:
  2652071:  283:		    rreq->comm = comm;
  2652071:  284:		    MPIR_Comm_add_ref(comm);
  2652071:  285:		    rreq->dev.user_buf = user_buf;
  2652071:  286:		    rreq->dev.user_count = user_count;
  2652071:  287:		    rreq->dev.datatype = datatype;
  2652071:  288:		    found = TRUE;
  2652071:  289:		    goto lock_exit;
        -:  290:		}
   846631:  291:		prev_rreq = rreq;
   846631:  292:		rreq      = rreq->dev.next;
   846631:  293:	    } while (rreq);
        -:  294:	}
        -:  295:	else {
   192908:  296:	    mask.parts.context_id = mask.parts.rank = mask.parts.tag = ~0;
   192908:  297:	    if (tag == MPI_ANY_TAG)
   157894:  298:		match.parts.tag = mask.parts.tag = 0;
   192908:  299:	    if (source == MPI_ANY_SOURCE)
    84960:  300:		match.parts.rank = mask.parts.rank = 0;
        -:  301:
        -:  302:	    do {
   237747:  303:		if (MATCH_WITH_LEFT_MASK(rreq->dev.match, match, mask)) {
   152046:  304:		    if (prev_rreq != NULL) {
    33913:  305:			prev_rreq->dev.next = rreq->dev.next;
        -:  306:		    }
        -:  307:		    else {
   118133:  308:			recvq_unexpected_head = rreq->dev.next;
        -:  309:		    }
   152046:  310:		    if (rreq->dev.next == NULL) {
   114070:  311:			recvq_unexpected_tail = prev_rreq;
        -:  312:		    }
   152046:  313:		    rreq->comm                 = comm;
   152046:  314:		    MPIR_Comm_add_ref(comm);
   152046:  315:		    rreq->dev.user_buf         = user_buf;
   152046:  316:		    rreq->dev.user_count       = user_count;
   152046:  317:		    rreq->dev.datatype         = datatype;
   152046:  318:		    found = TRUE;
   152046:  319:		    goto lock_exit;
        -:  320:		}
    85701:  321:		prev_rreq = rreq;
    85701:  322:		rreq = rreq->dev.next;
    85701:  323:	    } while (rreq);
        -:  324:	}
        -:  325:    }
        -:  326:    
        -:  327:    /* A matching request was not found in the unexpected queue, so we 
        -:  328:       need to allocate a new request and add it to the posted queue */
        -:  329:    {
  8571319:  330:	int mpi_errno=0;
  8571319:  331:	MPIDI_Request_create_rreq( rreq, mpi_errno, 
        -:  332:				   found = FALSE;goto lock_exit );
  8571319:  333:	rreq->dev.match.parts.tag	   = tag;
  8571319:  334:	rreq->dev.match.parts.rank	   = source;
  8571319:  335:	rreq->dev.match.parts.context_id   = context_id;
        -:  336:
        -:  337:	/* Added a mask for faster search on 64-bit capable
        -:  338:	 * platforms */
  8571319:  339:	rreq->dev.mask.parts.context_id = ~0;
  8571319:  340:	if (rreq->dev.match.parts.rank == MPI_ANY_SOURCE)
   114316:  341:	    rreq->dev.mask.parts.rank = 0;
        -:  342:	else
  8457003:  343:	    rreq->dev.mask.parts.rank = ~0;
  8571319:  344:	if (rreq->dev.match.parts.tag == MPI_ANY_TAG)
   242584:  345:	    rreq->dev.mask.parts.tag = 0;
        -:  346:	else
  8328735:  347:	    rreq->dev.mask.parts.tag = ~0;
        -:  348:
  8571319:  349:        rreq->comm                 = comm;
  8571319:  350:        MPIR_Comm_add_ref(comm);
  8571319:  351:        rreq->dev.user_buf         = user_buf;
  8571319:  352:        rreq->dev.user_count       = user_count;
  8571319:  353:        rreq->dev.datatype         = datatype;
  8571319:  354:	rreq->dev.next		   = NULL;
  8571319:  355:	if (recvq_posted_tail != NULL) {
   186803:  356:	    recvq_posted_tail->dev.next = rreq;
        -:  357:	}
        -:  358:	else {
  8384516:  359:	    recvq_posted_head = rreq;
        -:  360:	}
  8571319:  361:	recvq_posted_tail = rreq;
        -:  362:	/* This is for nemesis to know from where to expect a message */
  8571319:  363:	MPIDI_POSTED_RECV_ENQUEUE_HOOK (rreq);
        -:  364:    }
        -:  365:    
  8571319:  366:    found = FALSE;
        -:  367:
 11375436:  368:  lock_exit:
        -:  369:
 11375436:  370:    *foundp = found;
        -:  371:    

        -:  372:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDU_OR_AEP);

 11375436:  373:    return rreq;
        -:  374:}
        -:  375:
        -:  376:
        -:  377:/*
        -:  378: * MPIDI_CH3U_Recvq_DP()
        -:  379: *
        -:  380: * Given an existing request, dequeue that request from the posted queue, or 
        -:  381: * return NULL if the request was not in the posted queued
        -:  382: *
        -:  383: * Multithread - This routine is atomic
        -:  384: */
        -:  385:#undef FUNCNAME
        -:  386:#define FUNCNAME MPIDI_CH3U_Recvq_DP
        -:  387:#undef FCNAME
        -:  388:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  389:int MPIDI_CH3U_Recvq_DP(MPID_Request * rreq)
     2292:  390:{
        -:  391:    int found;
        -:  392:    MPID_Request * cur_rreq;
        -:  393:    MPID_Request * prev_rreq;
        -:  394:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
        -:  395:
        -:  396:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_DP);
        -:  397:    
     2292:  398:    found = FALSE;
     2292:  399:    prev_rreq = NULL;
        -:  400:    
        -:  401:    MPIU_THREAD_CS_ENTER(MSGQUEUE,);
     2292:  402:    cur_rreq = recvq_posted_head;
     4589:  403:    while (cur_rreq != NULL) {
     2281:  404:	if (cur_rreq == rreq) {
     2276:  405:	    if (prev_rreq != NULL) {
        3:  406:		prev_rreq->dev.next = cur_rreq->dev.next;
        -:  407:	    }
        -:  408:	    else {
     2273:  409:		recvq_posted_head = cur_rreq->dev.next;
        -:  410:	    }
     2276:  411:	    if (cur_rreq->dev.next == NULL) {
      226:  412:		recvq_posted_tail = prev_rreq;
        -:  413:	    }
        -:  414:	    /* This is for nemesis to know from where to expect a message */
     2276:  415:	    MPIDI_POSTED_RECV_DEQUEUE_HOOK (rreq);
     2276:  416:	    found = TRUE;
     2276:  417:	    break;
        -:  418:	}
        -:  419:	
        5:  420:	prev_rreq = cur_rreq;
        5:  421:	cur_rreq = cur_rreq->dev.next;
        -:  422:    }
        -:  423:
        -:  424:    MPIU_THREAD_CS_EXIT(MSGQUEUE,);
        -:  425:

        -:  426:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_DP);

     2292:  427:    return found;
        -:  428:}
        -:  429:
        -:  430:/*
        -:  431: * MPIDI_CH3U_Recvq_FDP_or_AEU()
        -:  432: *
        -:  433: * Locate a request in the posted queue and dequeue it, or allocate a new 
        -:  434: * request and enqueue it in the unexpected queue
        -:  435: *
        -:  436: * Multithread - This routine must be called from within a MSGQUEUE 
        -:  437: * critical section.  If a request is allocated, it must not release
        -:  438: * the MSGQUEUE until the request is completely valid, as another thread
        -:  439: * may then find it and dequeue it.
        -:  440: *
        -:  441: * This routine is used in ch3u_eager, ch3u_eagersync, ch3u_handle_recv_pkt,
        -:  442: * ch3u_rndv, and mpidi_isend_self.  Routines within the progress engine
        -:  443: * will need to be careful to avoid nested critical sections.  
        -:  444: *
        -:  445: * FIXME: Currently, the routines called from within the progress engine
        -:  446: * do not use the MSGQUEUE CS, because in the brief-global mode, that
        -:  447: * simply uses the global_mutex .  
        -:  448: */
        -:  449:#undef FUNCNAME
        -:  450:#define FUNCNAME MPIDI_CH3U_Recvq_FDP_or_AEU
        -:  451:#undef FCNAME
        -:  452:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  453:MPID_Request * MPIDI_CH3U_Recvq_FDP_or_AEU(MPIDI_Message_match * match, 
        -:  454:					   int * foundp)
 11373628:  455:{
        -:  456:    int found;
        -:  457:    MPID_Request * rreq;
        -:  458:    MPID_Request * prev_rreq;
        -:  459:    MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
        -:  460:
        -:  461:    MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);
        -:  462:    
 11373628:  463:    prev_rreq = NULL;
        -:  464:
 11373628:  465:    rreq = recvq_posted_head;
        -:  466:
 24042447:  467:    while (rreq != NULL) {
  9864234:  468:	if (MATCH_WITH_LEFT_RIGHT_MASK(rreq->dev.match, *match, rreq->dev.mask)) {
  8569043:  469:	    if (prev_rreq != NULL) {
   160823:  470:		prev_rreq->dev.next = rreq->dev.next;
        -:  471:	    }
        -:  472:	    else {
  8408220:  473:		recvq_posted_head = rreq->dev.next;
        -:  474:	    }
  8569043:  475:	    if (rreq->dev.next == NULL) {
  8540988:  476:		recvq_posted_tail = prev_rreq;
        -:  477:	    }
  8569043:  478:	    found = TRUE;
        -:  479:	    /* This is for the channel to know from where to expect a
        -:  480:	     * message */
  8569043:  481:	    MPIDI_POSTED_RECV_DEQUEUE_HOOK (rreq);
  8569043:  482:	    goto lock_exit;
        -:  483:	}
  1295191:  484:	prev_rreq = rreq;
  1295191:  485:	rreq = rreq->dev.next;
        -:  486:    }
        -:  487:
        -:  488:    /* A matching request was not found in the posted queue, so we 
        -:  489:       need to allocate a new request and add it to the unexpected queue */
        -:  490:    {
  2804585:  491:	int mpi_errno=0;
  2804585:  492:	MPIDI_Request_create_rreq( rreq, mpi_errno, 
        -:  493:				   found=FALSE;goto lock_exit );
  2804585:  494:	rreq->dev.match	= *match;
  2804585:  495:	rreq->dev.next	= NULL;
  2804585:  496:	if (recvq_unexpected_tail != NULL) {
   474540:  497:	    recvq_unexpected_tail->dev.next = rreq;
        -:  498:	}
        -:  499:	else {
  2330045:  500:	    recvq_unexpected_head = rreq;
        -:  501:	}
  2804585:  502:	recvq_unexpected_tail = rreq;
        -:  503:    }
        -:  504:    
  2804585:  505:    found = FALSE;
        -:  506:
 11373628:  507:  lock_exit:
        -:  508:
 11373628:  509:    *foundp = found;
        -:  510:

        -:  511:    MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECVQ_FDP_OR_AEU);

 11373628:  512:    return rreq;
        -:  513:}
        -:  514:

        -:  515:/* --BEGIN ERROR HANDLING-- */
        -:  516:/* pretty prints tag, returns out for calling convenience */
        -:  517:static char *tag_val_to_str(int tag, char *out, int max)
    #####:  518:{
    #####:  519:    if (tag == MPI_ANY_TAG) {
    #####:  520:        MPIU_Strncpy(out, "MPI_ANY_TAG", max);
        -:  521:    }
        -:  522:    else {
    #####:  523:        MPIU_Snprintf(out, max, "%d", tag);
        -:  524:    }
    #####:  525:    return out;
        -:  526:}
        -:  527:
        -:  528:/* pretty prints rank, returns out for calling convenience */
        -:  529:static char *rank_val_to_str(int rank, char *out, int max)
    #####:  530:{
    #####:  531:    if (rank == MPI_ANY_SOURCE) {
    #####:  532:        MPIU_Strncpy(out, "MPI_ANY_SOURCE", max);
        -:  533:    }
        -:  534:    else {
    #####:  535:        MPIU_Snprintf(out, max, "%d", rank);
        -:  536:    }
    #####:  537:    return out;
        -:  538:}
        -:  539:
        -:  540:/* satisfy the compiler */
        -:  541:void MPIDI_CH3U_Dbg_print_recvq(FILE *stream);
        -:  542:
        -:  543:/* This function can be called by a debugger to dump the recvq state to the
        -:  544: * given stream. */
        -:  545:#undef FUNCNAME
        -:  546:#define FUNCNAME MPIDI_CH3U_Dbg_print_recvq
        -:  547:#undef FCNAME
        -:  548:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  549:void MPIDI_CH3U_Dbg_print_recvq(FILE *stream)
    #####:  550:{
        -:  551:    MPID_Request * rreq;
        -:  552:    int i;
        -:  553:    char tag_buf[128];
        -:  554:    char rank_buf[128];
        -:  555:
    #####:  556:    fprintf(stream, "========================================\n");
    #####:  557:    fprintf(stream, "MPI_COMM_WORLD  ctx=%#x rank=%d\n", MPIR_Process.comm_world->context_id, MPIR_Process.comm_world->rank);
    #####:  558:    fprintf(stream, "MPI_COMM_SELF   ctx=%#x\n", MPIR_Process.comm_self->context_id);
    #####:  559:    if (MPIR_Process.comm_parent) {
    #####:  560:        fprintf(stream, "MPI_COMM_PARENT ctx=%#x recvctx=%#x\n",
        -:  561:                MPIR_Process.comm_self->context_id,
        -:  562:                MPIR_Process.comm_parent->recvcontext_id);
        -:  563:    }
        -:  564:    else {
    #####:  565:        fprintf(stream, "MPI_COMM_PARENT (NULL)\n");
        -:  566:    }
        -:  567:
    #####:  568:    fprintf(stream, "CH3 Posted RecvQ:\n");
    #####:  569:    rreq = recvq_posted_head;
    #####:  570:    i = 0;
    #####:  571:    while (rreq != NULL) {
    #####:  572:        fprintf(stream, "..[%d] rreq=%p ctx=%#x rank=%s tag=%s\n", i, rreq,
        -:  573:                        rreq->dev.match.parts.context_id,
        -:  574:                        rank_val_to_str(rreq->dev.match.parts.rank, rank_buf, sizeof(rank_buf)),
        -:  575:                        tag_val_to_str(rreq->dev.match.parts.tag, tag_buf, sizeof(tag_buf)));
    #####:  576:        ++i;
    #####:  577:        rreq = rreq->dev.next;
        -:  578:    }
        -:  579:
    #####:  580:    fprintf(stream, "CH3 Unexpected RecvQ:\n");
    #####:  581:    rreq = recvq_unexpected_head;
    #####:  582:    i = 0;
    #####:  583:    while (rreq != NULL) {
    #####:  584:        fprintf(stream, "..[%d] rreq=%p ctx=%#x rank=%s tag=%s\n", i, rreq,
        -:  585:                        rreq->dev.match.parts.context_id,
        -:  586:                        rank_val_to_str(rreq->dev.match.parts.rank, rank_buf, sizeof(rank_buf)),
        -:  587:                        tag_val_to_str(rreq->dev.match.parts.tag, tag_buf, sizeof(tag_buf)));
    #####:  588:        fprintf(stream, "..    status.src=%s status.tag=%s\n",
        -:  589:                        rank_val_to_str(rreq->status.MPI_SOURCE, rank_buf, sizeof(rank_buf)),
        -:  590:                        tag_val_to_str(rreq->status.MPI_TAG, tag_buf, sizeof(tag_buf)));
    #####:  591:        ++i;
    #####:  592:        rreq = rreq->dev.next;
        -:  593:    }
    #####:  594:    fprintf(stream, "========================================\n");
    #####:  595:}
        -:  596:/* --END ERROR HANDLING-- */
        -:  597:
        -:  598:/* returns the number of elements in the unexpected queue */
        -:  599:#undef FUNCNAME
        -:  600:#define FUNCNAME MPIDI_CH3U_Recvq_count_unexp
        -:  601:#undef FCNAME
        -:  602:#define FCNAME MPIDI_QUOTE(FUNCNAME)
        -:  603:int MPIDI_CH3U_Recvq_count_unexp(void)

    #####:  604:{
    #####:  605:    int count = 0;
    #####:  606:    MPID_Request *req = recvq_unexpected_head;
        -:  607:
    #####:  608:    while (req)
        -:  609:    {
    #####:  610:        ++count;
    #####:  611:        req = req->dev.next;
        -:  612:    }
        -:  613:
    #####:  614:    return count;
        -:  615:}