#include <stdio.h>
#include <stdlib.h>
#include <malloc.h>
#include <string.h>
#include <time.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/utsname.h>
#include <unistd.h>
#include <mpi.h>

#define MSG_NTEST	      (50000)
#define MSG_BEG_SIZE                1
#define MSG_END_SIZE          2097152
#define MSG_INCR_FAC                2
#define MSG_TARG_TIME             5.0
#define MEGA_BYTE             1048576
#define TRUTH(rank,i)         (char)(((rank + 1)*i) % 256)
#define INDEX_T                   int
#define DATA_T                   char
#define MPIDATA_T            MPI_BYTE

#define swap_vector(a,b)  {swap_buf = a; a = b; b = swap_buf;}

#define VERBOSE  (1<<0)
#define DOERR    (1<<1)
#define DOBUF    (1<<2)
#define STRIDE   (1<<3)
#define DOMAX    (1<<4)
#define DOAVG    (1<<5)
#define DOMEM    (1<<6)
#define BSBR1    (1<<7)
#define ISBR1    (1<<8)
#define BSIR1    (1<<9)
#define ISIR1    (1<<10)
#define BSBR2    (1<<11)
#define ISBR2    (1<<12)
#define BSIR2    (1<<13)
#define ISIR2    (1<<14)
#define IRIS2    (1<<15)
#define IRIS1    (1<<16)
#define SHIF1    (1<<17)
#define DENSE    (1<<18)
#define USERP    (1<<19)
#define STRID    (1<<20)
#define VECTR    (1<<21)
#define WORST    (1<<22)


long int flags=0;
double *swap_buf;

int print_mach_info();
double p2p(char *tag, int nmsg, int msglen, int me, int you, int dpat,
	   int cpat, int stride_pad, int stride_block);

int main(int argc, char *argv[])
{

 int    	me, you, size, *perm, flags=0;
 INDEX_T 	i,j, msglen, nmsg, nmsg_test=MSG_NTEST;
 INDEX_T        stride_pad, stride_block ;
 double		t_test;


 MPI_Init(&argc, &argv);
 MPI_Comm_rank(MPI_COMM_WORLD, &me);
 MPI_Comm_size(MPI_COMM_WORLD, &size);

 if(size%2) {
  printf("MPI_Comm_size must be an even integer (%d)\n",size);
  MPI_Finalize();
  exit(1);
 }

 if(!me) print_mach_info();

 if(argc==-1) {
  printf("\n");
  printf("usage: p2p [options] [size]\n");
  printf("  -v 			    verbose output\n");
  printf("  -mem 		    report memory usage\n");
  printf("  -late		    report latency\n");
  printf("  -stride pad,[block]	    data is block(default 1) elements\n");
  printf("                          then skipping pad elements\n");
  printf("  -oa                     alternating task order\n");
  printf("  -ow                     wrapped task order\n");
  printf("  -or                     random task order\n");
 }

/* parse commandline args */

 perm = (int *)malloc((size_t)size*sizeof(int));

 for(i=0;i<size;i++) { perm[i] = (i%2)?(i-1):(i+1); }

 
 while(--argc && argv++) {
  if(!strcmp("-v",*argv) ) {
   flags |= VERBOSE; 
  } else if(!strcmp("-err",*argv) ) {
   flags |= DOERR;
  } else if(!strcmp("-mem", *argv) ) {
   flags |= DOMEM;
  } else if(!strcmp("-stride", *argv) ) {
   --argc; argv++;
   flags |= STRID;
   if(!strstr(*argv,",")) {
    stride_block = 1;
    stride_pad = atoi(*argv);
   } else {
    sscanf(*argv,"%d,%d",&stride_pad,&stride_block);
   }
  }
  if(!strcmp(*argv,"-oa")) { /* alternating pattern -- best smp topo */
   for(i=0;i<size;i++) { perm[i] = (i%2)?(i-1):(i+1); }
  }
  if(!strcmp(*argv,"-ow")) { /* wrap around pattern -- worst smp topo */
   for(i=0;i<size;i++) { perm[i] =  (i < size/2)?(i+size/2):(i-size/2); }
  }
  if(!strcmp(*argv,"-or")) { /* musical chairs algorithm */
   if(!me) {
    srand(time(0));
    for(i=0;i<size;i++) {perm[i] = -1;}
    for(i=0;i<size;i++) {
     while(perm[i]==-1) {
      j = (size*rand())/RAND_MAX;
      if(j!=i && perm[j]==-1) {
       perm[i] = j; perm[j] = i; break;
      }
     }
    }
   }
   MPI_Bcast(perm, size, MPI_INT, 0, MPI_COMM_WORLD);
  }

 }

 you = perm[me];

 if(!me) {
  printf("# topo    %d : ",size);
  for(i=0;i<size/2;i++) {
   printf("(%d,%d) ",i,perm[i]);
  }
  printf("\n"); fflush(stdout);
 }
 if(!me) printf("# mpi     tasks   bandw(MB/s) latency(us) time(s)       bytes    nmsg  pad blk\n");


 nmsg = nmsg_test;
 for (msglen=MSG_BEG_SIZE; msglen<=MSG_END_SIZE; msglen *= MSG_INCR_FAC){
  t_test = p2p(0,nmsg,msglen,me,you,DENSE,BSBR2,0,0);
  nmsg =  (MSG_TARG_TIME / (t_test)) * (nmsg);
  MPI_Bcast(&nmsg, 1, MPI_INT, 0, MPI_COMM_WORLD);
  fflush(stdout);
  p2p("BSBR2",nmsg,msglen,me,you,DENSE,BSBR2,0,0);
  p2p("BSBR1",nmsg,msglen,me,you,DENSE,BSBR1,0,0);
  p2p("SHIF1",nmsg,msglen,me,you,DENSE,ISBR1,0,0); 
 }

 MPI_Finalize();
 exit(0);

 return 0;
}

double p2p(char *tag, int nmsg, int msglen, int me, int you, int dpat,
	   int cpat, int stride_pad, int stride_block) {
 int i;
 MPI_Status s;
 MPI_Request r;
 MPI_Datatype vec1, vec_n;
 int          blocklens[2];
 MPI_Aint     index[2];
 MPI_Datatype old_types[2];

 double t1=0, t2=0, bandwidth,latency;
 DATA_T *data,*data_buf;
 DATA_T *data_u, *data_buf_u;

 MPI_Comm_rank(MPI_COMM_WORLD, &i);
 if(me!=i) return -1;

 data = (DATA_T *)malloc((size_t)(sizeof(DATA_T)*msglen + 1));
 data_buf = (DATA_T *)malloc((size_t)(sizeof(DATA_T)*msglen + 1));

 if(!data || !data_buf) {
  printf("malloc error data  %d\n",sizeof(DATA_T)*MSG_END_SIZE);
  fflush(stdout); exit(1);
 }

 for (i=0; i<msglen; ++i) data[i] = TRUTH(0,i);
 for (i=0; i<msglen; ++i) data_buf[i] = 0;
 
 MPI_Barrier(MPI_COMM_WORLD);

 switch(dpat) {

/**********************************************************************/
  case DENSE: {

   switch(cpat) {

    case BSBR1: /* blocking send and recv - unidirectional */ {
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      if(me<you) {
       MPI_Send(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD);
      } else {
       MPI_Recv(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &s);
      }
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    case BSBR2: /* blocking send and recv - bidirectional */ {
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      if(me<you) {
       MPI_Send(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD);
       MPI_Recv(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &s);
      } else {
       MPI_Recv(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &s);
       MPI_Send(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD);
      }
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    case ISBR1: { /* does not make much sense for large nmsg */
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      if(me<you) {
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
      } else {
       MPI_Recv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &s);
      }
      /* does this synch belong here */
      MPI_Barrier(MPI_COMM_WORLD);
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    case ISBR2: {
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      if(me<you) {
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
       MPI_Recv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &s);
      } else {
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
       MPI_Recv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &s);
      }
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    case ISIR2: {
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      if(me<you) {
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
       MPI_Irecv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
      } else {
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
       MPI_Irecv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
      }
      MPI_Wait(&r,&s);
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    case IRIS2: {
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      if(me<you) {
       MPI_Irecv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
      } else {
       MPI_Irecv(data_buf, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
       MPI_Isend(data, msglen, MPIDATA_T, you, 0, MPI_COMM_WORLD, &r);
      }
      MPI_Wait(&r,&s);
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    case SHIF1: {
     t1 = MPI_Wtime();
     for (i=0; i<nmsg; ++i) {
      MPI_Sendrecv(data,msglen,MPIDATA_T,you,1,data_buf,msglen,MPIDATA_T,you,MPI_ANY_TAG,MPI_COMM_WORLD,&s);
     }
     MPI_Barrier(MPI_COMM_WORLD); t2 = MPI_Wtime() - t1;
     break;
    }

    default: {
     if(me<you) {
      printf("unknown MPI communcation pattern\n");
     }
     exit(1);
     break;
    }

   }

   break;
  }
/**********************************************************************/
  case STRID: {
   break;
  }

/**********************************************************************/
  default: {
   if(me<you) {
    printf("unknown MPI communcation pattern\n");
   }
   exit(1);
   break;
  }

 }

 for (i=0; i<msglen; ++i) {
  if(data[i] != TRUTH(0,i)) {
   printf("data integrity error: pattern=\"%s\" rank=%d", tag,me);
   printf(" msglen=%d nmsg=%d i=%d (%d != %d)\n",
   msglen,nmsg,i,data[i],TRUTH(0,i)); fflush(stdout);
   MPI_Finalize();
   exit(1);
  }
 }

 if(me < you) {

  bandwidth = ((msglen * 2 )/t2) * ((nmsg)/(double)MEGA_BYTE);
  latency = 1.0e6 * t2/((double)(nmsg) * 2);

  if(tag) {
   printf(" %-8.8s (%2d,%2d) %.2e    %.2e    %.2e %10d  %d  %d %d",
   tag,me,you,bandwidth,latency,t2,msglen,2*(nmsg),stride_pad,stride_block);
   if(flags & DOMEM) {
    printf("SDD MB");
   }
   printf("\n");
   fflush(stdout);
  }

 }

 free(data_buf);
 free(data);
 return t2;
}

int print_mach_info() {
 char           cbuf[4096];                     /* STRBUFFER */
 struct utsname mach_info;                      /* UNAME */
 time_t         ts;                             /* DATE */
        
 gethostname(cbuf,4096);
 printf("# host    %s ",cbuf);
 uname(&mach_info);
 printf("%s/%s", mach_info.machine, mach_info.sysname);
 printf(" word=%d ",sizeof(double));
 time(&ts);
 strftime(cbuf, 4096, "%b %e %k:%M:%S %Y", localtime(&ts));
 printf(" (%s) ", cbuf);
 printf("\n");
 return 0;
}