1 //
2 // messimpl.cc
3 //
4 // Copyright (C) 1996 Limit Point Systems, Inc.
5 //
6 // Author: Curtis Janssen <cljanss@limitpt.com>
7 // Maintainer: LPS
8 //
9 // This file is part of the SC Toolkit.
10 //
11 // The SC Toolkit is free software; you can redistribute it and/or modify
12 // it under the terms of the GNU Library General Public License as published by
13 // the Free Software Foundation; either version 2, or (at your option)
14 // any later version.
15 //
16 // The SC Toolkit is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 // GNU Library General Public License for more details.
20 //
21 // You should have received a copy of the GNU Library General Public License
22 // along with the SC Toolkit; see the file COPYING.LIB. If not, write to
23 // the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
24 //
25 // The U.S. Government is granted a limited license as per AL 91-7.
26 //
27
28 #include <string.h>
29
30 #include <util/misc/formio.h>
31 #include <util/misc/exenv.h>
32
33 #include <util/group/message.h>
34
35 #include <util/group/topology.h>
36 #include <util/group/hcube.h>
37 #ifdef HAVE_NX
38 # include <util/group/messpgon.h>
39 #endif
40 #ifdef HAVE_MPI
41 # include <util/group/messmpi.h>
42 #endif
43
44 using namespace std;
45
46 static ClassDesc MessageGrp_cd(
47 typeid(MessageGrp),"MessageGrp",1,"public DescribedClass",
48 0, 0, 0);
49
50 MessageGrp::MessageGrp(const Ref<KeyVal>& keyval):
51 me_(-1),
52 n_(-1),
53 index_to_classdesc_(0)
54 {
55 gop_max_ = keyval->intvalue("gop_max");
56 if (keyval->error() != KeyVal::OK) gop_max_ = 320000;
57 debug_ = keyval->booleanvalue("debug");
58 }
59
60 MessageGrp::MessageGrp():
61 me_(-1),
62 n_(-1),
63 index_to_classdesc_(0)
64 {
65 gop_max_ = 320000;
66 debug_ = 0;
67 }
68
69 MessageGrp::~MessageGrp()
70 {
71 if (index_to_classdesc_) delete[] index_to_classdesc_;
72 }
73
74 static Ref<MessageGrp> default_messagegrp;
75
76 void
77 MessageGrp::set_default_messagegrp(const Ref<MessageGrp>& grp)
78 {
79 default_messagegrp = grp;
80 }
81
82 MessageGrp*
83 MessageGrp::get_default_messagegrp()
84 1 0 0 0 0 0 0 0 0 0 {
85 if (default_messagegrp.null()) {
86 default_messagegrp = new ProcMessageGrp;
87 }
88 return default_messagegrp.pointer();
89 }
90
91 MessageGrp *
92 MessageGrp::initial_messagegrp(int &argc, char** argv)
93 {
94 MessageGrp *grp = 0;
95
96 char *keyval_string = 0;
97
98 // see if a message group is given on the command line
99 if (argc && argv) {
100 for (int i=0; i<argc; i++) {
101 if (argv[i] && !strcmp(argv[i], "-messagegrp")) {
102 char *messagegrp_string = argv[i];
103 i++;
104 if (i >= argc) {
105 ExEnv::err() << "-messagegrp must be following by an argument"
106 << endl;
107 abort();
108 }
109 keyval_string = argv[i];
110 // move the messagegrp arguments to the end of argv
111 int j;
112 for (j=i+1; j<argc; j++) {
113 argv[j-2] = argv[j];
114 }
115 argv[j++] = messagegrp_string;
116 argv[j++] = keyval_string;
117 // decrement argc to hide the last two arguments
118 argc -= 2;
119 break;
120 }
121 }
122 }
123
124 if (!keyval_string) {
125 // find out if the environment gives the containing message group
126 keyval_string = getenv("MESSAGEGRP");
127 if (keyval_string) {
128 if (!strncmp("MESSAGEGRP=", keyval_string, 11)) {
129 keyval_string = strchr(keyval_string, '=');
130 }
131 if (*keyval_string == '=') keyval_string++;
132 }
133 }
134
135 // if keyval input for a message group was found, then
136 // create it.
137 if (keyval_string) {
138 if (keyval_string[0] == '\0') return 0;
139 //ExEnv::out() << "Creating MessageGrp from \"" << keyval_string << "\"" << endl;
140 Ref<ParsedKeyVal> strkv = new ParsedKeyVal();
141 strkv->parse_string(keyval_string);
142 Ref<DescribedClass> dc = strkv->describedclassvalue();
143 grp = dynamic_cast<MessageGrp*>(dc.pointer());
144 if (dc.null()) {
145 ExEnv::err() << "initial_messagegrp: couldn't find a MessageGrp in "
146 << keyval_string << endl;
147 abort();
148 }
149 else if (!grp) {
150 ExEnv::err() << "initial_messagegrp: wanted MessageGrp but got "
151 << dc->class_name() << endl;
152 abort();
153 }
154 // prevent an accidental delete
155 grp->reference();
156 strkv = 0;
157 dc = 0;
158 // accidental delete not a problem anymore since all smart pointers
159 // to grp are dead
160 grp->dereference();
161 return grp;
162 }
163
164 #if defined(HAVE_MPI)
165 if (ExEnv::initialized() && !strcmp(ExEnv::program_name(), "mpqc-mpi")) {
166 grp = new MPIMessageGrp;
167 return grp;
168 }
169 #endif
170
171 #if defined(HAVE_PUMA_MPI2)
172 //grp = new MPIMessageGrp;
173 grp = new ParagonMessageGrp;
174 if (grp->n() == 1) { delete grp; return new ProcMessageGrp; }
175 else return grp;
176 #elif defined(HAVE_NX)
177 grp = new ParagonMessageGrp;
178 if (grp->n() == 1) { delete grp; return new ProcMessageGrp; }
179 else return grp;
180 #endif
181 return 0;
182 }
183
184 void
185 MessageGrp::initialize(int me, int n)
186 {
187 // This member is called by a CTOR and, ultimately, causes
188 // 'this' to be converted into a temporary Ref which causes
189 // this to be deleted (very bad), so reference 'this'
190 // (and dereference this down below).
191 this->reference();
192
193 if (topology_.null()) {
194 topology_ = new HypercubeTopology();
195 }
196
197 int i;
198 AVLMap<ClassKey,ClassDescP>::iterator J;
199
200 me_ = me;
201 n_ = n;
202
203 // get all of the classes known on this node
204 AVLMap<ClassKey,ClassDescP>& classes = ClassDesc::all();
205
206 // Keeps count of how many classes are known.
207 int iclass = 0;
208
209 for (i=0; i<n; i++) {
210 if (i==me) {
211 // Find out how many of my classes are not yet in the
212 // classdesc to index map.
213 int n_new_class = 0;
214 int buffer_size = 0;
215 for (J=classes.begin(); J!=classes.end(); J++) {
216 if (!classdesc_to_index_.contains(J.data())) {
217 n_new_class++;
218 buffer_size += strlen(J.data()->name()) + 1;
219 }
220 }
221 char* buffer = new char[buffer_size];
222 char* currentbuffer = buffer;
223 for (J=classes.begin(); J!=classes.end(); J++) {
224 if (!classdesc_to_index_.contains(J.data())) {
225 classdesc_to_index_[J.data()] = iclass;
226 iclass++;
227 #ifdef DEBUG
228 ExEnv::out() << scprintf("node %d adding class %d = \"%s\"\n",
229 me, iclass, J.data()->name());
230 #endif
231 strcpy(currentbuffer,J.data()->name());
232 currentbuffer += strlen(J.data()->name()) + 1;
233 }
234 }
235 #ifdef DEBUG
236 ExEnv::out() << scprintf("node %d bcast n_new_class = %d\n",me,n_new_class);
237 #endif
238 bcast(&n_new_class,1,i);
239 #ifdef DEBUG
240 ExEnv::out() << scprintf("node %d finished bcast\n",me);
241 #endif
242 if (n_new_class) {
243 bcast(&buffer_size,1,i);
244 bcast(buffer,buffer_size,i);
245 }
246 delete[] buffer;
247 }
248 else {
249 int j;
250 // Get new classnames and indices from node i.
251 int n_new_class;
252 #ifdef DEBUG
253 ExEnv::out() << scprintf("node %d begin recv bcast\n",me);
254 #endif
255 bcast(&n_new_class,1,i);
256 #ifdef DEBUG
257 ExEnv::out() << scprintf("node %d recv bcast n_new_class = %d\n",
258 me,n_new_class);
259 #endif
260 if (n_new_class) {
261 int buffer_size;
262 bcast(&buffer_size,1,i);
263 char* buffer = new char[buffer_size];
264 char* currentbuffer = buffer;
265 bcast(buffer,buffer_size,i);
266 for (j=0; j<n_new_class; j++) {
267 ClassDescP cd = ClassDesc::name_to_class_desc(currentbuffer);
268 if (cd) {
269 #ifdef DEBUG
270 ExEnv::out() << scprintf("node %d adding \"%s\"\n",
271 me, currentbuffer);
272 #endif
273 classdesc_to_index_[cd] = iclass;
274 }
275 #ifdef DEBUG
276 else {
277 ExEnv::out() << scprintf("node %d failed to add \"%s\"\n",
278 me, currentbuffer);
279 }
280 #endif
281 iclass++;
282 // advance the currentbuffer to the next classname
283 while(*currentbuffer != '\0') currentbuffer++;
284 currentbuffer++;
285 }
286 delete[] buffer;
287 }
288 }
289 }
290 nclass_ = iclass;
291
292 // Construct the mapping of index to classdesc.
293 index_to_classdesc_ = new ClassDescP[iclass];
294 for (i=0; i<nclass_; i++) {
295 index_to_classdesc_[i] = 0;
296 }
297 for (J=classes.begin(); J!=classes.end(); J++) {
298 if (classdesc_to_index_.contains(J.data())) {
299 index_to_classdesc_[classdesc_to_index_[J.data()]] = J.data();
300 }
301 }
302
303 this->dereference();
304 }
305
306 // Sequential send routines
307
308 void
309 MessageGrp::send(int target, double* data, int ndata)
310 {
311 raw_send(target, data, ndata*sizeof(double));
312 }
313 void
314 MessageGrp::send(int target, short* data, int ndata)
315 {
316 raw_send(target, data, ndata*sizeof(short));
317 }
318 void
319 MessageGrp::send(int target, long* data, int ndata)
320 {
321 raw_send(target, data, ndata*sizeof(long));
322 }
323 void
324 MessageGrp::send(int target, float* data, int ndata)
325 {
326 raw_send(target, data, ndata*sizeof(float));
327 }
328 void
329 MessageGrp::send(int target, unsigned int* data, int ndata)
330 {
331 raw_send(target, data, ndata*sizeof(int));
332 }
333 void
334 MessageGrp::send(int target, int* data, int ndata)
335 {
336 raw_send(target, data, ndata*sizeof(int));
337 }
338 void
339 MessageGrp::send(int target, char* data, int ndata)
340 {
341 raw_send(target, data, ndata);
342 }
343 void
344 MessageGrp::send(int target, unsigned char* data, int ndata)
345 {
346 raw_send(target, data, ndata);
347 }
348 void
349 MessageGrp::send(int target, signed char* data, int ndata)
350 {
351 raw_send(target, data, ndata);
352 }
353
354 // Sequential receive routines
355
356 void
357 MessageGrp::recv(int sender, double* data, int ndata)
358 {
359 raw_recv(sender, data, ndata*sizeof(double));
360 }
361 void
362 MessageGrp::recv(int sender, short* data, int ndata)
363 {
364 raw_recv(sender, data, ndata*sizeof(short));
365 }
366 void
367 MessageGrp::recv(int sender, long* data, int ndata)
368 {
369 raw_recv(sender, data, ndata*sizeof(long));
370 }
371 void
372 MessageGrp::recv(int sender, float* data, int ndata)
373 {
374 raw_recv(sender, data, ndata*sizeof(float));
375 }
376 void
377 MessageGrp::recv(int sender, unsigned int* data, int ndata)
378 {
379 raw_recv(sender, data, ndata*sizeof(int));
380 }
381 void
382 MessageGrp::recv(int sender, int* data, int ndata)
383 {
384 raw_recv(sender, data, ndata*sizeof(int));
385 }
386 void
387 MessageGrp::recv(int sender, char* data, int ndata)
388 {
389 raw_recv(sender, data, ndata);
390 }
391 void
392 MessageGrp::recv(int sender, unsigned char* data, int ndata)
393 {
394 raw_recv(sender, data, ndata);
395 }
396 void
397 MessageGrp::recv(int sender, signed char* data, int ndata)
398 {
399 raw_recv(sender, data, ndata);
400 }
401
402 // Typed send routines
403
404 void
405 MessageGrp::sendt(int target, int type, double* data, int ndata)
406 {
407 raw_sendt(target, type, data, ndata*sizeof(double));
408 }
409 void
410 MessageGrp::sendt(int target, int type, short* data, int ndata)
411 {
412 raw_sendt(target, type, data, ndata*sizeof(short));
413 }
414 void
415 MessageGrp::sendt(int target, int type, long* data, int ndata)
416 {
417 raw_sendt(target, type, data, ndata*sizeof(long));
418 }
419 void
420 MessageGrp::sendt(int target, int type, float* data, int ndata)
421 {
422 raw_sendt(target, type, data, ndata*sizeof(float));
423 }
424 void
425 MessageGrp::sendt(int target, int type, unsigned int* data, int ndata)
426 {
427 raw_sendt(target, type, data, ndata*sizeof(int));
428 }
429 void
430 MessageGrp::sendt(int target, int type, int* data, int ndata)
431 {
432 raw_sendt(target, type, data, ndata*sizeof(int));
433 }
434 void
435 MessageGrp::sendt(int target, int type, char* data, int ndata)
436 {
437 raw_sendt(target, type, data, ndata);
438 }
439 void
440 MessageGrp::sendt(int target, int type, unsigned char* data, int ndata)
441 {
442 raw_sendt(target, type, data, ndata);
443 }
444 void
445 MessageGrp::sendt(int target, int type, signed char* data, int ndata)
446 {
447 raw_sendt(target, type, data, ndata);
448 }
449
450 // Typed receive routines
451
452 void
453 MessageGrp::recvt(int type, double* data, int ndata)
454 {
455 raw_recvt(type, data, ndata*sizeof(double));
456 }
457 void
458 MessageGrp::recvt(int type, short* data, int ndata)
459 {
460 raw_recvt(type, data, ndata*sizeof(short));
461 }
462 void
463 MessageGrp::recvt(int type, long* data, int ndata)
464 {
465 raw_recvt(type, data, ndata*sizeof(long));
466 }
467 void
468 MessageGrp::recvt(int type, float* data, int ndata)
469 {
470 raw_recvt(type, data, ndata*sizeof(float));
471 }
472 void
473 MessageGrp::recvt(int type, unsigned int* data, int ndata)
474 {
475 raw_recvt(type, data, ndata*sizeof(int));
476 }
477 void
478 MessageGrp::recvt(int type, int* data, int ndata)
479 {
480 raw_recvt(type, data, ndata*sizeof(int));
481 }
482 void
483 MessageGrp::recvt(int type, char* data, int ndata)
484 {
485 raw_recvt(type, data, ndata);
486 }
487 void
488 MessageGrp::recvt(int type, unsigned char* data, int ndata)
489 {
490 raw_recvt(type, data, ndata);
491 }
492 void
493 MessageGrp::recvt(int type, signed char* data, int ndata)
494 {
495 raw_recvt(type, data, ndata);
496 }
497
498 // Broadcast operations
499
500 void
501 MessageGrp::bcast(double*data, int ndata, int from)
502 4 0 5 0 1 1 0 0 0 0 {
503 2 0 0 1 0 0 0 0 0 0 raw_bcast(data, ndata*sizeof(double), from);
504 0 0 1 0 0 0 0 0 0 0 }
505 void
506 MessageGrp::bcast(short*data, int ndata, int from)
507 {
508 raw_bcast(data, ndata*sizeof(short), from);
509 }
510 void
511 MessageGrp::bcast(long*data, int ndata, int from)
512 {
513 raw_bcast(data, ndata*sizeof(long), from);
514 }
515 void
516 MessageGrp::bcast(float*data, int ndata, int from)
517 {
518 raw_bcast(data, ndata*sizeof(float), from);
519 }
520 void
521 MessageGrp::bcast(unsigned int*data, int ndata, int from)
522 {
523 raw_bcast(data, ndata*sizeof(int), from);
524 }
525 void
526 MessageGrp::bcast(int*data, int ndata, int from)
527 1 0 1 1 4 0 0 0 0 0 {
528 2 0 1 0 0 0 0 0 0 0 raw_bcast(data, ndata*sizeof(int), from);
529 1 0 0 0 2 0 0 0 0 0 }
530 void
531 MessageGrp::bcast(char*data, int ndata, int from)
532 {
533 raw_bcast(data, ndata, from);
534 }
535 void
536 MessageGrp::bcast(unsigned char*data, int ndata, int from)
537 {
538 raw_bcast(data, ndata, from);
539 }
540 void
541 MessageGrp::bcast(signed char*data, int ndata, int from)
542 {
543 raw_bcast(data, ndata, from);
544 }
545
546 // Global classdesc indices
547
548 int
549 MessageGrp::classdesc_to_index(const ClassDesc* cdptr)
550 {
551 if (classdesc_to_index_.contains((ClassDesc*)cdptr)) {
552 return classdesc_to_index_[(ClassDesc*)cdptr];
553 }
554 else {
555 return -1;
556 }
557 }
558
559 const ClassDesc*
560 MessageGrp::index_to_classdesc(int index)
561 {
562 if (index < 0 || index >= nclass_) {
563 return 0;
564 }
565 else {
566 return index_to_classdesc_[index];
567 }
568 }
569
570 void
571 MessageGrp::raw_bcast(void* data, int nbyte, int from)
572 {
573 int nbyte_actual = nbyte;
574 int tgop_max = nbyte;
575 if (gop_max_ != 0) {
576 tgop_max = gop_max_;
577 gop_max_ = 0;
578 bcast(nbyte_actual,from);
579 gop_max_ = tgop_max;
580 }
581 for (int idat=0; idat<nbyte_actual; idat+=tgop_max) {
582 int ndat = (idat+tgop_max>nbyte_actual)?(nbyte_actual-idat):tgop_max;
583 Ref<GlobalMsgIter> i(topology_->global_msg_iter(this, from));
584 for (i->forwards(); !i->done(); i->next()) {
585 if (i->send()) {
586 raw_send(i->sendto(), &((char*)data)[idat], ndat);
587 }
588 if (i->recv()) {
589 raw_recv(i->recvfrom(), &((char*)data)[idat], ndat);
590 }
591 }
592 }
593 }
594
595 void
596 MessageGrp::sync()
597 {
598 Ref<GlobalMsgIter> i(topology_->global_msg_iter(this, 0));
599
600 for (i->backwards(); !i->done(); i->next()) {
601 if (i->send()) {
602 raw_send(i->sendto(), 0, 0);
603 }
604 if (i->recv()) {
605 raw_recv(i->recvfrom(), 0, 0);
606 }
607 }
608
609 for (i->forwards(); !i->done(); i->next()) {
610 if (i->send()) {
611 raw_send(i->sendto(), 0, 0);
612 }
613 if (i->recv()) {
614 raw_recv(i->recvfrom(), 0, 0);
615 }
616 }
617 }
618
619 void
620 MessageGrp::collect(const double *part, const int *lengths, double *whole)
621 {
622 raw_collect(part,lengths,whole,sizeof(double));
623 }
624
625 void
626 MessageGrp::raw_collect(const void *part, const int *lengths, void *whole,
627 int bytes_per_datum)
628 {
629 int offset = 0;
630 for (int i=0; i<n_; i++) {
631 int nbytes = lengths[i];
632 if (i==me_) memcpy(&((char*)whole)[offset], part, nbytes);
633 raw_bcast(&((char*)whole)[offset], nbytes, i);
634 offset += nbytes;
635 }
636 }
637
638 /////////////////////////////////////////////////////////////////////////////
639
640 // Local Variables:
641 // mode: c++
642 // c-file-style: "CLJ"
643 // End:
644