Nov 30 12:26:44 hezi-3 kernel: ctrl_open , pid 24374: entering Nov 30 12:26:44 hezi-3 kernel: ctrl_ioctl , pid 24374: entering op=4008a130 arg=0x9 Nov 30 12:26:44 hezi-3 kernel: ctrl_ioctl , pid 24374: entering op=4008a107 arg=0x4 Nov 30 12:26:44 hezi-3 kernel: cr_phase2_register , pid 24374: entering Nov 30 12:26:44 hezi-3 kernel: __cr_task_get , pid 24374: Alloc cr_task_t 00000103fba21920 for pid 24374 Nov 30 12:26:44 hezi-3 kernel: ctrl_ioctl , pid 24375: entering op=4008a105 arg=0x4 Nov 30 12:26:44 hezi-3 kernel: cr_phase1_register , pid 24375: entering Nov 30 12:26:44 hezi-3 kernel: __cr_task_get , pid 24375: Alloc cr_task_t 00000103fba21878 for pid 24375 Nov 30 12:26:44 hezi-3 kernel: ctrl_open , pid 24374: entering Nov 30 12:26:44 hezi-3 kernel: ctrl_ioctl , pid 24374: entering op=4008a130 arg=0x9 Nov 30 12:26:44 hezi-3 kernel: ctrl_ioctl , pid 24374: entering op=4008a110 arg=0x7fbfffe9d0 Nov 30 12:26:44 hezi-3 kernel: cr_chkpt_req , pid 24374: entering Nov 30 12:26:44 hezi-3 kernel: cr_log_request , pid 24374: checkpointing process tree 24206 with signal 15 Nov 30 12:26:44 hezi-3 kernel: cr_chkpt_req , pid 24374: checkpoint params: secs= 0, opts=0000000000000000, fd=3 Nov 30 12:26:44 hezi-3 kernel: alloc_request , pid 24374: Alloc cr_chkpt_req_t 00000103e7374e50 Nov 30 12:26:44 hezi-3 kernel: ctrl_open , pid 24374: entering Nov 30 12:26:44 hezi-3 kernel: cr_loc_init , pid 24374: Calling do_init_reg on fd 3 Nov 30 12:26:44 hezi-3 kernel: build_req_tree , pid 24374: in build_req_tree Nov 30 12:26:44 hezi-3 kernel: add_proc , pid 24374: Add proc pid=24206 Nov 30 12:26:44 hezi-3 kernel: add_task , pid 24374: entering task=000001041e963030 (24206) Nov 30 12:26:44 hezi-3 kernel: add_task , pid 24374: entering task=000001041fc94030 (24209) Nov 30 12:26:44 hezi-3 kernel: build_req_tree , pid 24374: scanning children Nov 30 12:26:44 hezi-3 kernel: build_req_tree , pid 24374: found child 24206 Nov 30 12:26:44 hezi-3 kernel: build_req_tree , pid 24374: found child 24209 Nov 30 12:26:45 hezi-3 kernel: do_trigger , pid 24374: triggered pid 24209 (mpirun) w/ retval=0 Nov 30 12:26:45 hezi-3 kernel: cr_suspend , pid 24209: leaving with pending checkpoint Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24374: entering op=c008a111 arg=0x0 Nov 30 12:26:45 hezi-3 kernel: cr_chkpt_done , pid 24374: entering Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24375: entering op=c008a106 arg=0x0 Nov 30 12:26:45 hezi-3 kernel: cr_suspend , pid 24375: entering Nov 30 12:26:45 hezi-3 kernel: ctrl_open , pid 24378: entering Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24378: entering op=4008a130 arg=0x9 Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24378: entering op=4008a107 arg=0x4 Nov 30 12:26:45 hezi-3 kernel: cr_phase2_register , pid 24378: entering Nov 30 12:26:45 hezi-3 kernel: __cr_task_get , pid 24378: Alloc cr_task_t 00000103fba217d0 for pid 24378 Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24379: entering op=4008a105 arg=0x4 Nov 30 12:26:45 hezi-3 kernel: cr_phase1_register , pid 24379: entering Nov 30 12:26:45 hezi-3 kernel: __cr_task_get , pid 24379: Alloc cr_task_t 00000103fba21db8 for pid 24379 Nov 30 12:26:45 hezi-3 kernel: ctrl_open , pid 24378: entering Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24378: entering op=4008a130 arg=0x9 Nov 30 12:26:45 hezi-3 kernel: ctrl_ioctl , pid 24378: entering op=4008a110 arg=0x7fbfffe970 Nov 30 12:26:45 hezi-3 kernel: cr_chkpt_req , pid 24378: entering Nov 30 12:26:45 hezi-3 kernel: cr_log_request , pid 24378: checkpointing process tree 24207 Nov 30 12:26:45 hezi-3 kernel: cr_chkpt_req , pid 24378: checkpoint params: secs= 0, opts=0000000000000000, fd=3 Nov 30 12:26:45 hezi-3 kernel: alloc_request , pid 24378: Alloc cr_chkpt_req_t 00000103e7374ce8 Nov 30 12:26:46 hezi-3 kernel: ctrl_open , pid 24378: entering Nov 30 12:26:46 hezi-3 kernel: cr_loc_init , pid 24378: Calling do_init_reg on fd 3 Nov 30 12:26:46 hezi-3 kernel: build_req_tree , pid 24378: in build_req_tree Nov 30 12:26:46 hezi-3 kernel: add_proc , pid 24378: Add proc pid=24207 Nov 30 12:26:46 hezi-3 kernel: add_task , pid 24378: entering task=0000010417a59030 (24207) Nov 30 12:26:46 hezi-3 kernel: add_task , pid 24378: entering task=0000010400f677f0 (24208) Nov 30 12:26:46 hezi-3 kernel: build_req_tree , pid 24378: scanning children Nov 30 12:26:46 hezi-3 kernel: build_req_tree , pid 24378: found child 24207 Nov 30 12:26:46 hezi-3 kernel: build_req_tree , pid 24378: found child 24208 Nov 30 12:26:46 hezi-3 kernel: do_trigger , pid 24378: triggered pid 24208 (hello) w/ retval=0 Nov 30 12:26:46 hezi-3 kernel: cr_suspend , pid 24208: leaving with pending checkpoint Nov 30 12:26:46 hezi-3 kernel: ctrl_ioctl , pid 24378: entering op=c008a111 arg=0x0 Nov 30 12:26:46 hezi-3 kernel: cr_chkpt_done , pid 24378: entering Nov 30 12:26:46 hezi-3 kernel: ctrl_ioctl , pid 24379: entering op=c008a106 arg=0x0 Nov 30 12:26:46 hezi-3 kernel: cr_suspend , pid 24379: entering Nov 30 12:26:46 hezi-3 kernel: ctrl_ioctl , pid 24208: entering op=4008a101 arg=0x0 Nov 30 12:26:46 hezi-3 kernel: cr_dump_self , pid 24208: entering flags=0x0 Nov 30 12:26:46 hezi-3 kernel: cr_dump_self , pid 24208: NOTIFY(&req->preshared_barrier) Nov 30 12:26:46 hezi-3 kernel: cr_dump_self , pid 24208: TEST(&req->preshared_barrier) returning 0 Nov 30 12:26:47 hezi-3 kernel: cr_save_file_header , pid 24208: Dumping file header Nov 30 12:26:47 hezi-3 kernel: cr_signal_phase_barrier , pid 24208: NOTIFY(&proc_req->phase_barrier) Nov 30 12:26:47 hezi-3 kernel: cr_signal_phase_barrier , pid 24208: ONCE(&proc_req->phase_barrier, 1) begin Nov 30 12:26:47 hezi-3 kernel: cr_signal_phase_barrier , pid 24208: ONCE(&proc_req->phase_barrier, 1) returning 1 Nov 30 12:26:47 hezi-3 kernel: do_trigger , pid 24208: triggered pid 24207 (hello) w/ retval=0 Nov 30 12:26:47 hezi-3 kernel: cr_signal_predump_barrier , pid 24208: NOTIFY(&proc_req->predump_barrier) Nov 30 12:26:47 hezi-3 kernel: cr_signal_predump_barrier , pid 24208: ONCE(&proc_req->predump_barrier, 1) begin Nov 30 12:26:47 hezi-3 kernel: ctrl_ioctl , pid 24207: entering op=4008a101 arg=0x0 Nov 30 12:26:47 hezi-3 kernel: cr_dump_self , pid 24207: entering flags=0x0 Nov 30 12:26:47 hezi-3 kernel: cr_dump_self , pid 24207: NOTIFY(&req->preshared_barrier) Nov 30 12:26:47 hezi-3 kernel: cr_dump_self , pid 24207: TEST(&req->preshared_barrier) returning 1 Nov 30 12:26:47 hezi-3 kernel: cr_signal_predump_barrier , pid 24207: NOTIFY(&proc_req->predump_barrier) Nov 30 12:26:47 hezi-3 kernel: cr_signal_predump_barrier , pid 24207: ONCE(&proc_req->predump_barrier, 1) begin Nov 30 12:26:47 hezi-3 kernel: cr_signal_predump_barrier , pid 24207: ONCE(&proc_req->predump_barrier, 1) returning 1 Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24207: Preparing to dump 2 threads of hello Nov 30 12:26:47 hezi-3 kernel: cr_save_header , pid 24207: Dumping header for 2 threads Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24207: Writing the per-process linkage. Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24207: Writing credentials Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24207: Writing vmadump. Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24207: thread finished vmadump Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24207: ENTER(&proc_req->vmadump_barrier) begin Nov 30 12:26:47 hezi-3 kernel: cr_signal_predump_barrier , pid 24208: ONCE(&proc_req->predump_barrier, 1) returning 0 Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24208: Writing vmadump. Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24208: thread finished vmadump Nov 30 12:26:47 hezi-3 kernel: cr_do_vmadump , pid 24208: ENTER(&proc_req->vmadump_barrier) begin Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: ENTER(&proc_req->vmadump_barrier) returning 1 Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: process finished vmadump Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: Writing the fs struct... Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: Writing the mmap()s table (if any)... Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: Writing POSIX interval timers... Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: WAIT(&req->preshared_barrier) begin Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: WAIT(&req->preshared_barrier) returning 0 Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: Writing mmap()ed pages (if any)... Nov 30 12:26:48 hezi-3 kernel: cr_do_vmadump , pid 24208: Writing the open file section... Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: entering Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...files_struct Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...files Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...0 is open character device. Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...1 is dup of 00000103e7b1e580 Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...2 is dup of 00000103e7b1e580 Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...3 is open socket. Nov 30 12:26:48 hezi-3 kernel: blcr: Skipping a socket. Nov 30 12:26:48 hezi-3 kernel: cr_save_all_files , pid 24208: ...4 is open socket. Nov 30 12:26:48 hezi-3 kernel: blcr: Skipping a socket. Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...5 is regular file. Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...10 is open character device. Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...14 is dup of 00000103e7b1e580 Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...15 is dup of 00000103e7b1e580 Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...16 is dup of 00000103e7b1e580 Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...17 is dup of 00000103e7b1e580 Nov 30 12:26:49 hezi-3 kernel: cr_save_all_files , pid 24208: ...18 is dup of 00000103e7b1e580 Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24208: ENTER(&req->postdump_barrier) begin Nov 30 12:26:49 hezi-3 kernel: cr_do_vmadump , pid 24207: ENTER(&proc_req->vmadump_barrier) returning 0 Nov 30 12:26:49 hezi-3 kernel: cr_do_vmadump , pid 24207: WAIT(&req->preshared_barrier) begin Nov 30 12:26:49 hezi-3 kernel: cr_do_vmadump , pid 24207: WAIT(&req->preshared_barrier) returning 0 Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24207: ENTER(&req->postdump_barrier) begin Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24207: ENTER(&req->postdump_barrier) returning 1 Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24207: Writing the trailer. Nov 30 12:26:49 hezi-3 kernel: cr_save_header , pid 24207: Dumping header for 0 threads Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24207: leaving Returning 0 Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24208: ENTER(&req->postdump_barrier) returning 0 Nov 30 12:26:49 hezi-3 kernel: cr_dump_self , pid 24208: leaving Returning 0 Nov 30 12:26:49 hezi-3 kernel: ctrl_ioctl , pid 24207: entering op=4008a108 arg=0x3 Nov 30 12:26:49 hezi-3 kernel: cr_hand_complete , pid 24207: entering Nov 30 12:26:49 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24207: NOTIFY(&proc_req->pre_complete_barrier) Nov 30 12:26:49 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24207: ONCE(&proc_req->pre_complete_barrier, 1) begin Nov 30 12:26:49 hezi-3 kernel: ctrl_ioctl , pid 24208: entering op=4008a108 arg=0x3 Nov 30 12:26:49 hezi-3 kernel: cr_hand_complete , pid 24208: entering Nov 30 12:26:50 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24208: NOTIFY(&proc_req->pre_complete_barrier) Nov 30 12:26:50 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24208: ONCE(&proc_req->pre_complete_barrier, 1) begin Nov 30 12:26:50 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24208: ONCE(&proc_req->pre_complete_barrier, 1) returning 1 Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_task_complete , pid 24208: NOTIFY(&proc_req->post_complete_barrier) Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_task_complete , pid 24208: WAIT(&proc_req->post_complete_barrier) begin Nov 30 12:26:50 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24207: ONCE(&proc_req->pre_complete_barrier, 1) returning 0 Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_task_complete , pid 24207: NOTIFY(&proc_req->post_complete_barrier) Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_task_complete , pid 24207: WAIT(&proc_req->post_complete_barrier) begin Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_task_complete , pid 24207: WAIT(&proc_req->post_complete_barrier) returning 1 Nov 30 12:26:50 hezi-3 kernel: cr_hand_complete , pid 24207: leaving Returning 0 Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_task_complete , pid 24208: WAIT(&proc_req->post_complete_barrier) returning 0 Nov 30 12:26:50 hezi-3 kernel: cr_hand_complete , pid 24208: leaving Returning 0 Nov 30 12:26:50 hezi-3 kernel: cr_chkpt_done , pid 24378: leaving Returning 1 Nov 30 12:26:50 hezi-3 kernel: ctrl_ioctl , pid 24208: entering op=c008a106 arg=0x0 Nov 30 12:26:50 hezi-3 kernel: cr_suspend , pid 24208: entering Nov 30 12:26:50 hezi-3 kernel: ctrl_ioctl , pid 24378: entering op=0000a112 arg=0xffffffffffffffff Nov 30 12:26:50 hezi-3 kernel: ctrl_release , pid 24378: entering Nov 30 12:26:50 hezi-3 kernel: release_request , pid 24378: Free cr_chkpt_req_t 00000103e7374ce8 Nov 30 12:26:50 hezi-3 kernel: ctrl_release , pid 24378: entering Nov 30 12:26:50 hezi-3 kernel: cr_suspend , pid 24379: leaving with pending signal Nov 30 12:26:50 hezi-3 kernel: ctrl_release , pid 24379: entering Nov 30 12:26:50 hezi-3 kernel: __cr_task_put , pid 24379: Free cr_task_t 00000103fba21db8 Nov 30 12:26:51 hezi-3 kernel: __cr_task_put , pid 24379: Free cr_task_t 00000103fba217d0 Nov 30 12:26:51 hezi-3 kernel: ctrl_ioctl , pid 24209: entering op=4008a101 arg=0x0 Nov 30 12:26:51 hezi-3 kernel: cr_dump_self , pid 24209: entering flags=0x0 Nov 30 12:26:51 hezi-3 kernel: cr_dump_self , pid 24209: NOTIFY(&req->preshared_barrier) Nov 30 12:26:51 hezi-3 kernel: cr_dump_self , pid 24209: TEST(&req->preshared_barrier) returning 0 Nov 30 12:26:51 hezi-3 kernel: cr_save_file_header , pid 24209: Dumping file header Nov 30 12:26:51 hezi-3 kernel: cr_signal_phase_barrier , pid 24209: NOTIFY(&proc_req->phase_barrier) Nov 30 12:26:51 hezi-3 kernel: cr_signal_phase_barrier , pid 24209: ONCE(&proc_req->phase_barrier, 1) begin Nov 30 12:26:51 hezi-3 kernel: cr_signal_phase_barrier , pid 24209: ONCE(&proc_req->phase_barrier, 1) returning 1 Nov 30 12:26:51 hezi-3 kernel: do_trigger , pid 24209: triggered pid 24206 (mpirun) w/ retval=0 Nov 30 12:26:51 hezi-3 kernel: cr_signal_predump_barrier , pid 24209: NOTIFY(&proc_req->predump_barrier) Nov 30 12:26:51 hezi-3 kernel: cr_signal_predump_barrier , pid 24209: ONCE(&proc_req->predump_barrier, 1) begin Nov 30 12:26:51 hezi-3 kernel: ctrl_ioctl , pid 24206: entering op=4008a101 arg=0x0 Nov 30 12:26:51 hezi-3 kernel: cr_dump_self , pid 24206: entering flags=0x0 Nov 30 12:26:51 hezi-3 kernel: cr_dump_self , pid 24206: NOTIFY(&req->preshared_barrier) Nov 30 12:26:51 hezi-3 kernel: cr_dump_self , pid 24206: TEST(&req->preshared_barrier) returning 1 Nov 30 12:26:51 hezi-3 kernel: cr_signal_predump_barrier , pid 24206: NOTIFY(&proc_req->predump_barrier) Nov 30 12:26:51 hezi-3 kernel: cr_signal_predump_barrier , pid 24206: ONCE(&proc_req->predump_barrier, 1) begin Nov 30 12:26:51 hezi-3 kernel: cr_signal_predump_barrier , pid 24206: ONCE(&proc_req->predump_barrier, 1) returning 1 Nov 30 12:26:51 hezi-3 kernel: cr_do_vmadump , pid 24206: Preparing to dump 2 threads of mpirun Nov 30 12:26:51 hezi-3 kernel: cr_save_header , pid 24206: Dumping header for 2 threads Nov 30 12:26:51 hezi-3 kernel: cr_do_vmadump , pid 24206: Writing the per-process linkage. Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24206: Writing credentials Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24206: Writing vmadump. Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24206: thread finished vmadump Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24206: ENTER(&proc_req->vmadump_barrier) begin Nov 30 12:26:52 hezi-3 kernel: cr_signal_predump_barrier , pid 24209: ONCE(&proc_req->predump_barrier, 1) returning 0 Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: Writing vmadump. Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: thread finished vmadump Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: ENTER(&proc_req->vmadump_barrier) begin Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: ENTER(&proc_req->vmadump_barrier) returning 1 Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: process finished vmadump Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: Writing the fs struct... Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: Writing the mmap()s table (if any)... Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: Writing POSIX interval timers... Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: WAIT(&req->preshared_barrier) begin Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: WAIT(&req->preshared_barrier) returning 0 Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: Writing mmap()ed pages (if any)... Nov 30 12:26:52 hezi-3 kernel: cr_do_vmadump , pid 24209: Writing the open file section... Nov 30 12:26:52 hezi-3 kernel: cr_save_all_files , pid 24209: entering Nov 30 12:26:52 hezi-3 kernel: cr_save_all_files , pid 24209: ...files_struct Nov 30 12:26:52 hezi-3 kernel: cr_save_all_files , pid 24209: ...files Nov 30 12:26:52 hezi-3 kernel: cr_save_all_files , pid 24209: ...0 is open character device. Nov 30 12:26:53 hezi-3 kernel: cr_save_all_files , pid 24209: ...1 is dup of 00000103e7b1e580 Nov 30 12:26:53 hezi-3 kernel: cr_save_all_files , pid 24209: ...2 is dup of 00000103e7b1e580 Nov 30 12:26:53 hezi-3 kernel: cr_save_all_files , pid 24209: ...3 is open socket. Nov 30 12:26:53 hezi-3 kernel: blcr: Skipping a socket. Nov 30 12:26:53 hezi-3 kernel: cr_save_all_files , pid 24209: ...4 is open socket. Nov 30 12:26:53 hezi-3 kernel: blcr: Skipping a socket. Nov 30 12:26:53 hezi-3 kernel: cr_save_all_files , pid 24209: ...5 is regular file. Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24209: ENTER(&req->postdump_barrier) begin Nov 30 12:26:53 hezi-3 kernel: cr_do_vmadump , pid 24206: ENTER(&proc_req->vmadump_barrier) returning 0 Nov 30 12:26:53 hezi-3 kernel: cr_do_vmadump , pid 24206: WAIT(&req->preshared_barrier) begin Nov 30 12:26:53 hezi-3 kernel: cr_do_vmadump , pid 24206: WAIT(&req->preshared_barrier) returning 0 Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24206: ENTER(&req->postdump_barrier) begin Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24206: ENTER(&req->postdump_barrier) returning 1 Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24206: Writing the trailer. Nov 30 12:26:53 hezi-3 kernel: cr_save_header , pid 24206: Dumping header for 0 threads Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24209: ENTER(&req->postdump_barrier) returning 0 Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24206: leaving Returning 0 Nov 30 12:26:53 hezi-3 kernel: cr_dump_self , pid 24209: leaving Returning 0 Nov 30 12:26:53 hezi-3 kernel: ctrl_ioctl , pid 24209: entering op=4008a108 arg=0x3 Nov 30 12:26:53 hezi-3 kernel: cr_hand_complete , pid 24209: entering Nov 30 12:26:53 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24209: NOTIFY(&proc_req->pre_complete_barrier) Nov 30 12:26:53 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24209: ONCE(&proc_req->pre_complete_barrier, 1) begin Nov 30 12:26:53 hezi-3 kernel: ctrl_ioctl , pid 24206: entering op=4008a108 arg=0x3 Nov 30 12:26:53 hezi-3 kernel: cr_hand_complete , pid 24206: entering Nov 30 12:26:54 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24206: NOTIFY(&proc_req->pre_complete_barrier) Nov 30 12:26:54 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24206: ONCE(&proc_req->pre_complete_barrier, 1) begin Nov 30 12:26:54 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24206: ONCE(&proc_req->pre_complete_barrier, 1) returning 1 Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_task_complete , pid 24206: NOTIFY(&proc_req->post_complete_barrier) Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_task_complete , pid 24206: WAIT(&proc_req->post_complete_barrier) begin Nov 30 12:26:54 hezi-3 kernel: cr_signal_chkpt_complete_barrier , pid 24209: ONCE(&proc_req->pre_complete_barrier, 1) returning 0 Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_task_complete , pid 24209: NOTIFY(&proc_req->post_complete_barrier) Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_task_complete , pid 24209: WAIT(&proc_req->post_complete_barrier) begin Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_task_complete , pid 24209: WAIT(&proc_req->post_complete_barrier) returning 1 Nov 30 12:26:54 hezi-3 kernel: cr_hand_complete , pid 24209: leaving Returning 0 Nov 30 12:26:54 hezi-3 kernel: ctrl_ioctl , pid 24209: entering op=c008a106 arg=0x0 Nov 30 12:26:54 hezi-3 kernel: cr_suspend , pid 24209: entering Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_task_complete , pid 24206: WAIT(&proc_req->post_complete_barrier) returning 0 Nov 30 12:26:54 hezi-3 kernel: cr_hand_complete , pid 24206: leaving Returning 0 Nov 30 12:26:54 hezi-3 kernel: cr_chkpt_done , pid 24374: leaving Returning 1 Nov 30 12:26:54 hezi-3 kernel: ctrl_ioctl , pid 24374: entering op=0000a112 arg=0xffffffffffffffff Nov 30 12:26:54 hezi-3 kernel: ctrl_release , pid 24374: entering Nov 30 12:26:54 hezi-3 kernel: release_request , pid 24374: Free cr_chkpt_req_t 00000103e7374e50 Nov 30 12:26:54 hezi-3 kernel: ctrl_release , pid 24374: entering Nov 30 12:26:54 hezi-3 kernel: cr_suspend , pid 24208: leaving with pending signal Nov 30 12:26:54 hezi-3 kernel: ctrl_release , pid 24208: entering Nov 30 12:26:54 hezi-3 kernel: __cr_task_put , pid 24208: Free cr_task_t 00000103fba219c8 Nov 30 12:26:55 hezi-3 kernel: __cr_task_put , pid 24208: Free cr_task_t 00000103fba21a70 Nov 30 12:26:55 hezi-3 kernel: cr_suspend , pid 24375: leaving with pending signal Nov 30 12:26:55 hezi-3 kernel: ctrl_release , pid 24375: entering Nov 30 12:26:55 hezi-3 kernel: __cr_task_put , pid 24375: Free cr_task_t 00000103fba21878 Nov 30 12:26:55 hezi-3 kernel: __cr_task_put , pid 24375: Free cr_task_t 00000103fba21920 Nov 30 12:26:55 hezi-3 kernel: cr_suspend , pid 24209: leaving with pending signal Nov 30 12:26:55 hezi-3 kernel: ctrl_release , pid 24209: entering Nov 30 12:26:55 hezi-3 kernel: __cr_task_put , pid 24209: Free cr_task_t 00000103fba21e60 Nov 30 12:26:55 hezi-3 kernel: __cr_task_put , pid 24209: Free cr_task_t 00000103fba21f08 Nov 30 12:27:33 hezi-3 kernel: ctrl_open , pid 24384: entering Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24384: entering op=4008a130 arg=0x9 Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24384: entering op=4008a107 arg=0x3 Nov 30 12:27:33 hezi-3 kernel: cr_phase2_register , pid 24384: entering Nov 30 12:27:33 hezi-3 kernel: __cr_task_get , pid 24384: Alloc cr_task_t 00000103f101da30 for pid 24384 Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24385: entering op=4008a105 arg=0x3 Nov 30 12:27:33 hezi-3 kernel: cr_phase1_register , pid 24385: entering Nov 30 12:27:33 hezi-3 kernel: __cr_task_get , pid 24385: Alloc cr_task_t 00000103f101d988 for pid 24385 Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24385: entering op=c008a106 arg=0x0 Nov 30 12:27:33 hezi-3 kernel: cr_suspend , pid 24385: entering Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24384: entering op=4008a120 arg=0x7fbffff230 Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_request_restart , pid 24384: entering Nov 30 12:27:33 hezi-3 kernel: alloc_rstrt_req , pid 24384: Alloc cr_rstrt_req_t 00000103f1d64e80 Nov 30 12:27:33 hezi-3 kernel: cr_loc_init , pid 24384: Calling do_init_reg on fd 4 Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_request_restart , pid 24384: cr_magic = 67 82, cr_version = 7, scope = 3, arch = 5. Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24384: entering op=8008a126 arg=0x7fbfffec70 Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_procs , pid 24384: entering Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_procs , pid 24384: leaving Returning 1 Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24386: entering op=0000a123 arg=0xffffffffffffffff Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_child , pid 24386: entering Nov 30 12:27:33 hezi-3 kernel: __cr_task_get , pid 24386: Alloc cr_task_t 00000103f101d8e0 for pid 24386 Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_child , pid 24386: NOTIFY(&proc_req->vmadump_barrier) Nov 30 12:27:33 hezi-3 kernel: cr_reserve_ids , pid 24386: Now reserving required ids... Nov 30 12:27:33 hezi-3 kernel: cr_reserve_ids , pid 24386: Now reserving required ids... Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_child , pid 24386: 24386: Restoring credentials Nov 30 12:27:33 hezi-3 kernel: cr_rstrt_child , pid 24386: WAIT_INTERRUPTIBLE(&proc_req->vmadump_barrier) begin Nov 30 12:27:33 hezi-3 kernel: ctrl_ioctl , pid 24384: entering op=8008a126 arg=0x7fbfffec70 Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_procs , pid 24384: entering Nov 30 12:27:34 hezi-3 kernel: ctrl_ioctl , pid 24387: entering op=0000a123 arg=0xffffffffffffffff Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24387: entering Nov 30 12:27:34 hezi-3 kernel: __cr_task_get , pid 24387: Alloc cr_task_t 00000103f101d838 for pid 24387 Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24387: NOTIFY(&proc_req->vmadump_barrier) Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24387: WAIT_INTERRUPTIBLE(&proc_req->vmadump_barrier) begin Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24387: WAIT_INTERRUPTIBLE(&proc_req->vmadump_barrier) returning 1 Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24386: WAIT_INTERRUPTIBLE(&proc_req->vmadump_barrier) returning 0 Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24386: Formerly mpirun PID 24206 Nov 30 12:27:34 hezi-3 kernel: cr_restore_pids , pid 24386: ENTER_INTERRUPTIBLE(&proc_req->linkage_barrier) begin Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24387: Formerly mpirun PID 24209 Nov 30 12:27:34 hezi-3 kernel: cr_restore_pids , pid 24387: Now restoring the pids... Nov 30 12:27:34 hezi-3 kernel: cr_restore_pids , pid 24209: Linkage restore finished... Nov 30 12:27:34 hezi-3 kernel: cr_restore_pids , pid 24209: ENTER_INTERRUPTIBLE(&proc_req->linkage_barrier) begin Nov 30 12:27:34 hezi-3 kernel: cr_restore_pids , pid 24209: ENTER_INTERRUPTIBLE(&proc_req->linkage_barrier) returning 1 Nov 30 12:27:34 hezi-3 kernel: cr_restore_pids , pid 24206: ENTER_INTERRUPTIBLE(&proc_req->linkage_barrier) returning 0 Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24209: 24209: loading mmap()s table Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24209: Reading POSIX interval timers... Nov 30 12:27:34 hezi-3 kernel: cr_rstrt_child , pid 24209: Reading mmap()ed pages (if any)... Nov 30 12:27:34 hezi-3 kernel: cr_restore_all_files , pid 24209: close-on-exec of callers files Nov 30 12:27:35 hezi-3 kernel: cr_restore_all_files , pid 24209: recovering fs_struct... Nov 30 12:27:35 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_all_files , pid 24209: fd=0 dnr=1 Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_chr , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_chr , pid 24209: Character device. major=5, minor=0 path='/dev/tty' Nov 30 12:27:35 hezi-3 kernel: cr_restore_file_locks , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_all_files , pid 24209: fd=1 dnr=1 Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_dup , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_dup , pid 24209: cr_dup2(00000103e7b1e580 -> 1) successful Nov 30 12:27:35 hezi-3 kernel: cr_restore_file_locks , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_all_files , pid 24209: fd=2 dnr=1 Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_dup , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_dup , pid 24209: cr_dup2(00000103e7b1e580 -> 2) successful Nov 30 12:27:35 hezi-3 kernel: cr_restore_file_locks , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_all_files , pid 24209: fd=3 dnr=0 Nov 30 12:27:35 hezi-3 kernel: cr_restore_open_socket , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_file_locks , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:35 hezi-3 kernel: cr_restore_all_files , pid 24209: fd=4 dnr=0 Nov 30 12:27:36 hezi-3 kernel: cr_restore_open_socket , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_restore_file_locks , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_restore_all_files , pid 24209: fd=5 dnr=0 Nov 30 12:27:36 hezi-3 kernel: cr_restore_open_file , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: ctrl_open , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_restore_file_locks , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_load_file_info , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_end_of_file_restore , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_finish_all_file_restores , pid 24209: entering Nov 30 12:27:36 hezi-3 kernel: cr_finish_all_file_restores , pid 24209: current->files=00000103e7bb14c0 Nov 30 12:27:36 hezi-3 kernel: cr_finish_all_file_restores , pid 24209: fd=0 filp=00000103e7b1e580 filp->dentry=00000103dbe7fbc8 d_count=2 d_inode=00000104177204a8 Nov 30 12:27:36 hezi-3 kernel: cr_finish_all_file_restores , pid 24209: fd=1 filp=00000103e7b1e580 filp->dentry=00000103dbe7fbc8 d_count=2 d_inode=00000104177204a8 Nov 30 12:27:36 hezi-3 kernel: cr_finish_all_file_restores , pid 24209: fd=2 filp=00000103e7b1e580 filp->dentry=00000103dbe7fbc8 d_count=2 d_inode=00000104177204a8 Nov 30 12:27:36 hezi-3 kernel: cr_finish_all_file_restores , pid 24209: fd=5 filp=00000103efd31080 filp->dentry=00000103e5fac138 d_count=2 d_inode=00000103eabcc058 Nov 30 12:27:36 hezi-3 kernel: cr_rstrt_child , pid 24209: 24209: closing request descriptor Nov 30 12:27:36 hezi-3 kernel: cr_rstrt_child , pid 24209: 24209: closing context file descriptor Nov 30 12:27:36 hezi-3 kernel: cr_rstrt_child , pid 24209: NOTIFY(&req->barrier) Nov 30 12:27:36 hezi-3 kernel: cr_rstrt_child , pid 24209: WAIT(&req->barrier) begin Nov 30 12:27:36 hezi-3 kernel: cr_rstrt_child , pid 24206: NOTIFY(&req->barrier) Nov 30 12:27:36 hezi-3 kernel: cr_rstrt_child , pid 24206: WAIT(&req->barrier) begin Nov 30 12:27:36 hezi-3 kernel: cr_restore_parents , pid 24384: Now restoring the parent linkage... Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_procs , pid 24384: NOTIFY(&req->barrier) Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_procs , pid 24384: leaving Returning 0 Nov 30 12:27:37 hezi-3 kernel: ctrl_ioctl , pid 24384: entering op=c008a121 arg=0x0 Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_done , pid 24384: entering Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_child , pid 24206: WAIT(&req->barrier) returning 1 Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_child , pid 24206: leaving Returning 24384 Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_child , pid 24209: WAIT(&req->barrier) returning 0 Nov 30 12:27:37 hezi-3 kernel: cr_rstrt_child , pid 24209: leaving Returning 24384 Nov 30 12:27:37 hezi-3 kernel: ctrl_ioctl , pid 24209: entering op=4008a108 arg=0x3 Nov 30 12:27:37 hezi-3 kernel: cr_hand_complete , pid 24209: entering Nov 30 12:27:37 hezi-3 kernel: cr_signal_rstrt_complete_barrier , pid 24209: NOTIFY(&proc_req->pre_complete_barrier) Nov 30 12:27:37 hezi-3 kernel: cr_signal_rstrt_complete_barrier , pid 24209: ONCE(&proc_req->pre_complete_barrier, 1) begin