SPC #16
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| /* | ||
| * Copyright (c) 2018 The University of Tennessee and The University | ||
|
There was a problem hiding this comment. respect the indentation. move everything left by 5 spaces (for -xxxx the end year). |
||
| * of Tennessee Research Foundation. All rights | ||
| * reserved. | ||
| * | ||
| * Simple example usage of SPCs through MPI_T. | ||
| */ | ||
|
|
||
| #include <stdio.h> | ||
| #include <stdlib.h> | ||
| #include <string.h> | ||
|
|
||
| #include <mpi.h> | ||
|
|
||
| /* Sends 'num_messages' messages of 'message_size' bytes from rank 0 to rank 1. | ||
| * All messages are send synchronously and with the same tag in MPI_COMM_WORLD. | ||
| */ | ||
| void message_exchange(int num_messages, int message_size) | ||
| { | ||
| int i, rank; | ||
| /* Use calloc to initialize data to 0's */ | ||
| char *data = (char*)calloc(message_size, sizeof(char)); | ||
| MPI_Status status; | ||
|
|
||
| MPI_Comm_rank(MPI_COMM_WORLD, &rank); | ||
|
|
||
| if(rank == 0) { | ||
| for(i = 0; i < num_messages; i++) | ||
| MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD); | ||
| } else if(rank == 1) { | ||
| for(i = 0; i < num_messages; i++) | ||
| MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status); | ||
| } | ||
|
|
||
| free(data); | ||
| } | ||
|
|
||
| int main(int argc, char **argv) | ||
| { | ||
| int num_messages, message_size; | ||
|
|
||
| if(argc < 3) { | ||
| printf("Usage: mpirun -np 2 --mca mpi_spc_attach all --mca mpi_spc_dump_enabled true ./test [num_messages] [message_size]\n"); | ||
| return -1; | ||
| } else { | ||
| num_messages = atoi(argv[1]); | ||
| message_size = atoi(argv[2]); | ||
| } | ||
|
|
||
| int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index; | ||
| MPI_Datatype datatype; | ||
| MPI_T_enum enumtype; | ||
| MPI_Comm comm; | ||
| char name[256], description[256]; | ||
|
|
||
| /* Counter names to be read by ranks 0 and 1 */ | ||
| char counter_names[2][40]; | ||
| sprintf(counter_names[0], "runtime_spc_OMPI_BYTES_SENT_USER"); | ||
| sprintf(counter_names[1], "runtime_spc_OMPI_BYTES_RECEIVED_USER"); | ||
|
|
||
| MPI_Init(NULL, NULL); | ||
| MPI_T_init_thread(MPI_THREAD_SINGLE, &provided); | ||
|
|
||
| MPI_Comm_rank(MPI_COMM_WORLD, &rank); | ||
| MPI_Comm_size(MPI_COMM_WORLD, &size); | ||
| if(size != 2) { | ||
| fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n"); | ||
| return -1; | ||
| } | ||
|
|
||
| /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */ | ||
| index = -1; | ||
| MPI_T_pvar_get_num(&num); | ||
| for(i = 0; i < num; i++) { | ||
| name_len = desc_len = 256; | ||
| PMPI_T_pvar_get_info(i, name, &name_len, &verbosity, | ||
| &var_class, &datatype, &enumtype, description, &desc_len, &bind, | ||
| &readonly, &continuous, &atomic); | ||
| if(strcmp(name, counter_names[rank]) == 0) { | ||
| index = i; | ||
| printf("[%d] %s -> %s\n", rank, name, description); | ||
| } | ||
| } | ||
|
|
||
| /* Make sure we found the counters */ | ||
| if(index == -1) { | ||
| fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n"); | ||
| return -1; | ||
| } | ||
|
|
||
| int ret; | ||
| long long value; | ||
|
|
||
| MPI_T_pvar_session session; | ||
| MPI_T_pvar_handle handle; | ||
| /* Create the MPI_T sessions/handles for the counters and start the counters */ | ||
| ret = MPI_T_pvar_session_create(&session); | ||
| ret = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count); | ||
| ret = MPI_T_pvar_start(session, handle); | ||
|
|
||
| message_exchange(num_messages, message_size); | ||
|
|
||
| ret = MPI_T_pvar_read(session, handle, &value); | ||
| /* Print the counter values in order by rank */ | ||
| for(i = 0; i < 2; i++) { | ||
| if(i == rank) { | ||
| printf("[%d] Value Read: %lld\n", rank, value); | ||
| fflush(stdout); | ||
| } | ||
| MPI_Barrier(MPI_COMM_WORLD); | ||
| } | ||
| /* Stop the MPI_T session, free the handle, and then free the session */ | ||
| ret = MPI_T_pvar_stop(session, handle); | ||
| ret = MPI_T_pvar_handle_free(session, &handle); | ||
| ret = MPI_T_pvar_session_free(&session); | ||
|
|
||
| MPI_T_finalize(); | ||
| MPI_Finalize(); | ||
|
|
||
| return 0; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,6 +39,7 @@ | |
| #include "ompi/mca/pml/pml.h" | ||
| #include "ompi/peruse/peruse-internal.h" | ||
| #include "ompi/memchecker.h" | ||
| #include "ompi/runtime/ompi_spc.h" | ||
|
|
||
| #include "pml_ob1.h" | ||
| #include "pml_ob1_comm.h" | ||
|
|
@@ -313,10 +314,17 @@ mca_pml_ob1_recv_frag_t* | |
| check_cantmatch_for_match(mca_pml_ob1_comm_proc_t *proc) | ||
| { | ||
| mca_pml_ob1_recv_frag_t *frag = proc->frags_cant_match; | ||
| #if SPC_ENABLE == 1 | ||
|
There was a problem hiding this comment. This changed with @thananon commit, and I don't think there is a valid reason to check how long it takes to handle OOS match time. I more interesting counter would be to know for how long a fragment stayed in the OOS, but I have no idea how to compute this without adding SPC to the OOB recv frag. |
||
| opal_timer_t timer = 0; | ||
| #endif | ||
|
|
||
| SPC_TIMER_START(OMPI_OOS_MATCH_TIME, &timer); | ||
| if( (NULL != frag) && (frag->hdr.hdr_match.hdr_seq == proc->expected_sequence) ) { | ||
| mca_pml_ob1_recv_frag_t* ret = remove_head_from_ordered_list(&proc->frags_cant_match); | ||
| SPC_TIMER_STOP(OMPI_OOS_MATCH_TIME, &timer); | ||
| return ret; | ||
| } | ||
| SPC_TIMER_STOP(OMPI_OOS_MATCH_TIME, &timer); | ||
| return NULL; | ||
| } | ||
|
|
||
|
|
@@ -388,6 +396,7 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl, | |
| MCA_PML_OB1_RECV_FRAG_ALLOC(frag); | ||
| MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl); | ||
| append_frag_to_ordered_list(&proc->frags_cant_match, frag, proc->expected_sequence); | ||
| SPC_RECORD(OMPI_OUT_OF_SEQUENCE, 1); | ||
| OB1_MATCHING_UNLOCK(&comm->matching_lock); | ||
| return; | ||
| } | ||
|
|
@@ -453,6 +462,8 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl, | |
| &iov_count, | ||
| &bytes_received ); | ||
| match->req_bytes_received = bytes_received; | ||
| SPC_USER_OR_MPI(match->req_recv.req_base.req_ompi.req_status.MPI_TAG, (long long)bytes_received, | ||
| OMPI_BYTES_RECEIVED_USER, OMPI_BYTES_RECEIVED_MPI); | ||
| /* | ||
| * Unpacking finished, make the user buffer unaccessable again. | ||
| */ | ||
|
|
@@ -777,6 +788,11 @@ match_one(mca_btl_base_module_t *btl, | |
| mca_pml_ob1_comm_proc_t *proc, | ||
| mca_pml_ob1_recv_frag_t* frag) | ||
| { | ||
| #if SPC_ENABLE == 1 | ||
| opal_timer_t timer = 0; | ||
| #endif | ||
| SPC_TIMER_START(OMPI_MATCH_TIME, &timer); | ||
|
|
||
| mca_pml_ob1_recv_request_t *match; | ||
| mca_pml_ob1_comm_t *comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm; | ||
|
|
||
|
|
@@ -814,19 +830,25 @@ match_one(mca_btl_base_module_t *btl, | |
| num_segments); | ||
| /* this frag is already processed, so we want to break out | ||
| of the loop and not end up back on the unexpected queue. */ | ||
| SPC_TIMER_STOP(OMPI_MATCH_TIME, &timer); | ||
| return NULL; | ||
| } | ||
|
|
||
| PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_MSG_MATCH_POSTED_REQ, | ||
| &(match->req_recv.req_base), PERUSE_RECV); | ||
| SPC_TIMER_STOP(OMPI_MATCH_TIME, &timer); | ||
| return match; | ||
| } | ||
|
|
||
| /* if no match found, place on unexpected queue */ | ||
| append_frag_to_list(&proc->unexpected_frags, btl, hdr, segments, | ||
| num_segments, frag); | ||
| SPC_RECORD(OMPI_UNEXPECTED, 1); | ||
| SPC_RECORD(OMPI_UNEXPECTED_IN_QUEUE, 1); | ||
| SPC_UPDATE_WATERMARK(OMPI_MAX_UNEXPECTED_IN_QUEUE, OMPI_UNEXPECTED_IN_QUEUE); | ||
| PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm_ptr, | ||
| hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); | ||
| SPC_TIMER_STOP(OMPI_MATCH_TIME, &timer); | ||
| return NULL; | ||
| } while(true); | ||
| } | ||
|
|
@@ -920,6 +942,11 @@ static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl, | |
| MCA_PML_OB1_RECV_FRAG_ALLOC(frag); | ||
| MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl); | ||
| append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected); | ||
|
|
||
| SPC_RECORD(OMPI_OUT_OF_SEQUENCE, 1); | ||
| SPC_RECORD(OMPI_OOS_IN_QUEUE, 1); | ||
| SPC_UPDATE_WATERMARK(OMPI_MAX_OOS_IN_QUEUE, OMPI_OOS_IN_QUEUE); | ||
|
|
||
| OB1_MATCHING_UNLOCK(&comm->matching_lock); | ||
| return OMPI_SUCCESS; | ||
| } | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
?