diff --git a/autogen.sh b/autogen.sh index 0c98ebe7005..83abcf9a6d8 100755 --- a/autogen.sh +++ b/autogen.sh @@ -531,7 +531,7 @@ fn_gen_binding_c() { fn_json_gen() { echo_n "generating json char arrays... " - ./maint/tuning/coll/json_gen.sh + ./maint/json_gen.sh echo "done" } diff --git a/doc/mpich/tuning_parameters.md b/doc/mpich/tuning_parameters.md index b581a6297a8..45f19e0628e 100644 --- a/doc/mpich/tuning_parameters.md +++ b/doc/mpich/tuning_parameters.md @@ -124,7 +124,7 @@ node and on different nodes. This JSON file is used by setting `MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE=/path/to/ch4_selection_tuning.json`. 2. Coll: This file determines which algorithm used for inter or intra algorithms, depending on the composition. This JSON file can be used by -setting `MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE=/path/to/coll_selection_tuning.json`. +setting `MPIR_CVAR_COLL_SELECTION_JSON_FILE=/path/to/coll_selection_tuning.json`. This file contains algorithm covered in the blocking collectives section (Section 2.3). 3. Posix: This file determines algorithms to be used for intra node cases. diff --git a/doc/wiki/notes/cvar.md b/doc/wiki/notes/cvar.md index 2d29982a039..f9407c36678 100644 --- a/doc/wiki/notes/cvar.md +++ b/doc/wiki/notes/cvar.md @@ -17,7 +17,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) ring - Force ring algorithm brucks - Force brucks algorithm diff --git a/dummy b/dummy new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/dummy @@ -0,0 +1 @@ +1 diff --git a/maint/gen_coll.py b/maint/gen_coll.py index de4fbcbdfd6..a1b3c2dcb3b 100644 --- a/maint/gen_coll.py +++ b/maint/gen_coll.py @@ -9,575 +9,667 @@ from local_python.binding_common import * def main(): + G.coll_names = ["barrier", "bcast", "gather", "gatherv", "scatter", "scatterv", "allgather", "allgatherv", "alltoall", "alltoallv", "alltoallw", "reduce", "allreduce", "reduce_scatter", "reduce_scatter_block", "scan", "exscan", "neighbor_allgather", "neighbor_allgatherv", "neighbor_alltoall", "neighbor_alltoallv", "neighbor_alltoallw"] + binding_dir = G.get_srcdir_path("src/binding") c_dir = "src/binding/c" func_list = load_C_func_list(binding_dir, silent=True) - G.algos = load_coll_algos("src/mpi/coll/coll_algorithms.txt") - coll_names = ["barrier", "bcast", "gather", "gatherv", "scatter", "scatterv", "allgather", "allgatherv", "alltoall", "alltoallv", "alltoallw", "reduce", "allreduce", "reduce_scatter", "reduce_scatter_block", "scan", "exscan", "neighbor_allgather", "neighbor_allgatherv", "neighbor_alltoall", "neighbor_alltoallv", "neighbor_alltoallw"] + # Loading coll_algorithms.txt. It sets - + # - G.conditions: a list of conditions that can be used as restrictions and in JSON tuning files + # - G.algos: a two level array: [func-commkind][algo] + load_coll_algos("src/mpi/coll/coll_algorithms.txt") + # Prepare a one level algo array for conveninece - + # - G.algo_list: a one level array [algo] + G.algo_list = collect_algo_list() - G.out = [] - G.prototypes_hash = {} + G.out = [] # output to C file + G.out2 = [] # output to header G.prototypes = [] G.out.append("#include \"mpiimpl.h\"") - G.out.append("#include \"iallgatherv/iallgatherv.h\"") - for a in coll_names: - dump_coll(a, "blocking") - dump_coll(a, "nonblocking") - dump_coll(a, "persistent") - dump_c_file("src/mpi/coll/mpir_coll.c", G.out) - dump_prototypes("src/mpi/coll/include/coll_algos.h", G.prototypes) + G.out.append("#include \"coll_csel.h\"") + + # dump impl functions + for a in G.coll_names: + dump_coll_impl(a, "blocking") + dump_coll_impl(a, "nonblocking") + dump_coll_impl(a, "persistent") + + # TEMP: dump mpir functions. + # Current code base call MPIR_ functions in copositinal algorithms. Create a wrapper that call _impl + # for now. We will refactor the compositional algorithms later. + for a in G.coll_names: + dump_coll_mpir(a, "blocking") + dump_coll_mpir(a, "nonblocking") + + # dump the container version of the algorithms + dump_algo_cnt_fns() + add_algo_prototypes() + for a in G.coll_names: + add_sched_auto_prototypes(a) + + # initialize MPIR_Coll_algo_table and MPIR_Coll_algo_names + dump_MPII_Coll_algo_init() + # initialize MPIR_Coll_cvar_table and MPIR_Coll_type_names + dump_MPII_Coll_type_init() + # initialize MPIR_Coll_condition_names + dump_MPII_Csel_init_condition_names() + # parsing routines for loading JSONs + dump_MPII_Csel_parse_container() + dump_MPII_Csel_parse_operator() + # routine for Csel search + dump_MPII_Csel_run_condition() + # routines for checking algorithm CVARs + dump_MPIR_Coll_cvar_to_algo_id() + dump_MPIR_Coll_init_algo_container() + dump_MPIR_Coll_check_algo_restriction() + + # enum for coll_type, define MPIR_CSEL_NUM_COLL_TYPES + dump_MPIR_Csel_coll_type_e() + # enum for algorithm id, define MPIR_CSEL_NUM_ALGORITHMS + dump_MPIR_Csel_container_type_e() + # enum CSEL_NODE_TYPE, define MPIR_CSEL_NUM_CONDITIONS + dump_MPIR_Csel_node_type_e() + # algorithm container struct + dump_MPII_Csel_container() + G.out2.append("") -def add_prototype(l): - if RE.match(r'int\s+(\w+)\(', l): - func_name = RE.m.group(1) - if func_name not in G.prototypes_hash: - G.prototypes_hash[func_name] = 1 - G.prototypes.append(l) - else: + dump_c_file("src/mpi/coll/mpir_coll.c", G.out) + dump_coll_algos_h("src/mpi/coll/include/coll_algos.h", G.prototypes, G.out2) + +def collect_algo_list(): + algo_list = [] + for coll in G.coll_names: + for commkind in ("intra", "inter"): + for blocking in (True, False): + if blocking: + func_commkind = coll + '-' + commkind + else: + func_commkind = 'i' + coll + '-' + commkind + if func_commkind in G.algos: + for algo in G.algos[func_commkind]: + if "allcomm" in algo and func_commkind.endswith("inter"): + continue + algo_list.append(algo) + for algo in G.algos['general']: + algo_list.append(algo) + return algo_list + +def dump_algo_cnt_fns(): + def get_algo_args(coll_name, algo): + args = get_coll_args(coll_name, "csel") + if 'extra_params' in algo: + args += ", " + get_algo_extra_args(algo, "csel") + + if algo['func-commkind'].startswith('i'): + args += ", coll_sig->sched" + elif algo['func-commkind'].startswith('neighbor_'): pass + else: + args += ", 0" # coll_attr -def load_coll_algos(algo_txt): - All = {} - with open(algo_txt) as In: - (func_commkind, algo_list, algo) = (None, None, None) - for line in In: - if RE.match(r'(\w+-(intra|inter)):', line): - func_commkind = RE.m.group(1) - algo_list = [] - All[func_commkind] = algo_list - elif RE.match(r'\s+(\w+)\s*$', line): - algo = {"name": RE.m.group(1), "func-commkind": func_commkind} - algo_list.append(algo) - elif RE.match(r'\s+(\w+):\s*(.+)', line): - (key, value) = RE.m.group(1,2) - algo[key] = value - return All - -def dump_coll(name, blocking_type): - if blocking_type == "blocking": - dump_allcomm_auto_blocking(name) - dump_mpir_impl_blocking(name) - elif blocking_type == "nonblocking": - dump_allcomm_sched_auto(name) - dump_sched_impl(name) - dump_mpir_impl_nonblocking(name) - elif blocking_type == "persistent": - dump_mpir_impl_persistent(name) - else: - raise Exception("Wrong blocking_type") - dump_mpir(name, blocking_type) + return args -def dump_allcomm_auto_blocking(name): - """ MPIR_Xxx_allcomm_auto - use Csel selections """ - blocking_type = "blocking" - func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, "blocking") - func_args = get_func_args(args, name, "blocking") + def dump_algo_prep(algo): + if algo['func-commkind'].startswith('i'): + if algo['name'].startswith('tsp_'): + G.out.append("MPII_CSEL_CREATE_TSP_SCHED(coll_sig);") + else: + G.out.append("MPII_CSEL_CREATE_SCHED(coll_sig);") + + for algo in G.algo_list: + if algo["func-commkind"] != 'general': + coll_name = get_algo_coll_name(algo) + algo_funcname = get_algo_funcname(algo) + algo_args = get_algo_args(coll_name, algo) + decl = "int %s_cnt(%s)" % (algo_funcname, "MPIR_Csel_coll_sig_s * coll_sig, MPII_Csel_container_s * cnt") + add_prototype(decl) + dump_split(0, decl) + dump_open('{') + G.out.append("int mpi_errno = MPI_SUCCESS;") + G.out.append("") + dump_algo_prep(algo) + dump_split(1, "mpi_errno = %s(%s);" % (algo_funcname, algo_args)) + G.out.append("MPIR_ERR_CHECK(mpi_errno);") + G.out.append("") + G.out.append("fn_exit:") + G.out.append("return mpi_errno;") + G.out.append("fn_fail:") + G.out.append("goto fn_exit;") + dump_close('}') + G.out.append("") - # e.g. ibcast, Ibcast, IBCAST - func_name = get_func_name(name, blocking_type) - Name = func_name.capitalize() - NAME = func_name.upper() +def add_algo_prototypes(): + def get_algo_params(algo): + coll_name = get_algo_coll_name(algo) + params = get_coll_params(coll_name) + if 'extra_params' in algo: + params += ", " + get_algo_extra_params(algo) + if algo['func-commkind'].startswith('i'): + if algo['name'].startswith('tsp_'): + params += ", MPIR_TSP_sched_t s" + else: + params += ", MPIR_Sched_t s" + elif algo['func-commkind'].startswith('neighbor_'): + pass + else: + params += ", int coll_attr" # coll_attr + + return params + + for algo in G.algo_list: + if algo['func-commkind'] == 'general': + decl = "int %s(%s)" % (algo['name'], "MPIR_Csel_coll_sig_s * coll_sig, MPII_Csel_container_s * cnt") + add_prototype(decl) + else: + algo_funcname = get_algo_funcname(algo) + algo_params = get_algo_params(algo) + decl = "int %s(%s)" % (algo_funcname, algo_params) + add_prototype(decl) + +def add_sched_auto_prototypes(coll_name): + params = get_coll_params(coll_name) + params += ", MPIR_Sched_t s" + add_prototype("int MPIR_I%s_intra_sched_auto(%s)" % (coll_name, params)) + if not re.match(r'(scan|exscan|neighbor_)', coll_name): + add_prototype("int MPIR_I%s_inter_sched_auto(%s)" % (coll_name, params)) + +def dump_MPII_Coll_type_init(): G.out.append("") - G.out.append("/* ---- %s ---- */" % func_name) - G.out.append("") - add_prototype("int MPIR_%s_allcomm_auto(%s)" % (Name, func_params)) - dump_split(0, "int MPIR_%s_allcomm_auto(%s)" % (Name, func_params)) + decl = "void MPII_Coll_type_init(void)" + add_prototype(decl) + G.out.append(decl) dump_open('{') - G.out.append("int mpi_errno = MPI_SUCCESS;") + for a in G.coll_names: + for commkind in ("intra", "inter"): + for is_blocking in (True, False): + if commkind == "inter" and re.match(r'(scan|exscan|neighbor_)', a): + # CVARs for these inter-collective does not exist + G.out.append("MPIR_Coll_cvar_table[%s] = 0;" % (coll_type(a, is_blocking, commkind))) + else: + G.out.append("MPIR_Coll_cvar_table[%s] = %s;" % (coll_type(a, is_blocking, commkind), cvar_name(a, is_blocking, commkind))) + + for a in G.coll_names: + for commkind in ("intra", "inter"): + for is_blocking in (True, False): + if is_blocking: + coll_type_name = a + '-' + commkind + else: + coll_type_name = 'i' + a + '-' + commkind + G.out.append("MPIR_Coll_type_names[%s] = \"%s\";" % (coll_type(a, is_blocking, commkind), coll_type_name)) + dump_close('}') + +def dump_MPII_Csel_init_condition_names(): G.out.append("") + decl = "void MPII_Csel_init_condition_names(void)" + add_prototype(decl) + G.out.append(decl) + dump_open('{') + for a in G.conditions: + if RE.match(r'(.+)\(thresh\)', a): + cond = RE.m.group(1) + else: + cond = a + G.out.append("MPIR_Csel_condition_names[%s] = \"%s\";" % (condition_id(cond), cond)) + dump_close('}') - # -- Csel_search - dump_open("MPIR_Csel_coll_sig_s coll_sig = {") - G.out.append(".coll_type = MPIR_CSEL_COLL_TYPE__%s," % NAME) - G.out.append(".comm_ptr = comm_ptr,") - for p in func['parameters']: - if not re.match(r'comm$', p['name']): - G.out.append(".u.%s.%s = %s," % (func_name, p['name'], p['name'])) - dump_close("};") +def dump_MPII_Coll_algo_init(): G.out.append("") - G.out.append("MPII_Csel_container_s *cnt = MPIR_Csel_search(comm_ptr->csel_comm, coll_sig);") - G.out.append("MPIR_Assert(cnt);") + decl = "void MPII_Coll_algo_init(void)" + add_prototype(decl) + G.out.append(decl) + dump_open('{') + for a in G.algo_list: + algo_funcname = get_algo_funcname(a) + idx = algo_id(algo_funcname) + if a['func-commkind'] != 'general': + algo_funcname += "_cnt" + G.out.append("MPIR_Coll_algo_table[%s] = %s;" % (idx, algo_funcname)) + for a in G.algo_list: + algo_funcname = get_algo_funcname(a) + idx = algo_id(algo_funcname) + G.out.append("MPIR_Coll_algo_names[%s] = \"%s\";" % (idx, algo_funcname)) + dump_close('}') + + +def dump_MPII_Csel_parse_container(): G.out.append("") + def dump_json_foreach_open(): + dump_open("json_object_object_foreach(obj, key, val) {") + G.out.append("char *ckey = MPL_strdup_no_spaces(key);") - # -- switch - def dump_cnt_algo_blocking(algo, commkind): - if "allcomm" in algo: - commkind = "allcomm" - algo_name = get_algo_name(algo) - algo_args = get_algo_args(args, algo, "csel") - algo_params = get_algo_params(params, algo) - add_prototype("int MPIR_%s_%s_%s(%s)" % (Name, commkind, algo_name, algo_params)) - dump_split(3, "mpi_errno = MPIR_%s_%s_%s(%s);" % (Name, commkind, algo_name, algo_args)) - - dump_open("switch (cnt->id) {") - for commkind in ("intra", "inter"): - if commkind == "inter" and re.match(r'(scan|exscan|neighbor_)', name): - continue - for algo in G.algos[func_name + "-" + commkind]: - if "allcomm" in algo: - if commkind == "intra": - commkind = "allcomm" - else: - # skip inter since it is covered already - continue - G.out.append("case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_%s_%s_%s:" % (Name, commkind, algo['name'])) - G.out.append("INDENT") - dump_cnt_algo_blocking(algo, commkind) - G.out.append("break;"); - G.out.append("DEDENT") - G.out.append("") - G.out.append("case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_%s_allcomm_nb:" % Name) - add_prototype("int MPIR_%s_allcomm_nb(%s);" % (Name, func_params)) - dump_split(2, " mpi_errno = MPIR_%s_allcomm_nb(%s);" % (Name, func_args)) - G.out.append(" break;"); - G.out.append("") - G.out.append("default:") - G.out.append(" MPIR_Assert(0);") - dump_close("}") + def dump_json_foreach_close(): + G.out.append("MPL_free(ckey);") + dump_close("}") - # -- return - G.out.append("MPIR_ERR_CHECK(mpi_errno);") - dump_fn_exit() - dump_close("}") + def dump_parse_params(): + dump_open("switch (cnt->id) {") + for algo in G.algo_list: + if 'extra_params' in algo: + struct_name = algo_struct_name(algo) + extra_params = algo['extra_params'].replace(' ', '').split(',') + G.out.append("case %s:" % algo_id(get_algo_funcname(algo))) + dump_open('{') # protect json_object_object_foreach + G.out.append("int num_params = 0;") + num_expect = 0 + dump_json_foreach_open() + ifstr = "if" + for a in extra_params: + if re.match(r'\w+=(.+)', a): + # skip constant parameter + continue + else: + num_expect += 1 + n = len(a) + 1 + atoi = "atoi" + if a == "tree_type": + atoi = "get_tree_type_from_string" + G.out.append("%s (!strncmp(ckey, \"%s=\", %d)) {" % (ifstr, a, n)) + G.out.append(" cnt->u.%s.%s = %s(ckey + %d);" % (struct_name, a, atoi, n)) + G.out.append(" num_params++;") + ifstr = "} else if" + G.out.append("}") + dump_json_foreach_close(); + dump_open("if (num_params != %d) {" % num_expect) + G.out.append("printf(\"MPII_Csel_parse_container_params: algorithm %s expect %d parameters\\\n\");" % (struct_name, num_expect)) + dump_close('}') + G.out.append("MPIR_Assert(num_params == %d);" % num_expect) + dump_close('}') + G.out.append(" break;") + G.out.append("default:") + G.out.append(" break;") + dump_close('}') # switch -def dump_allcomm_sched_auto(name): - """ MPIR_Xxx_allcomm_sched_auto - use Csel selections """ - blocking_type = "nonblocking" - func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, "allcomm_sched_auto") + decl = "int MPII_Csel_parse_container_params(void *json_obj, void *container)" + add_prototype(decl) + G.out.append(decl) + dump_open('{') + G.out.append("struct json_object *obj = json_obj;") + G.out.append("MPII_Csel_container_s *cnt = container;") + # G.out.append("obj = json_object_object_get(obj, key);") + dump_parse_params() + G.out.append("") + G.out.append("return MPI_SUCCESS;") + dump_close('}') - # e.g. ibcast, Ibcast, IBCAST - func_name = get_func_name(name, blocking_type) - Name = func_name.capitalize() - NAME = func_name.upper() +def dump_MPII_Csel_parse_operator(): + decl = "int MPII_Csel_parse_operator(const char *ckey, void *operator_node)" + add_prototype(decl) + G.out.append(decl) + dump_open('{') + G.out.append("MPIR_Csel_node_s *csel_node = operator_node;") + dump_open("if (ckey[0] == '!') {") + G.out.append("csel_node->u.condition.negate = true;") + G.out.append("ckey++;") + dump_else() + G.out.append("csel_node->u.condition.negate = false;") + dump_close('}') + + if_clase = "if" + for a in G.conditions: + cond = a + has_thresh = False + if RE.match(r'(.+)\(thresh\)', a): + cond = RE.m.group(1) + has_thresh = True + n = len(cond) + if has_thresh: + G.out.append("%s (strncmp(ckey, \"%s\", %d) == 0) {" % (if_clase, cond, n)) + G.out.append(" csel_node->type = %s;" % condition_id(cond)) + G.out.append(" MPIR_Assert(ckey[%d] == '(');" % n) + G.out.append(" csel_node->u.condition.thresh = atoi(ckey + %d);" % (n + 1)) + else: + G.out.append("%s (strcmp(ckey, \"%s\") == 0) {" % (if_clase, cond)) + G.out.append(" csel_node->type = %s;" % condition_id(cond)) + if_clase = "} else if" + G.out.append("} else {") + G.out.append(" return MPI_ERR_OTHER;") + G.out.append("}") G.out.append("") - G.out.append("/* ---- %s ---- */" % func_name) - G.out.append("") - add_prototype("int MPIR_%s_allcomm_sched_auto(%s)" % (Name, func_params)) - dump_split(0, "int MPIR_%s_allcomm_sched_auto(%s)" % (Name, func_params)) + G.out.append("return MPI_SUCCESS;") + dump_close('}') + +def dump_MPII_Csel_run_condition(): + decl = "void *MPII_Csel_run_condition(void *operator_node, MPIR_Csel_coll_sig_s *coll_sig)" + add_prototype(decl) + G.out.append(decl) dump_open('{') - G.out.append("int mpi_errno = MPI_SUCCESS;") - G.out.append("") + G.out.append("MPIR_Csel_node_s *node = operator_node;") + G.out.append("bool cond;") + dump_open("switch(node->type) {") + for a in G.conditions: + cond = a + has_thresh = False + if RE.match(r'(.+)\(thresh\)', a): + cond = RE.m.group(1) + has_thresh = True + dump_open("case %s:" % condition_id(cond)) + if has_thresh: + G.out.append("cond = (node->u.condition.thresh <= %s(coll_sig));" % G.conditions[a]) + else: + G.out.append("cond = %s(coll_sig);" % G.conditions[a]) + G.out.append("if (node->u.condition.negate) cond = !cond;") + G.out.append("return cond ? node->success : node->failure;") + dump_close("") + G.out.append("default:") + G.out.append(" break;") + dump_close('}') # switch + G.out.append("return NULL;") + dump_close('}') - # -- Csel_search - dump_open("MPIR_Csel_coll_sig_s coll_sig = {") - G.out.append(".coll_type = MPIR_CSEL_COLL_TYPE__%s," % NAME) - G.out.append(".comm_ptr = comm_ptr,") - for p in func['parameters']: - if not re.match(r'comm$', p['name']): - G.out.append(".u.%s.%s = %s," % (func_name, p['name'], p['name'])) - dump_close("};") - G.out.append("") - G.out.append("MPII_Csel_container_s *cnt = MPIR_Csel_search(comm_ptr->csel_comm, coll_sig);") - G.out.append("MPIR_Assert(cnt);") +def dump_MPIR_Coll_cvar_to_algo_id(): G.out.append("") + def dump_cvar_cases(name, commkind): + algo_id_prefix = "MPII_CSEL_CONTAINER_TYPE__ALGORITHM" + + dump_open("switch (cvar_val) {") + G.out.append("case MPIR_CVAR_%s_%s_ALGORITHM_auto:" % (name.upper(), commkind.upper())) + G.out.append(" MPIR_Assert(0); /* auto cvar_val should be 0 and shouldn't be called here */") + G.out.append(" return %s;" % algo_id_END()) + if not name.startswith("i"): # blocking + G.out.append("case MPIR_CVAR_%s_%s_ALGORITHM_nb:" % (name.upper(), commkind.upper())) + G.out.append(" return %s__MPIR_Coll_nb;" % (algo_id_prefix)) + + func_commkind = name + '-' + commkind + for algo in G.algos[func_commkind]: + G.out.append("case MPIR_CVAR_%s_%s_ALGORITHM_%s:" % (name.upper(), commkind.upper(), algo['name'])) + G.out.append(" return %s__%s;" % (algo_id_prefix, get_algo_funcname(algo))) + dump_close("}") - # -- add shced_auto prototypes - sched_auto_params = get_func_params(params, name, "sched_auto") - add_prototype("int MPIR_%s_intra_sched_auto(%s)" % (Name, sched_auto_params)) - if not re.match(r'(scan|exscan|neighbor_)', name): - add_prototype("int MPIR_%s_inter_sched_auto(%s)" % (Name, sched_auto_params)) - - # -- switch - def dump_cnt_algo_tsp(algo, commkind): - G.out.append("MPII_GENTRAN_CREATE_SCHED_P();") - algo_name = get_algo_name(algo) - algo_args = get_algo_args(args, algo, "csel") - algo_params = get_algo_params(params, algo) - add_prototype("int MPIR_TSP_%s_sched_%s_%s(%s)" % (Name, commkind, algo_name, algo_params)) - dump_split(3, "mpi_errno = MPIR_TSP_%s_sched_%s_%s(%s);" % (Name, commkind, algo_name, algo_args)) - - def dump_cnt_algo_sched(algo, commkind): - G.out.append("MPII_SCHED_CREATE_SCHED_P();") - algo_name = get_algo_name(algo) - algo_args = get_algo_args(args, algo, "csel") - algo_params = get_algo_params(params, algo) - add_prototype("int MPIR_%s_%s_%s(%s)" % (Name, commkind, algo_name, algo_params)) - dump_split(3, "mpi_errno = MPIR_%s_%s_%s(%s);" % (Name, commkind, algo_name, algo_args)) - - dump_open("switch (cnt->id) {") - for commkind in ("intra", "inter"): - if commkind == "inter" and re.match(r'(scan|exscan|neighbor_)', name): - continue - for algo in G.algos[func_name + "-" + commkind]: - use_commkind = commkind - if "allcomm" in algo: - if commkind == "intra": - use_commkind = "allcomm" + decl = "int MPIR_Coll_cvar_to_algo_id(int coll_type, int cvar_val)" + add_prototype(decl) + G.out.append(decl) + dump_open("{") + G.out.append("MPIR_Assert(cvar_val > 0);") + dump_open("switch (coll_type) {") + for coll in G.coll_names: + for commkind in ("intra", "inter"): + for is_blocking in (True, False): + if is_blocking: + name = coll else: - # skip inter since it is covered already + name = 'i' + coll + if commkind == "inter" and re.match(r'(scan|exscan|neighbor_)', coll): continue - G.out.append("case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_%s_%s_%s:" % (Name, use_commkind, algo['name'])) - G.out.append("INDENT") - if algo['name'].startswith('tsp_'): - dump_cnt_algo_tsp(algo, use_commkind) - else: - dump_cnt_algo_sched(algo, use_commkind) - G.out.append("break;"); - G.out.append("DEDENT") - G.out.append("") - G.out.append("default:") - G.out.append(" MPIR_Assert(0);") - dump_close("}") - - # -- return - G.out.append("MPIR_ERR_CHECK(mpi_errno);") - dump_fn_exit() - dump_close("}") - -def dump_mpir_impl_blocking(name): - """ MPIR_Xxx_impl - """ - blocking_type = "blocking" - func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, "blocking") - func_args = get_func_args(args, name, "blocking") - - func_name = get_func_name(name, blocking_type) - Name = func_name.capitalize() - NAME = func_name.upper() - - need_fallback = False - - def dump_algo(algo, commkind): - if "allcomm" in algo: - commkind = "allcomm" - algo_name = get_algo_name(algo) - algo_args = get_algo_args(args, algo, "cvar") - dump_split(3, "mpi_errno = MPIR_%s_%s_%s(%s);" % (Name, commkind, algo_name, algo_args)) - - def dump_cases(commkind): - nonlocal need_fallback - CVAR_PREFIX = "MPIR_CVAR_%s_%s_ALGORITHM" % (NAME, commkind.upper()) - for algo in G.algos[func_name + '-' + commkind]: - if algo['name'] != "auto" and algo['name'] != "nb": - G.out.append("case %s_%s:" % (CVAR_PREFIX, algo['name'])) + G.out.append("case %s:" % coll_type(coll, is_blocking, commkind)) G.out.append("INDENT") - if 'restrictions' in algo: - dump_fallback(algo) - need_fallback = True - dump_algo(algo, commkind) - G.out.append("break;"); + dump_cvar_cases(name, commkind) + G.out.append("break;") G.out.append("DEDENT") - G.out.append("case %s_nb:" % CVAR_PREFIX) - dump_split(3, " mpi_errno = MPIR_%s_allcomm_nb(%s);" % (Name, func_args)) - G.out.append(" break;"); - G.out.append("case %s_auto:" % CVAR_PREFIX) - if commkind == "intra": - G.out.append("#ifdef MPIR_%s_fallback" % (Name)) - G.out.append(" if (!comm_ptr->csel_comm) {") - G.out.append(" mpi_errno = MPIR_%s_fallback(%s);" % (Name, func_args)) - G.out.append(" break;") - G.out.append(" }") - G.out.append("#endif"); - dump_split(3, " mpi_errno = MPIR_%s_allcomm_auto(%s);" % (Name, func_args)) - G.out.append(" break;"); - G.out.append("default:") - G.out.append(" MPIR_Assert(0);") + dump_close("}") + G.out.append("MPIR_Assert(0);") + G.out.append("return 0;") + dump_close("}") - # ---------------- - G.out.append("") - add_prototype("int MPIR_%s_impl(%s)" % (Name, func_params)) - dump_split(0, "int MPIR_%s_impl(%s)" % (Name, func_params)) - dump_open('{') - G.out.append("int mpi_errno = MPI_SUCCESS;") +def dump_MPIR_Coll_init_algo_container(): G.out.append("") - - dump_open("if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) {") - dump_open("switch (MPIR_CVAR_%s_INTRA_ALGORITHM) {" % NAME) - dump_cases("intra") + decl = "void MPIR_Coll_init_algo_container(MPIR_Csel_coll_sig_s * coll_sig, int algo_id, MPII_Csel_container_s * cnt)" + add_prototype(decl) + G.out.append(decl) + dump_open("{") + G.out.append("memset(cnt, 0, sizeof(*cnt));") + G.out.append("cnt->id = algo_id;") + dump_open("switch (algo_id) {") + for algo in G.algo_list: + if "extra_params" in algo: + struct_name = algo_struct_name(algo) + extra_params = algo['extra_params'].replace(' ', '').split(',') + cvar_params = algo['cvar_params'].replace(' ', '').split(',') + coll_name = algo['func-commkind'].split('-')[0] + G.out.append("case %s:" % algo_id(get_algo_funcname(algo))) + for i, a in enumerate(extra_params): + if re.match(r'\w+=(.+)', a): + # skip constant parameter + continue + else: + cvar_param = "MPIR_CVAR_%s_%s" % (coll_name.upper(), cvar_params[i]) + if a == "tree_type": + cvar_param = "get_tree_type_from_string(%s)" % cvar_param + elif cvar_params[i] == "THROTTLE": + cvar_param = "MPIR_CVAR_ALLTOALL_THROTTLE" + G.out.append(" cnt->u.%s.%s = %s;" % (struct_name, a, cvar_param)) + G.out.append(" break;") dump_close("}") - dump_else() - if re.match(r'(scan|exscan|neighbor_)', name): - G.out.append("MPIR_Assert_error(\"Only intra-communicator allowed\");") - else: - dump_open("switch (MPIR_CVAR_%s_INTER_ALGORITHM) {" % NAME) - dump_cases("inter") - dump_close("}") dump_close("}") - G.out.append("MPIR_ERR_CHECK(mpi_errno);") - if need_fallback: - G.out.append("goto fn_exit;") - G.out.append("") - G.out.append("fallback:") - dump_split(1, "mpi_errno = MPIR_%s_allcomm_auto(%s);" % (Name, func_args)) +def dump_MPIR_Coll_check_algo_restriction(): G.out.append("") - dump_fn_exit() + def dump_check_restriction(restriction): + r = restriction + negate = False + if restriction.startswith('!'): + r = restriction[1:] + negate = True + if RE.match(r'.*\(.*\)', r): + raise Exception("Threshold condition %s cannot be used as a restriction" % r) + + cond = None + if r in G.conditions: + cond = "%s(coll_sig)" % G.conditions[r] + else: + raise Exception("Restriction %s not listed" % restriction) + + if negate: + G.out.append(" if (%s) return false;" % cond) + else: + G.out.append(" if (!%s) return false;" % cond) + + decl = "bool MPIR_Coll_check_algo_restriction(MPIR_Csel_coll_sig_s * coll_sig, int algo_id)" + add_prototype(decl) + G.out.append(decl) + dump_open("{") + dump_open("switch (algo_id) {") + for algo in G.algo_list: + if "restrictions" in algo: + restrictions = algo['restrictions'].replace(' ', '').split(',') + G.out.append("case %s:" % algo_id(get_algo_funcname(algo))) + for r in restrictions: + dump_check_restriction(r) + G.out.append(" break;") dump_close("}") + G.out.append("return true;") + dump_close("}") + +# e.g. MPIR_CSEL_COLL_TYPE__BARRIER, etc. +def dump_MPIR_Csel_coll_type_e(): + G.out2.append("") + G.out2.append("typedef enum {") + # IMPORTANT: MPIR_Coll_nb algorithm relies on that blocking coll_type is even and its + # nonblocking correspondent is coll_type+1. + for a in G.coll_names: + for commkind in ("intra", "inter"): + for is_blocking in (True, False): + G.out2.append(" %s," % coll_type(a, is_blocking, commkind)) + G.out2.append(" %s" % coll_type_END()) + G.out2.append("} MPIR_Csel_coll_type_e;") + G.out2.append("") + G.out2.append("#define MPIR_CSEL_NUM_COLL_TYPES %s\n" % coll_type_END()) + +def dump_MPIR_Csel_container_type_e(): + G.out2.append("") + G.out2.append("typedef enum {") + for a in G.algo_list: + algo_funcname = get_algo_funcname(a) + G.out2.append(" %s," % algo_id(algo_funcname)) + G.out2.append(" %s" % algo_id_END()) + G.out2.append("} MPIR_Csel_container_type_e;") + G.out2.append("") + G.out2.append("#define MPIR_CSEL_NUM_ALGORITHMS %s\n" % algo_id_END()) + +def dump_MPIR_Csel_node_type_e(): + G.out2.append("") + G.out2.append("typedef enum {") + for a in G.conditions: + G.out2.append(" %s," % condition_id(a)) + G.out2.append(" CSEL_NODE_TYPE__OPERATOR__COLLECTIVE,") + G.out2.append(" CSEL_NODE_TYPE__OPERATOR__ANY,") + G.out2.append(" CSEL_NODE_TYPE__OPERATOR__CALL,") + G.out2.append(" CSEL_NODE_TYPE__CONTAINER,") + G.out2.append("} MPIR_Csel_node_type_e;") + G.out2.append("") + G.out2.append("#define MPIR_CSEL_NUM_CONDITIONS %s\n" % "CSEL_NODE_TYPE__OPERATOR__COLLECTIVE") + +def dump_MPIR_Csel_node_s(): + G.out2.append("") + G.out2.append("typedef struct MPIR_Csel_node {") + G.out2.append(" MPIR_Csel_node_type_e type;") + G.out2.append(" MPI_Aint thresh;") + G.out2.append(" struct MPIR_Csel_node *success;") + G.out2.append(" struct MPIR_Csel_node *failure;") + G.out2.append("} MPIR_Csel_node_s;") + +def dump_MPII_Csel_container(): + G.out2.append("") + def dump_algo_params(): + for algo in G.algo_list: + if 'extra_params' in algo: + extra_params = algo['extra_params'].replace(' ', '').split(',') + G.out2.append(" struct {") + for a in extra_params: + if re.match(r'\w+=(.+)', a): + # skip constant parameter + continue + else: + G.out2.append(" int %s;" % a) + G.out2.append(" } %s;" % algo_struct_name(algo)) + + G.out2.append("typedef struct MPII_Csel_container {") + G.out2.append(" MPIR_Csel_container_type_e id;") + G.out2.append(" union {") + dump_algo_params() + G.out2.append(" } u;") + G.out2.append("} MPII_Csel_container_s;") + +#---------------------------------------- +def add_prototype(l): + G.prototypes.append(l) -def dump_sched_impl(name): - """ MPIR_Xxx_impl - """ - blocking_type = "nonblocking" +def load_coll_algos(algo_txt): + G.algos = {} + G.conditions = {} + with open(algo_txt) as In: + (func_commkind, algo_list, algo) = (None, None, None) + for line in In: + if RE.match(r'(\w+-(intra|inter)|general|conditions):', line): + func_commkind = RE.m.group(1) + if func_commkind != "conditions": + algo_list = [] + G.algos[func_commkind] = algo_list + elif func_commkind == "conditions": + if RE.match(r'\s+([\w()-]+):\s*(\w+)', line): + G.conditions[RE.m.group(1)] = RE.m.group(2) + elif func_commkind: + if RE.match(r'\s+(\w+)\s*$', line): + algo = {"name": RE.m.group(1), "func-commkind": func_commkind} + algo_list.append(algo) + elif RE.match(r'\s+(\w+):\s*(.+)', line): + algo[RE.m.group(1)] = RE.m.group(2) + +def dump_coll_impl(name, blocking_type): func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, "sched_impl") + func_params = get_func_params(func, name, blocking_type) func_name = get_func_name(name, blocking_type) Name = func_name.capitalize() - NAME = func_name.upper() - - need_fallback = False - - def dump_algo_tsp(algo, commkind): - G.out.append("MPII_GENTRAN_CREATE_SCHED_P();") - algo_name = get_algo_name(algo) - algo_args = get_algo_args(args, algo, "cvar") - if "allcomm" in algo: - commkind = "allcomm" - dump_split(3, "mpi_errno = MPIR_TSP_%s_sched_%s_%s(%s);" % (Name, commkind, algo_name, algo_args)) - - def dump_algo_sched(algo, commkind): - G.out.append("MPII_SCHED_CREATE_SCHED_P();") - algo_name = get_algo_name(algo) - algo_args = get_algo_args(args, algo, "cvar") - if "allcomm" in algo: - commkind = "allcomm" - dump_split(3, "mpi_errno = MPIR_%s_%s_%s(%s);" % (Name, commkind, algo_name, algo_args)) - - def dump_cases(commkind): - nonlocal need_fallback - CVAR_PREFIX = "MPIR_CVAR_%s_%s_ALGORITHM" % (NAME, commkind.upper()) - for algo in G.algos[func_name + '-' + commkind]: - if algo['name'] != "auto" and algo['name'] != "nb": - G.out.append("case %s_%s:" % (CVAR_PREFIX, algo['name'])) - G.out.append("INDENT") - if 'restrictions' in algo: - dump_fallback(algo) - need_fallback = True - if algo['name'].startswith('tsp_'): - dump_algo_tsp(algo, commkind) - else: - dump_algo_sched(algo, commkind) - G.out.append("break;"); - G.out.append("DEDENT") - G.out.append("case %s_auto:" % CVAR_PREFIX) - func_args = get_func_args(args, name, "allcomm_sched_auto") - dump_split(3, " mpi_errno = MPIR_%s_allcomm_sched_auto(%s);" % (Name, func_args)) - G.out.append(" break;"); - G.out.append("default:") - G.out.append(" MPIR_Assert(0);") - # ---------------- G.out.append("") - add_prototype("int MPIR_%s_sched_impl(%s)" % (Name, func_params)) - dump_split(0, "int MPIR_%s_sched_impl(%s)" % (Name, func_params)) + add_prototype("int MPIR_%s_impl(%s)" % (Name, func_params)) + dump_split(0, "int MPIR_%s_impl(%s)" % (Name, func_params)) dump_open('{') G.out.append("int mpi_errno = MPI_SUCCESS;") G.out.append("") + # Initialize coll_sig + G.out.append("MPIR_Csel_coll_sig_s coll_sig;") + NAME = name.upper() + if blocking_type != "blocking": + # nonblocking and persistent + NAME = 'I' + NAME dump_open("if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) {") - dump_open("switch (MPIR_CVAR_%s_INTRA_ALGORITHM) {" % NAME) - dump_cases("intra") - dump_close("}") + G.out.append("coll_sig.coll_type = MPIR_CSEL_COLL_TYPE__INTRA_%s;" % NAME) dump_else() - if re.match(r'(scan|exscan|neighbor_)', name): - G.out.append("MPIR_Assert_error(\"Only intra-communicator allowed\");") + G.out.append("coll_sig.coll_type = MPIR_CSEL_COLL_TYPE__INTER_%s;" % NAME) + dump_close('}') + G.out.append("coll_sig.comm_ptr = comm_ptr;") + if blocking_type == "persistent": + G.out.append("coll_sig.is_persistent = true;") else: - dump_open("switch (MPIR_CVAR_%s_INTER_ALGORITHM) {" % NAME) - dump_cases("inter") - dump_close("}") - dump_close("}") + G.out.append("coll_sig.is_persistent = false;") + G.out.append("coll_sig.sched = NULL;") - G.out.append("MPIR_ERR_CHECK(mpi_errno);") - if need_fallback: - G.out.append("goto fn_exit;") - G.out.append("") - G.out.append("fallback:") - func_args = get_func_args(args, name, "allcomm_sched_auto") - dump_split(1, "mpi_errno = MPIR_%s_allcomm_sched_auto(%s);" % (Name, func_args)) - G.out.append("") - dump_fn_exit() - dump_close("}") + G.out.append("memset(&coll_sig.cache, 0, sizeof(coll_sig.cache));"); -def dump_mpir_impl_nonblocking(name): - blocking_type = "nonblocking" - func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, "nonblocking") + phash = {} + for p in func['parameters']: + if p['name'] == 'comm': + pass + else: + phash[p['name']] = 1 + G.out.append("coll_sig.u.%s.%s = %s;" % (name, p['name'], p['name'])) - func_name = get_func_name(name, blocking_type) - Name = func_name.capitalize() - NAME = func_name.upper() + G.out.append("MPIR_Init_coll_sig(&coll_sig);") + G.out.append("MPID_Init_coll_sig(&coll_sig);") + # Call csel G.out.append("") - add_prototype("int MPIR_%s_impl(%s)" % (Name, func_params)) - dump_split(0, "int MPIR_%s_impl(%s)" % (Name, func_params)) - dump_open('{') - G.out.append("int mpi_errno = MPI_SUCCESS;") - G.out.append("enum MPIR_sched_type sched_type;") - G.out.append("void *sched;") - G.out.append("") - G.out.append("*request = NULL;") - func_args = get_func_args(args, name, "mpir_impl_nonblocking") - dump_split(1, "mpi_errno = MPIR_%s_sched_impl(%s);" % (Name, func_args)) + G.out.append("mpi_errno = MPIR_Coll_json(&coll_sig);") G.out.append("MPIR_ERR_CHECK(mpi_errno);") - G.out.append("MPII_SCHED_START(sched_type, sched, comm_ptr, request);") G.out.append("") - G.out.append("fn_exit:") - G.out.append("return mpi_errno;") - G.out.append("fn_fail:") - G.out.append("goto fn_exit;") - dump_close("}") - -def dump_mpir_impl_persistent(name): - blocking_type = "persistent" - func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, "persistent") - func_name = get_func_name(name, blocking_type) - Name = func_name.capitalize() - NAME = func_name.upper() + # Set request if nonblocking or persistent + if blocking_type == "blocking": + pass + elif blocking_type == "nonblocking": + G.out.append("MPII_SCHED_START(coll_sig.sched_type, coll_sig.sched, comm_ptr, request);") + G.out.append("") + elif blocking_type == "persistent": + G.out.append("MPIR_Request *req = MPIR_Request_create(MPIR_REQUEST_KIND__PREQUEST_COLL);") + G.out.append("MPIR_ERR_CHKANDJUMP(!req, mpi_errno, MPI_ERR_OTHER, \"**nomem\");") + G.out.append("MPIR_Comm_add_ref(comm_ptr);") + G.out.append("req->comm = comm_ptr;") + G.out.append("MPIR_Comm_save_inactive_request(comm_ptr, req);") + G.out.append("req->u.persist_coll.sched_type = coll_sig.sched_type;") + G.out.append("req->u.persist_coll.sched = coll_sig.sched;") + G.out.append("*request = req;") + G.out.append("") + else: + raise Exception("Wrong blocking_type") - G.out.append("") - add_prototype("int MPIR_%s_impl(%s)" % (Name, func_params)) - dump_split(0, "int MPIR_%s_impl(%s)" % (Name, func_params)) - dump_open('{') - G.out.append("int mpi_errno = MPI_SUCCESS;") - G.out.append("") - G.out.append("MPIR_Request *req = MPIR_Request_create(MPIR_REQUEST_KIND__PREQUEST_COLL);") - G.out.append("MPIR_ERR_CHKANDJUMP(!req, mpi_errno, MPI_ERR_OTHER, \"**nomem\");") - G.out.append("MPIR_Comm_add_ref(comm_ptr);") - G.out.append("req->comm = comm_ptr;") - G.out.append("MPIR_Comm_save_inactive_request(comm_ptr, req);") - G.out.append("req->u.persist_coll.sched_type = MPIR_SCHED_INVALID;") - G.out.append("req->u.persist_coll.real_request = NULL;") - - func_args = get_func_args(args, name, "mpir_impl_persistent") - dump_split(1, "mpi_errno = MPIR_I%s_sched_impl(%s);" % (name, func_args)) - G.out.append("MPIR_ERR_CHECK(mpi_errno);") - G.out.append("") - G.out.append("*request = req;") - G.out.append("") G.out.append("fn_exit:") G.out.append("return mpi_errno;") G.out.append("fn_fail:") G.out.append("goto fn_exit;") - dump_close("}") + dump_close('}') + +def dump_coll_mpir(name, blocking_type): + def get_func_args(func): + args = [] + for p in func['parameters']: + if p['name'] == 'comm': + args.append('comm_ptr') + else: + args.append(p['name']) + return ', '.join(args) -def dump_mpir(name, blocking_type): - """ MPIR_Xxx - """ func = G.FUNCS["mpi_" + name] - params, args = get_params_and_args(func) - func_params = get_func_params(params, name, blocking_type) - func_args = get_func_args(args, name, blocking_type) + func_params = get_func_params(func, name, blocking_type) + func_args = get_func_args(func) + if blocking_type == "blocking": + func_params += ", int coll_attr" + else: + func_args += ", request" func_name = get_func_name(name, blocking_type) Name = func_name.capitalize() - NAME = func_name.upper() - - def dump_buffer_swap_pre(): - G.out.append("void *in_recvbuf = recvbuf;") - G.out.append("void *host_sendbuf = NULL;") - G.out.append("void *host_recvbuf = NULL;") - G.out.append("") - if name == "reduce_scatter": - G.out.append("MPI_Aint count = 0;") - G.out.append("for (int i = 0; i < MPIR_Comm_size(comm_ptr); i++) {") - G.out.append(" count += recvcounts[i];") - G.out.append("}") - G.out.append("") - elif name == "reduce_scatter_block": - G.out.append("MPI_Aint count = MPIR_Comm_size(comm_ptr) * recvcount;") - - if name == "reduce": - use_recvbuf = "(comm_ptr->rank == root || root == MPI_ROOT) ? recvbuf : NULL" - else: - use_recvbuf = "recvbuf" - - G.out.append("if(!MPIR_Typerep_reduce_is_supported(op, count, datatype))") - G.out.append(" MPIR_Coll_host_buffer_alloc(sendbuf, %s, count, datatype, &host_sendbuf, &host_recvbuf);" % use_recvbuf) - G.out.append("") - - for buf in ("sendbuf", "recvbuf"): - G.out.append("if (host_%s) {" % buf); - G.out.append(" %s = host_%s;" % (buf, buf)); - G.out.append("}") - G.out.append("") - - def dump_buffer_swap_post(): - count = "count" - if name == "reduce_scatter": - count = "recvcounts[comm_ptr->rank]" - elif name == "reduce_scatter_block": - count = "recvcount" - - if blocking_type == "blocking": - G.out.append("if (host_recvbuf) {") - G.out.append(" recvbuf = in_recvbuf;") - G.out.append(" MPIR_Localcopy(host_recvbuf, count, datatype, recvbuf, count, datatype);") - G.out.append("}") - G.out.append("MPIR_Coll_host_buffer_free(host_sendbuf, host_recvbuf);") - elif blocking_type == "nonblocking": - G.out.append("MPIR_Coll_host_buffer_swap_back(host_sendbuf, host_recvbuf, in_recvbuf, %s, datatype, *request);" % count) - elif blocking_type == "persistent": - G.out.append("MPIR_Coll_host_buffer_persist_set(host_sendbuf, host_recvbuf, in_recvbuf, %s, datatype, *request);" % count) G.out.append("") add_prototype("int MPIR_%s(%s)" % (Name, func_params)) dump_split(0, "int MPIR_%s(%s)" % (Name, func_params)) dump_open('{') - G.out.append("int mpi_errno = MPI_SUCCESS;") - G.out.append("") - - need_buffer_swap = False - if re.match(r'(reduce|allreduce|scan|exscan|reduce_scatter)', name): - need_buffer_swap = True - if need_buffer_swap: - dump_buffer_swap_pre() - - cond1 = "MPIR_CVAR_DEVICE_COLLECTIVES == MPIR_CVAR_DEVICE_COLLECTIVES_all" - cond2 = "MPIR_CVAR_DEVICE_COLLECTIVES == MPIR_CVAR_DEVICE_COLLECTIVES_percoll" - cond3 = "MPIR_CVAR_%s_DEVICE_COLLECTIVE" % NAME - G.out.append("if ((%s) ||" % cond1) - G.out.append(" ((%s) &&" % cond2) - G.out.append(" %s)) {" % cond3) - G.out.append("INDENT") - dump_split(2, "mpi_errno = MPID_%s(%s);" % (Name, func_args)) - dump_else() - dump_split(2, "mpi_errno = MPIR_%s_impl(%s);" % (Name, func_args)) - dump_close("}") - if need_buffer_swap: - dump_buffer_swap_post() - G.out.append("") - G.out.append("return mpi_errno;") - dump_close("}") - -# ---- -def dump_fallback(algo): - cond_list = [] - for a in algo['restrictions'].replace(" ","").split(','): - if a == "inplace": - cond_list.append("sendbuf == MPI_IN_PLACE") - elif a == "noinplace": - cond_list.append("sendbuf != MPI_IN_PLACE") - elif a == "power-of-two": - cond_list.append("MPL_is_pof2(comm_ptr->local_size)") - elif a == "size-ge-pof2": - cond_list.append("count >= MPL_pof2(comm_ptr->local_size)") - elif a == "commutative": - cond_list.append("MPIR_Op_is_commutative(op)") - elif a== "builtin-op": - cond_list.append("HANDLE_IS_BUILTIN(op)") - elif a == "parent-comm": - cond_list.append("MPIR_Comm_is_parent_comm(comm_ptr)") - elif a == "node-consecutive": - cond_list.append("MPII_Comm_is_node_consecutive(comm_ptr)") - elif a == "displs-ordered": - # assume it's allgatherv - cond_list.append("MPII_Iallgatherv_is_displs_ordered(comm_ptr->local_size, recvcounts, displs)") - else: - raise Exception("Unsupported restrictions - %s" % a) - (func_name, commkind) = algo['func-commkind'].split('-') - G.out.append("MPII_COLLECTIVE_FALLBACK_CHECK(comm_ptr->rank, %s, mpi_errno," % ' && '.join(cond_list)) - G.out.append(" \"%s %s cannot be applied.\\n\");" % (func_name.capitalize(), algo['name'])) + G.out.append("return MPIR_%s_impl(%s);" % (Name, func_args)) + dump_close('}') # ---- def get_func_name(name, blocking_type): @@ -588,23 +680,58 @@ def get_func_name(name, blocking_type): elif blocking_type == "persistent": return name + "_init" -def get_params_and_args(func): - mapping = G.MAPS['SMALL_C_KIND_MAP'] +def get_algo_coll_name(algo): + if algo["func-commkind"] == "general": + raise Exception("general algo!") - params = [] + func_name, commkind = algo["func-commkind"].split("-") + if func_name.startswith('i'): + return func_name[1:] + else: + return func_name + +def get_algo_funcname(algo): + if algo["func-commkind"] == "general": + return algo['name'] + + func_name, commkind = algo["func-commkind"].split("-") + if 'allcomm' in algo: + commkind = 'allcomm' + Name = func_name.capitalize() + if func_name.startswith('i'): + if algo['name'].startswith('tsp_'): + return "MPIR_TSP_%s_sched_%s_%s" % (Name, commkind, get_algo_name(algo)) + else: + return "MPIR_%s_%s_%s" % (Name, commkind, get_algo_name(algo)) + else: + return "MPIR_%s_%s_%s" % (Name, commkind, get_algo_name(algo)) + +def get_coll_args(coll_name, kind): + func = G.FUNCS["mpi_" + coll_name] args = [] + if kind == "csel": + for p in func['parameters']: + if p['name'] == 'comm': + args.append("coll_sig->comm_ptr") + else: + args.append("coll_sig->u.%s.%s" % (coll_name, p['name'])) + else: + raise Exception("unexpected kind") + return ', '.join(args) + +def get_coll_params(coll_name): + func = G.FUNCS["mpi_" + coll_name] + mapping = G.MAPS['SMALL_C_KIND_MAP'] + params = [] for p in func['parameters']: if p['name'] == 'comm': params.append("MPIR_Comm * comm_ptr") - args.append("comm_ptr") else: s = get_C_param(p, func, mapping) if p['kind'].startswith('POLY'): s = re.sub(r'\bint ', 'MPI_Aint ', s) params.append(s) - args.append(p['name']) - - return (', '.join(params), ', '.join(args)) + return ', '.join(params) def get_algo_extra_args(algo, kind): (func_name, commkind) = algo['func-commkind'].split('-') @@ -620,7 +747,7 @@ def get_algo_extra_args(algo, kind): out_list.append(RE.m.group(1)) else: if kind == "csel": - prefix = "cnt->u.%s.%s_%s." % (func_name, commkind, algo['name']) + prefix = "cnt->u.%s." % (algo_struct_name(algo)) out_list.append(prefix + extra_params[i]) elif kind == "cvar": prefix = "MPIR_CVAR_%s_" % func_name.upper() @@ -649,82 +776,71 @@ def get_algo_extra_params(algo): return ', '.join(out_list) # additional wrappers -def get_algo_args(args, algo, kind): - algo_args = args - if 'extra_params' in algo: - algo_args += ", " + get_algo_extra_args(algo, kind) - +def get_algo_name(algo): + # the name used in algo function name if algo['name'].startswith('tsp_'): - algo_args += ", *sched_p" - elif algo['func-commkind'].startswith('i'): - algo_args += ", *sched_p" - elif not algo['func-commkind'].startswith('neighbor_'): - algo_args += ", coll_attr" + return algo['name'][4:] + else: + return algo['name'] - return algo_args +def get_func_params(func, name, blocking_type): + mapping = G.MAPS['SMALL_C_KIND_MAP'] -def get_algo_params(params, algo): - algo_params = params - if 'extra_params' in algo: - algo_params += ", " + get_algo_extra_params(algo) + params = [] + for p in func['parameters']: + if p['name'] == 'comm': + params.append("MPIR_Comm * comm_ptr") + else: + s = get_C_param(p, func, mapping) + if p['kind'].startswith('POLY'): + s = re.sub(r'\bint ', 'MPI_Aint ', s) + params.append(s) - if algo['name'].startswith('tsp_'): - algo_params += ", MPIR_TSP_sched_t sched" - elif algo['func-commkind'].startswith('i'): - algo_params += ", MPIR_Sched_t s" - elif not algo['func-commkind'].startswith('neighbor_'): - algo_params += ", int coll_attr" + if blocking_type == "blocking": + pass + elif blocking_type == "nonblocking": + params.append("MPIR_Request ** request") + elif blocking_type == "persistent": + params.append("MPIR_Info * info_ptr") + params.append("MPIR_Request ** request") + else: + raise Exception("get_func_params - unexpected blocking_type = %s" % blocking_type) - return algo_params + return ', '.join(params) -def get_algo_name(algo): - # the name used in algo function name - if "func_name" in algo: - return algo['func_name'] - elif algo['name'].startswith('tsp_'): - return algo['name'][4:] +def coll_type(coll_name, is_blocking, commkind): + prefix = "MPIR_CSEL_COLL_TYPE" + if is_blocking: + return "%s__%s_%s" % (prefix, commkind.upper(), coll_name.upper()) else: - return algo['name'] + return "%s__%s_I%s" % (prefix, commkind.upper(), coll_name.upper()) -def get_func_params(params, name, kind): - func_params = params - if kind == "blocking": - if not name.startswith('neighbor_'): - func_params += ", int coll_attr" - elif kind == "nonblocking": - func_params += ", MPIR_Request ** request" - elif kind == "persistent": - func_params += ", MPIR_Info * info_ptr, MPIR_Request ** request" - elif kind == "sched_auto": - func_params += ", MPIR_Sched_t s" - elif kind == "allcomm_sched_auto": - func_params += ", bool is_persistent, void **sched_p, enum MPIR_sched_type *sched_type_p" - elif kind == "sched_impl": - func_params += ", bool is_persistent, void **sched_p, enum MPIR_sched_type *sched_type_p" +def coll_type_END(): + return "MPIR_CSEL_COLL_TYPE__END" + +def cvar_name(coll_name, is_blocking, commkind): + if is_blocking: + return "MPIR_CVAR_%s_%s_ALGORITHM" % (coll_name.upper(), commkind.upper()) else: - raise Exception("get_func_params - unexpected kind = %s" % kind) + return "MPIR_CVAR_I%s_%s_ALGORITHM" % (coll_name.upper(), commkind.upper()) - return func_params +def algo_id(algo_funcname): + prefix = "MPII_CSEL_CONTAINER_TYPE__ALGORITHM" + return "%s__%s" % (prefix, algo_funcname) -def get_func_args(args, name, kind): - func_args = args - if kind == "blocking": - if not name.startswith('neighbor_'): - func_args += ", coll_attr" - elif kind == "nonblocking": - func_args += ", request" - elif kind == "persistent": - func_args += ", info_ptr, request" - elif kind == "allcomm_sched_auto": - func_args += ", is_persistent, sched_p, sched_type_p" - elif kind == "mpir_impl_nonblocking": - func_args += ", false, &sched, &sched_type" - elif kind == "mpir_impl_persistent": - func_args += ", true, &req->u.persist_coll.sched, &req->u.persist_coll.sched_type" - else: - raise Exception("get_func_args - unexpected kind = %s" % kind) +def algo_id_END(): + return "MPII_CSEL_CONTAINER_TYPE__ALGORITHM__END" + +def condition_id(name): + prefix = "CSEL_NODE_TYPE__OPERATOR__" + a = re.sub(r'-', '_', name) + a = re.sub(r'\(thresh\)$', '', a) + return prefix + a - return func_args +def algo_struct_name(algo): + algo_funcname = get_algo_funcname(algo) + struct_name = re.sub(r'MPIR_', '', algo_funcname).lower() + return struct_name # ---------------------- def dump_c_file(f, lines): @@ -750,14 +866,19 @@ def dump_c_file(f, lines): print(" " * indent, end='', file=Out) print(l, file=Out) -def dump_prototypes(f, prototypes): +def dump_coll_algos_h(f, prototypes, lines): print(" --> [%s]" % f) with open(f, "w") as Out: for l in G.copyright_c: print(l, file=Out) + print("#ifndef COLL_ALGOS_H_INCLUDED", file=Out) print("#define COLL_ALGOS_H_INCLUDED", file=Out) print("", file=Out) + + for l in lines: + print(l, file=Out) + for l in prototypes: lines = split_line_with_break(l + ';', '', 80) for l2 in lines: @@ -788,6 +909,16 @@ def dump_split(indent, l): tlist = split_line_with_break(l, "", 100 - indent * 4) G.out.extend(tlist) +def dump_macro_open(macro, dowhile=False): + G.out2.append("#define %s \\" % macro) + if dowhile: + G.out2.append(" do { \\") + +def dump_macro_close(dowhile=False): + if dowhile: + G.out2.append(" } while (0)") + G.out2.append("") + # --------------------------------------------------------- if __name__ == "__main__": main() diff --git a/maint/tuning/coll/json_gen.sh b/maint/json_gen.sh similarity index 75% rename from maint/tuning/coll/json_gen.sh rename to maint/json_gen.sh index 7413a32bbb6..00bbeb11b59 100755 --- a/maint/tuning/coll/json_gen.sh +++ b/maint/json_gen.sh @@ -33,6 +33,4 @@ cat > $cfile<handle;") - elif RE.match(r'mpi_neighbor_', func['name'], re.IGNORECASE): - dump_line_with_break("mpi_errno = %s(%s);" % (mpir_name, args)) - dump_error_check("") - else: - # blocking collectives - dump_line_with_break("mpi_errno = %s(%s, 0);" % (mpir_name, args)) - dump_error_check("") - def dump_coll_v_swap(func): # -- wrappers to make code cleaner def replace_arg(old, new): @@ -2064,8 +2033,6 @@ def dump_body_normal(): dump_body_topo_fns(func, RE.m.group(1)) else: print("Error: unhandled special impl: [%s]" % func['impl']) - elif func['dir'] == 'coll': - dump_body_coll(func) else: dump_body_impl(func, "mpir") diff --git a/maint/tuning/coll/ch4/generic.json b/maint/tuning/coll/ch4/generic.json deleted file mode 100644 index aa94ba85d5b..00000000000 --- a/maint/tuning/coll/ch4/generic.json +++ /dev/null @@ -1,213 +0,0 @@ -{ - "collective=bcast": - { - "comm_type=intra": - { - "comm_hierarchy=parent": - { - "comm_size<8": - { - "avg_msg_size=any": - { - "composition=MPIDI_Bcast_intra_composition_alpha":{} - } - }, - "comm_size=pow2": - { - "avg_msg_size<12288": - { - "composition=MPIDI_Bcast_intra_composition_alpha":{} - }, - "avg_msg_size<524288": - { - "composition=MPIDI_Bcast_intra_composition_beta":{} - }, - "avg_msg_size=any": - { - "composition=MPIDI_Bcast_intra_composition_gamma":{} - } - }, - "comm_size=any": - { - "avg_msg_size<12288": - { - "composition=MPIDI_Bcast_intra_composition_alpha":{} - }, - "avg_msg_size=any": - { - "composition=MPIDI_Bcast_intra_composition_gamma":{} - } - } - }, - "comm_hierarchy=any": - { - "composition=MPIDI_Bcast_intra_composition_gamma":{} - } - } - }, - "collective=allreduce": - { - "comm_type=intra": - { - "comm_size=node_comm_size": - { - "composition=MPIDI_Allreduce_intra_composition_gamma":{} - }, - "comm_size=any": - { - "comm_hierarchy=parent": - { - "is_commutative=yes": - { - "avg_msg_size=any": - { - "composition=MPIDI_Allreduce_intra_composition_alpha":{} - } - }, - "is_commutative=no": - { - "composition=MPIDI_Allreduce_intra_composition_beta":{} - } - }, - "comm_hierarchy=any": - { - "composition=MPIDI_Allreduce_intra_composition_beta":{} - } - } - } - }, - "collective=reduce": - { - "comm_type=intra": - { - "comm_hierarchy=parent": - { - "is_commutative=yes": - { - "avg_msg_size<=2048": - { - "composition=MPIDI_Reduce_intra_composition_beta":{} - }, - "avg_msg_size=any": - { - "composition=MPIDI_Reduce_intra_composition_alpha":{} - } - }, - "is_commutative=no": - { - "composition=MPIDI_Reduce_intra_composition_gamma":{} - } - }, - "comm_hierarchy=any": - { - "composition=MPIDI_Reduce_intra_composition_gamma":{} - } - } - }, - "collective=barrier": - { - "comm_type=intra": - { - "comm_hierarchy=parent": - { - "composition=MPIDI_Barrier_intra_composition_alpha":{} - }, - "comm_hierarchy=any": - { - "composition=MPIDI_Barrier_intra_composition_beta":{} - } - } - }, - "collective=gather": - { - "comm_type=intra": - { - "composition=MPIDI_Gather_intra_composition_alpha":{} - } - }, - "collective=gatherv": - { - "comm_type=intra": - { - "composition=MPIDI_Gatherv_intra_composition_alpha":{} - } - }, - "collective=scatter": - { - "comm_type=intra": - { - "composition=MPIDI_Scatter_intra_composition_alpha":{} - } - }, - "collective=scatterv": - { - "comm_type=intra": - { - "composition=MPIDI_Scatterv_intra_composition_alpha":{} - } - }, - "collective=alltoall": - { - "comm_type=intra": - { - "composition=MPIDI_Alltoall_intra_composition_beta":{} - } - }, - "collective=alltoallv": - { - "comm_type=intra": - { - "composition=MPIDI_Alltoallv_intra_composition_alpha":{} - } - }, - "collective=alltoallw": - { - "comm_type=intra": - { - "composition=MPIDI_Alltoallw_intra_composition_alpha":{} - } - }, - "collective=allgather": - { - "comm_type=intra": - { - "composition=MPIDI_Allgather_intra_composition_beta":{} - } - }, - "collective=allgatherv": - { - "comm_type=intra": - { - "composition=MPIDI_Allgatherv_intra_composition_alpha":{} - } - }, - "collective=reduce_scatter": - { - "comm_type=intra": - { - "composition=MPIDI_Reduce_scatter_intra_composition_alpha":{} - } - }, - "collective=reduce_scatter_block": - { - "comm_type=intra": - { - "composition=MPIDI_Reduce_scatter_block_intra_composition_alpha":{} - } - }, - "collective=scan": - { - "is_node_consecutive=yes": - { - "composition=MPIDI_Scan_intra_composition_alpha":{} - }, - "is_node_consecutive=no": - { - "composition=MPIDI_Scan_intra_composition_beta":{} - } - }, - "collective=exscan": - { - "composition=MPIDI_Exscan_intra_composition_alpha":{} - } -} diff --git a/maint/tuning/coll/ch4/posix_generic.json b/maint/tuning/coll/ch4/posix_generic.json deleted file mode 100644 index 87ad336ba48..00000000000 --- a/maint/tuning/coll/ch4/posix_generic.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "collective=bcast": - { - "comm_type=intra": - { - "is_multi_threaded=no": - { - "algorithm=MPIDI_POSIX_mpi_bcast_release_gather":{} - } - } - }, - "collective=barrier": - { - "comm_type=intra": - { - "is_multi_threaded=no": - { - "algorithm=MPIDI_POSIX_mpi_barrier_release_gather":{} - } - } - }, - "collective=allreduce": - { - "comm_type=intra": - { - "is_multi_threaded=no": - { - "is_commutative=yes": - { - "algorithm=MPIDI_POSIX_mpi_allreduce_release_gather":{} - } - } - } - }, - "collective=reduce": - { - "comm_type=intra": - { - "is_multi_threaded=no": - { - "is_commutative=yes": - { - "algorithm=MPIDI_POSIX_mpi_reduce_release_gather":{} - } - } - } - } -} diff --git a/maint/tuning/coll/mpir/generic.json b/maint/tuning/coll/mpir/generic.json deleted file mode 100644 index 02b1bf36155..00000000000 --- a/maint/tuning/coll/mpir/generic.json +++ /dev/null @@ -1,965 +0,0 @@ -{ - "collective=bcast": - { - "comm_type=intra": - { - "comm_size<8": - { - "comm_hierarchy=parent": - { - "avg_msg_size<=0": - { - "algorithm=MPIR_Bcast_intra_smp":{} - }, - "avg_msg_size=any": - { - "algorithm=MPIR_Bcast_intra_binomial":{} - } - }, - "comm_hierarchy=any": - { - "avg_msg_size=any": - { - "algorithm=MPIR_Bcast_intra_binomial":{} - } - } - }, - "comm_size=pow2": - { - "comm_hierarchy=parent": - { - "avg_msg_size<=0": - { - "algorithm=MPIR_Bcast_intra_smp":{} - }, - "avg_msg_size<=12288": - { - "algorithm=MPIR_Bcast_intra_binomial":{} - }, - "avg_msg_size<=524288": - { - "algorithm=MPIR_Bcast_intra_scatter_recursive_doubling_allgather":{} - }, - "avg_msg_size=any": - { - "algorithm=MPIR_Bcast_intra_scatter_ring_allgather":{} - } - }, - "comm_hierarchy=any": - { - "avg_msg_size<=12288": - { - "algorithm=MPIR_Bcast_intra_binomial":{} - }, - "avg_msg_size<=524288": - { - "algorithm=MPIR_Bcast_intra_scatter_recursive_doubling_allgather":{} - }, - "avg_msg_size=any": - { - "algorithm=MPIR_Bcast_intra_scatter_ring_allgather":{} - } - } - }, - "comm_size=any": - { - "comm_hierarchy=parent": - { - "avg_msg_size<=0": - { - "algorithm=MPIR_Bcast_intra_smp":{} - }, - "avg_msg_size<=12288": - { - "algorithm=MPIR_Bcast_intra_binomial":{} - }, - "avg_msg_size=any": - { - "algorithm=MPIR_Bcast_intra_scatter_ring_allgather":{} - } - }, - "comm_hierarchy=any": - { - "avg_msg_size<=12288": - { - "algorithm=MPIR_Bcast_intra_binomial":{} - }, - "avg_msg_size=any": - { - "algorithm=MPIR_Bcast_intra_scatter_ring_allgather":{} - } - } - } - }, - "comm_type=inter": - { - "algorithm=MPIR_Bcast_inter_remote_send_local_bcast":{} - } - }, - "collective=allreduce": - { - "comm_type=intra": - { - "avg_msg_size<=0": - { - "is_commutative=yes": - { - "algorithm=MPIR_Allreduce_intra_smp":{} - }, - "is_commutative=no": - { - "algorithm=MPIR_Allreduce_intra_recursive_doubling":{} - } - }, - "avg_msg_size<=8": - { - "algorithm=MPIR_Allreduce_intra_recursive_doubling":{} - }, - "avg_msg_size=any": - { - "is_op_built_in=no": - { - "algorithm=MPIR_Allreduce_intra_recursive_doubling":{} - }, - "is_op_built_in=yes": - { - "countlocal_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allreduce_fallback(MPI_IN_PLACE, &acthigh, 1, MPIR_INT_INTERNAL, + MPI_SUM, intercomm_ptr->local_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* acthigh must either == 0 or the size of the local comm */ if (acthigh != 0 && acthigh != intercomm_ptr->local_size) { diff --git a/src/include/mpiimpl.h b/src/include/mpiimpl.h index 5133e6f5752..a9e91fd5491 100644 --- a/src/include/mpiimpl.h +++ b/src/include/mpiimpl.h @@ -182,7 +182,6 @@ typedef struct MPIR_Stream MPIR_Stream; #endif #include "coll_types.h" -#include "coll_impl.h" /*****************************************************************************/ /********************** PART 4: DEVICE PRE DECLARATION ***********************/ @@ -206,7 +205,6 @@ typedef struct MPIR_Stream MPIR_Stream; #include "mpir_progress_hook.h" #include "mpir_win.h" #include "mpir_coll.h" -#include "mpir_csel.h" #include "mpir_func.h" #include "mpir_nbc.h" #include "mpir_bsend.h" diff --git a/src/include/mpir_coll.h b/src/include/mpir_coll.h index d150998e47a..44ee599961b 100644 --- a/src/include/mpir_coll.h +++ b/src/include/mpir_coll.h @@ -6,9 +6,192 @@ #ifndef MPIR_COLL_H_INCLUDED #define MPIR_COLL_H_INCLUDED +typedef struct MPIR_Csel_coll_sig MPIR_Csel_coll_sig_s; +typedef struct MPII_Csel_container MPII_Csel_container_s; + #include "coll_impl.h" #include "coll_algos.h" +/* Define values for collective attribute. + * - The first 8 bits are passed down to basic collective algorithms. + * - The higher bits are used to assist algorithm selections + * - The lower 32 bits are reserved by MPIR-layer + * - The higher 32 bits are reserved for MPID-layer + */ +#define MPIR_COLL_ATTR_CORE_BITS 8 +#define MPIR_COLL_ATTR_MPIR_BITS 32 + +/* bit 0-7 */ +#define MPIR_COLL_ATTR_SYNC 0x1 /* It's an internal collective that focuses + * on synchronization rather than batch latency. + * In particular, advise netmod to avoid using + * injection send. */ +#define MPIR_ERR_PROC_FAILED 0x2 +#define MPIR_ERR_OTHER 0x4 +#define MPIR_COLL_ATTR_ERR_MASK 0x6 + +#define MPIR_COLL_ATTR_HAS_ERR(coll_attr) ((coll_attr) & MPIR_COLL_ATTR_ERR_MASK) + +/* bit 8-31, MPIR-layer */ +#define MPIR_COLL_ATTR__inplace 0x00000100 +#define MPIR_COLL_ATTR__pof2 0x00000200 +#define MPIR_COLL_ATTR__commutative 0x00000400 +#define MPIR_COLL_ATTR__builtin_op 0x00000800 + +struct MPIR_Csel_coll_sig { + MPIR_Csel_coll_type_e coll_type; + MPIR_Comm *comm_ptr; + void *sched; + enum MPIR_sched_type sched_type; + bool is_persistent; + + struct { + bool is_gpu; + } cache; + + union { + struct { + const void *sendbuf; + MPI_Aint sendcount; + MPI_Datatype sendtype; + void *recvbuf; + MPI_Aint recvcount; + MPI_Datatype recvtype; + } allgather, iallgather, neighbor_allgather, ineighbor_allgather; + struct { + const void *sendbuf; + MPI_Aint sendcount; + MPI_Datatype sendtype; + void *recvbuf; + const MPI_Aint *recvcounts; + const MPI_Aint *displs; + MPI_Datatype recvtype; + } allgatherv, iallgatherv, neighbor_allgatherv, ineighbor_allgatherv; + struct { + const void *sendbuf; + void *recvbuf; + MPI_Aint count; + MPI_Datatype datatype; + MPI_Op op; + } allreduce, iallreduce; + struct { + const void *sendbuf; + MPI_Aint sendcount; + MPI_Datatype sendtype; + MPI_Aint recvcount; + void *recvbuf; + MPI_Datatype recvtype; + } alltoall, ialltoall, neighbor_alltoall, ineighbor_alltoall; + struct { + const void *sendbuf; + const MPI_Aint *sendcounts; + const MPI_Aint *sdispls; + MPI_Datatype sendtype; + void *recvbuf; + const MPI_Aint *recvcounts; + const MPI_Aint *rdispls; + MPI_Datatype recvtype; + } alltoallv, ialltoallv, neighbor_alltoallv, ineighbor_alltoallv; + struct { + const void *sendbuf; + const MPI_Aint *sendcounts; + const MPI_Aint *sdispls; + const MPI_Datatype *sendtypes; + void *recvbuf; + const MPI_Aint *recvcounts; + const MPI_Aint *rdispls; + const MPI_Datatype *recvtypes; + } alltoallw, ialltoallw; + struct { + const void *sendbuf; + const MPI_Aint *sendcounts; + const MPI_Aint *sdispls; + const MPI_Datatype *sendtypes; + void *recvbuf; + const MPI_Aint *recvcounts; + const MPI_Aint *rdispls; + const MPI_Datatype *recvtypes; + } neighbor_alltoallw, ineighbor_alltoallw; + struct { + int dummy; /* some compiler (suncc) doesn't like empty struct */ + } barrier, ibarrier; + struct { + void *buffer; + MPI_Aint count; + MPI_Datatype datatype; + int root; + } bcast, ibcast; + struct { + const void *sendbuf; + void *recvbuf; + MPI_Aint count; + MPI_Datatype datatype; + MPI_Op op; + } exscan, iexscan; + struct { + const void *sendbuf; + MPI_Aint sendcount; + MPI_Datatype sendtype; + MPI_Aint recvcount; + void *recvbuf; + MPI_Datatype recvtype; + int root; + } gather, igather, scatter, iscatter; + struct { + const void *sendbuf; + MPI_Aint sendcount; + MPI_Datatype sendtype; + void *recvbuf; + const MPI_Aint *recvcounts; + const MPI_Aint *displs; + MPI_Datatype recvtype; + int root; + } gatherv, igatherv; + struct { + const void *sendbuf; + void *recvbuf; + MPI_Aint count; + MPI_Datatype datatype; + MPI_Op op; + int root; + } reduce, ireduce; + struct { + const void *sendbuf; + void *recvbuf; + const MPI_Aint *recvcounts; + MPI_Datatype datatype; + MPI_Op op; + } reduce_scatter, ireduce_scatter; + struct { + const void *sendbuf; + void *recvbuf; + MPI_Aint recvcount; + MPI_Datatype datatype; + MPI_Op op; + } reduce_scatter_block, ireduce_scatter_block; + struct { + const void *sendbuf; + void *recvbuf; + MPI_Aint count; + MPI_Datatype datatype; + MPI_Op op; + } scan, iscan; + struct { + const void *sendbuf; + const MPI_Aint *sendcounts; + const MPI_Aint *displs; + MPI_Datatype sendtype; + MPI_Aint recvcount; + void *recvbuf; + MPI_Datatype recvtype; + int root; + } scatterv, iscatterv; + } u; +}; + +typedef int (*MPIR_Coll_algo_fn) (MPIR_Csel_coll_sig_s * coll_sig, MPII_Csel_container_s * cnt); +void MPIR_Init_coll_sig(MPIR_Csel_coll_sig_s * coll_sig); + /* During init, not all algorithms are safe to use. For example, the csel * may not have been initialized. We define a set of fallback routines that * are safe to use during init. They are all intra algorithms. @@ -66,4 +249,7 @@ int MPIR_TSP_Ibarrier_sched_intra_tsp_auto(MPIR_Comm * comm, MPIR_TSP_sched_t sc int MPIR_TSP_Ireduce_sched_intra_tsp_auto(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, MPI_Op op, int root, MPIR_Comm * comm_ptr, MPIR_TSP_sched_t sched); +int MPIR_Ireduce_sched_intra_tsp_flat_auto(const void *sendbuf, void *recvbuf, MPI_Aint count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm_ptr, MPIR_TSP_sched_t sched); #endif /* MPIR_COLL_H_INCLUDED */ diff --git a/src/include/mpir_comm.h b/src/include/mpir_comm.h index 97ebe5109fb..84ed858fb44 100644 --- a/src/include/mpir_comm.h +++ b/src/include/mpir_comm.h @@ -224,8 +224,6 @@ struct MPIR_Comm { MPIR_Treealgo_tree_t *topo_wave_tree; } coll; - void *csel_comm; /* collective selector handle */ - #if defined HAVE_HCOLL hcoll_comm_priv_t hcoll_priv; #endif /* HAVE_HCOLL */ diff --git a/src/include/mpir_csel.h b/src/include/mpir_csel.h deleted file mode 100644 index 07f061e98ef..00000000000 --- a/src/include/mpir_csel.h +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#ifndef MPIR_CSEL_H_INCLUDED -#define MPIR_CSEL_H_INCLUDED - -#include "json.h" -#include "coll_impl.h" - -typedef enum { - MPIR_CSEL_COLL_TYPE__ALLGATHER = 0, - MPIR_CSEL_COLL_TYPE__ALLGATHERV, - MPIR_CSEL_COLL_TYPE__ALLREDUCE, - MPIR_CSEL_COLL_TYPE__ALLTOALL, - MPIR_CSEL_COLL_TYPE__ALLTOALLV, - MPIR_CSEL_COLL_TYPE__ALLTOALLW, - MPIR_CSEL_COLL_TYPE__BARRIER, - MPIR_CSEL_COLL_TYPE__BCAST, - MPIR_CSEL_COLL_TYPE__EXSCAN, - MPIR_CSEL_COLL_TYPE__GATHER, - MPIR_CSEL_COLL_TYPE__GATHERV, - MPIR_CSEL_COLL_TYPE__IALLGATHER, - MPIR_CSEL_COLL_TYPE__IALLGATHERV, - MPIR_CSEL_COLL_TYPE__IALLREDUCE, - MPIR_CSEL_COLL_TYPE__IALLTOALL, - MPIR_CSEL_COLL_TYPE__IALLTOALLV, - MPIR_CSEL_COLL_TYPE__IALLTOALLW, - MPIR_CSEL_COLL_TYPE__IBARRIER, - MPIR_CSEL_COLL_TYPE__IBCAST, - MPIR_CSEL_COLL_TYPE__IEXSCAN, - MPIR_CSEL_COLL_TYPE__IGATHER, - MPIR_CSEL_COLL_TYPE__IGATHERV, - MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLGATHER, - MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLGATHERV, - MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLTOALL, - MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLTOALLV, - MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLTOALLW, - MPIR_CSEL_COLL_TYPE__IREDUCE, - MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER, - MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER_BLOCK, - MPIR_CSEL_COLL_TYPE__ISCAN, - MPIR_CSEL_COLL_TYPE__ISCATTER, - MPIR_CSEL_COLL_TYPE__ISCATTERV, - MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLGATHER, - MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLGATHERV, - MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLTOALL, - MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLTOALLV, - MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLTOALLW, - MPIR_CSEL_COLL_TYPE__REDUCE, - MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER, - MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER_BLOCK, - MPIR_CSEL_COLL_TYPE__SCAN, - MPIR_CSEL_COLL_TYPE__SCATTER, - MPIR_CSEL_COLL_TYPE__SCATTERV, - MPIR_CSEL_COLL_TYPE__END, -} MPIR_Csel_coll_type_e; - -typedef struct { - MPIR_Csel_coll_type_e coll_type; - MPIR_Comm *comm_ptr; - - union { - struct { - const void *sendbuf; - MPI_Aint sendcount; - MPI_Datatype sendtype; - void *recvbuf; - MPI_Aint recvcount; - MPI_Datatype recvtype; - } allgather, iallgather, neighbor_allgather, ineighbor_allgather; - struct { - const void *sendbuf; - MPI_Aint sendcount; - MPI_Datatype sendtype; - void *recvbuf; - const MPI_Aint *recvcounts; - const MPI_Aint *displs; - MPI_Datatype recvtype; - } allgatherv, iallgatherv, neighbor_allgatherv, ineighbor_allgatherv; - struct { - const void *sendbuf; - void *recvbuf; - MPI_Aint count; - MPI_Datatype datatype; - MPI_Op op; - } allreduce, iallreduce; - struct { - const void *sendbuf; - MPI_Aint sendcount; - MPI_Datatype sendtype; - MPI_Aint recvcount; - void *recvbuf; - MPI_Datatype recvtype; - } alltoall, ialltoall, neighbor_alltoall, ineighbor_alltoall; - struct { - const void *sendbuf; - const MPI_Aint *sendcounts; - const MPI_Aint *sdispls; - MPI_Datatype sendtype; - void *recvbuf; - const MPI_Aint *recvcounts; - const MPI_Aint *rdispls; - MPI_Datatype recvtype; - } alltoallv, ialltoallv, neighbor_alltoallv, ineighbor_alltoallv; - struct { - const void *sendbuf; - const MPI_Aint *sendcounts; - const MPI_Aint *sdispls; - const MPI_Datatype *sendtypes; - void *recvbuf; - const MPI_Aint *recvcounts; - const MPI_Aint *rdispls; - const MPI_Datatype *recvtypes; - } alltoallw, ialltoallw; - struct { - const void *sendbuf; - const MPI_Aint *sendcounts; - const MPI_Aint *sdispls; - const MPI_Datatype *sendtypes; - void *recvbuf; - const MPI_Aint *recvcounts; - const MPI_Aint *rdispls; - const MPI_Datatype *recvtypes; - } neighbor_alltoallw, ineighbor_alltoallw; - struct { - int dummy; /* some compiler (suncc) doesn't like empty struct */ - } barrier, ibarrier; - struct { - void *buffer; - MPI_Aint count; - MPI_Datatype datatype; - int root; - } bcast, ibcast; - struct { - const void *sendbuf; - void *recvbuf; - MPI_Aint count; - MPI_Datatype datatype; - MPI_Op op; - } exscan, iexscan; - struct { - const void *sendbuf; - MPI_Aint sendcount; - MPI_Datatype sendtype; - MPI_Aint recvcount; - void *recvbuf; - MPI_Datatype recvtype; - int root; - } gather, igather, scatter, iscatter; - struct { - const void *sendbuf; - MPI_Aint sendcount; - MPI_Datatype sendtype; - void *recvbuf; - const MPI_Aint *recvcounts; - const MPI_Aint *displs; - MPI_Datatype recvtype; - int root; - } gatherv, igatherv; - struct { - const void *sendbuf; - void *recvbuf; - MPI_Aint count; - MPI_Datatype datatype; - MPI_Op op; - int root; - } reduce, ireduce; - struct { - const void *sendbuf; - void *recvbuf; - const MPI_Aint *recvcounts; - MPI_Datatype datatype; - MPI_Op op; - } reduce_scatter, ireduce_scatter; - struct { - const void *sendbuf; - void *recvbuf; - MPI_Aint recvcount; - MPI_Datatype datatype; - MPI_Op op; - } reduce_scatter_block, ireduce_scatter_block; - struct { - const void *sendbuf; - void *recvbuf; - MPI_Aint count; - MPI_Datatype datatype; - MPI_Op op; - } scan, iscan; - struct { - const void *sendbuf; - const MPI_Aint *sendcounts; - const MPI_Aint *displs; - MPI_Datatype sendtype; - MPI_Aint recvcount; - void *recvbuf; - MPI_Datatype recvtype; - int root; - } scatterv, iscatterv; - } u; -} MPIR_Csel_coll_sig_s; - -int MPIR_Csel_create_from_file(const char *json_file, - void *(*create_container) (struct json_object *), void **csel); -int MPIR_Csel_create_from_buf(const char *json, - void *(*create_container) (struct json_object *), void **csel); -int MPIR_Csel_free(void *csel); -int MPIR_Csel_prune(void *root_csel, MPIR_Comm * comm_ptr, void **comm_csel); -void *MPIR_Csel_search(void *csel, MPIR_Csel_coll_sig_s coll_sig); - -#endif /* MPIR_CSEL_H_INCLUDED */ diff --git a/src/include/mpir_misc.h b/src/include/mpir_misc.h index a4308eeecb2..9ce40983196 100644 --- a/src/include/mpir_misc.h +++ b/src/include/mpir_misc.h @@ -14,19 +14,6 @@ #define MPIR_FINALIZE_CALLBACK_DEFAULT_PRIO 0 #define MPIR_FINALIZE_CALLBACK_MAX_PRIO 10 -/* Define values for collective attribute. Collective attributes pass - * down contexts including error flags. - */ -#define MPIR_COLL_ATTR_SYNC 0x1 /* It's an internal collective that focuses - * on synchronization rather than batch latency. - * In particular, advise netmod to avoid using - * injection send. */ -#define MPIR_ERR_PROC_FAILED 0x2 -#define MPIR_ERR_OTHER 0x4 -#define MPIR_COLL_ATTR_ERR_MASK 0x6 - -#define MPIR_COLL_ATTR_HAS_ERR(coll_attr) ((coll_attr) & MPIR_COLL_ATTR_ERR_MASK) - /*E MPIR_Lang_t - Known language bindings for MPI diff --git a/src/include/mpir_request.h b/src/include/mpir_request.h index fee9e9efa5c..ad82142e91f 100644 --- a/src/include/mpir_request.h +++ b/src/include/mpir_request.h @@ -204,7 +204,7 @@ struct MPIR_Request { struct { /* Persistent requests have their own "real" requests */ struct MPIR_Request *real_request; - MPIR_TSP_sched_t sched; + void *sched; } persist; /* kind : MPIR_REQUEST_KIND__PREQUEST_SEND or MPIR_REQUEST_KIND__PREQUEST_RECV */ struct { struct MPIR_Request *real_request; diff --git a/src/mpi/coll/algorithms/treealgo/treeutil.c b/src/mpi/coll/algorithms/treealgo/treeutil.c index 53ae9c925dd..ee164415275 100644 --- a/src/mpi/coll/algorithms/treealgo/treeutil.c +++ b/src/mpi/coll/algorithms/treealgo/treeutil.c @@ -758,9 +758,9 @@ int MPII_Treeutil_tree_topology_aware_k_init(MPIR_Comm * comm, int k, int root, } else { /* rank level - build a tree on the ranks */ /* Do an allgather to know the current num_children on each rank */ - mpi_errno = MPIR_Allgather_impl(&(ct->num_children), 1, MPIR_INT_INTERNAL, - num_childrens, 1, MPIR_INT_INTERNAL, comm, - MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(&(ct->num_children), 1, MPIR_INT_INTERNAL, + num_childrens, 1, MPIR_INT_INTERNAL, comm, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); int switch_leader = tree_ut_int_elt(&level->ranks, level->root_idx); diff --git a/src/mpi/coll/allgather/Makefile.mk b/src/mpi/coll/allgather/Makefile.mk index 0085e0965df..2ef57a17846 100644 --- a/src/mpi/coll/allgather/Makefile.mk +++ b/src/mpi/coll/allgather/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/allgather/allgather_allcomm_nb.c \ src/mpi/coll/allgather/allgather_intra_smp.c \ src/mpi/coll/allgather/allgather_intra_recursive_doubling.c \ src/mpi/coll/allgather/allgather_intra_brucks.c \ diff --git a/src/mpi/coll/allgather/allgather_allcomm_nb.c b/src/mpi/coll/allgather/allgather_allcomm_nb.c deleted file mode 100644 index 4e4f09ff6c7..00000000000 --- a/src/mpi/coll/allgather/allgather_allcomm_nb.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Allgather_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr, - &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/allgather/allgather_intra_recexch.c b/src/mpi/coll/allgather/allgather_intra_recexch.c index d2ee4702ba4..7167a61d3d6 100644 --- a/src/mpi/coll/allgather/allgather_intra_recexch.c +++ b/src/mpi/coll/allgather/allgather_intra_recexch.c @@ -12,14 +12,15 @@ * on the recursive doubling algorithm described by Thakur et al, "Optimization of * Collective Communication Operations in MPICH", 2005. The recursive doubling * algorithm has been extended for any radix k. A variant of the algorithm called - * as distance halving (selected by setting recexch_type=1) is based on the + * as distance halving (selected by setting is_halving=true) is based on the * paper, Sack et al, "Faster topology-aware collective algorithms through * non-minimal communication", 2012. * */ -int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm, - int recexch_type, int k, int single_phase_recv, int coll_attr) +static int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, + MPI_Datatype recvtype, MPIR_Comm * comm, + bool is_halving, int k, int single_phase_recv, + int coll_attr) { int mpi_errno = MPI_SUCCESS; int is_inplace, i, j; @@ -106,7 +107,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, MPIR_ERR_CHECK(mpi_errno); } - if (step1_sendto != -1) { /* non-participating rank sends the data to a partcipating rank */ + if (step1_sendto != -1) { /* non-participating rank sends the data to a participating rank */ void *buf_to_send; send_offset = rank * recv_extent * recvcount; if (is_inplace) @@ -134,7 +135,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, /* For distance halving algorithm, exchange the data with digit reversed partner * so that finally the data is in the correct order. */ - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) { + if (is_halving) { if (step1_sendto == -1) { /* get the partner with whom I should exchange data */ partner = MPII_Recexchalgo_reverse_digits_step2(rank, nranks, k); @@ -163,7 +164,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, } } - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) { + if (is_halving) { phase = step2_nphases - 1; recv_phase = step2_nphases - 1; } else { @@ -179,7 +180,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, for (iter = 0; iter < total_phases && (j + iter) < step2_nphases; iter++) { for (i = 0; i < k - 1; i++) { nbr = step2_nbrs[recv_phase][i]; - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) + if (is_halving) rank_for_offset = MPII_Recexchalgo_reverse_digits_step2(nbr, nranks, k); else rank_for_offset = nbr; @@ -191,7 +192,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, MPIR_ALLGATHER_TAG, comm, &recv_reqs[num_rreq++]); MPIR_ERR_CHECK(mpi_errno); } - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) + if (is_halving) recv_phase--; else recv_phase++; @@ -200,7 +201,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, /* send data to all the neighbors */ for (i = 0; i < k - 1; i++) { nbr = step2_nbrs[phase][i]; - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) + if (is_halving) rank_for_offset = MPII_Recexchalgo_reverse_digits_step2(rank, nranks, k); else rank_for_offset = rank; @@ -214,7 +215,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, /* wait on prev recvs */ MPIC_Waitall((k - 1), recv_reqs, MPI_STATUSES_IGNORE); - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) + if (is_halving) phase--; else phase++; @@ -225,7 +226,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, /* send data to all the neighbors once more */ for (i = 0; i < k - 1; i++) { nbr = step2_nbrs[phase][i]; - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) + if (is_halving) rank_for_offset = MPII_Recexchalgo_reverse_digits_step2(rank, nranks, k); else rank_for_offset = rank; @@ -242,7 +243,7 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, mpi_errno = MPIC_Waitall((k - 1), recv_reqs + (k - 1), MPI_STATUSES_IGNORE); MPIR_ERR_CHECK(mpi_errno); - if (recexch_type == MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING) + if (is_halving) phase--; else phase++; @@ -286,3 +287,21 @@ int MPIR_Allgather_intra_recexch(const void *sendbuf, MPI_Aint sendcount, fn_fail: goto fn_exit; } + +int MPIR_Allgather_intra_recexch_doubling(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, + MPI_Datatype recvtype, MPIR_Comm * comm, + int k, int single_phase_recv, int coll_attr) +{ + return MPIR_Allgather_intra_recexch(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm, false, k, single_phase_recv, coll_attr); +} + +int MPIR_Allgather_intra_recexch_halving(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, + MPI_Datatype recvtype, MPIR_Comm * comm, + int k, int single_phase_recv, int coll_attr) +{ + return MPIR_Allgather_intra_recexch(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm, true, k, single_phase_recv, coll_attr); +} diff --git a/src/mpi/coll/allgather/allgather_intra_smp.c b/src/mpi/coll/allgather/allgather_intra_smp.c index 6edb566b9e5..7af8c1c0186 100644 --- a/src/mpi/coll/allgather/allgather_intra_smp.c +++ b/src/mpi/coll/allgather/allgather_intra_smp.c @@ -29,8 +29,9 @@ int MPIR_Allgather_intra_smp_no_order(const void *sendbuf, MPI_Aint sendcount, int external_rank = comm_ptr->external_rank; if (local_size == comm_size || external_size == comm_size) { - mpi_errno = MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, coll_attr); + mpi_errno = + MPIR_Allgather_fallback(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm_ptr, coll_attr); goto fn_exit; } @@ -80,20 +81,20 @@ int MPIR_Allgather_intra_smp_no_order(const void *sendbuf, MPI_Aint sendcount, MPIR_Datatype_get_extent_macro(recvtype, recvtype_extent); local_recvbuf = (char *) recvbuf + displs[external_rank] * recvtype_extent; } - mpi_errno = MPIR_Gather_impl(sendbuf, sendcount, sendtype, - local_recvbuf, recvcount, recvtype, 0, node_comm, coll_attr); + mpi_errno = MPIR_Gather_fallback(sendbuf, sendcount, sendtype, + local_recvbuf, recvcount, recvtype, 0, node_comm, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* -- allgatherv over node roots -- */ if (local_rank == 0) { - mpi_errno = MPIR_Allgatherv_impl(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, - recvbuf, counts, displs, recvtype, - node_roots_comm, coll_attr); + mpi_errno = MPIR_Allgatherv_fallback(MPI_IN_PLACE, 0, MPI_DATATYPE_NULL, + recvbuf, counts, displs, recvtype, + node_roots_comm, coll_attr); MPIR_ERR_CHECK(mpi_errno); } /* -- bcast over node -- */ - mpi_errno = MPIR_Bcast_impl(recvbuf, total_count, recvtype, 0, node_comm, coll_attr); + mpi_errno = MPIR_Bcast_fallback(recvbuf, total_count, recvtype, 0, node_comm, coll_attr); MPIR_ERR_CHECK(mpi_errno); fn_exit: diff --git a/src/mpi/coll/allgatherv/Makefile.mk b/src/mpi/coll/allgatherv/Makefile.mk index cf96245a97c..d69ad2daa20 100644 --- a/src/mpi/coll/allgatherv/Makefile.mk +++ b/src/mpi/coll/allgatherv/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/allgatherv/allgatherv_allcomm_nb.c \ src/mpi/coll/allgatherv/allgatherv_intra_recursive_doubling.c \ src/mpi/coll/allgatherv/allgatherv_intra_brucks.c \ src/mpi/coll/allgatherv/allgatherv_intra_ring.c \ diff --git a/src/mpi/coll/allgatherv/allgatherv_allcomm_nb.c b/src/mpi/coll/allgatherv/allgatherv_allcomm_nb.c deleted file mode 100644 index fd2aa393b73..00000000000 --- a/src/mpi/coll/allgatherv/allgatherv_allcomm_nb.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Allgatherv_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Iallgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, - comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/allgatherv/allgatherv_inter_remote_gather_local_bcast.c b/src/mpi/coll/allgatherv/allgatherv_inter_remote_gather_local_bcast.c index 16229a4df11..1a36720b38a 100644 --- a/src/mpi/coll/allgatherv/allgatherv_inter_remote_gather_local_bcast.c +++ b/src/mpi/coll/allgatherv/allgatherv_inter_remote_gather_local_bcast.c @@ -71,7 +71,7 @@ int MPIR_Allgatherv_inter_remote_gather_local_bcast(const void *sendbuf, MPI_Ain mpi_errno = MPIR_Type_commit_impl(&newtype); MPIR_ERR_CHECK(mpi_errno); - mpi_errno = MPIR_Bcast_allcomm_auto(recvbuf, 1, newtype, 0, newcomm_ptr, coll_attr); + mpi_errno = MPIR_Bcast_fallback(recvbuf, 1, newtype, 0, newcomm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); MPIR_Type_free_impl(&newtype); diff --git a/src/mpi/coll/allreduce/Makefile.mk b/src/mpi/coll/allreduce/Makefile.mk index 5db420bd059..73ca4b901eb 100644 --- a/src/mpi/coll/allreduce/Makefile.mk +++ b/src/mpi/coll/allreduce/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/allreduce/allreduce_allcomm_nb.c \ src/mpi/coll/allreduce/allreduce_intra_recursive_doubling.c \ src/mpi/coll/allreduce/allreduce_intra_recursive_multiplying.c \ src/mpi/coll/allreduce/allreduce_intra_reduce_scatter_allgather.c \ diff --git a/src/mpi/coll/allreduce/allreduce_allcomm_nb.c b/src/mpi/coll/allreduce/allreduce_allcomm_nb.c deleted file mode 100644 index 04e4c366c2c..00000000000 --- a/src/mpi/coll/allreduce/allreduce_allcomm_nb.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Allreduce_allcomm_nb(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Iallreduce(sendbuf, recvbuf, count, datatype, op, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/allreduce/allreduce_intra_ccl.c b/src/mpi/coll/allreduce/allreduce_intra_ccl.c index 57e3820762d..38c6732ea5e 100644 --- a/src/mpi/coll/allreduce/allreduce_intra_ccl.c +++ b/src/mpi/coll/allreduce/allreduce_intra_ccl.c @@ -46,5 +46,6 @@ int MPIR_Allreduce_intra_ccl(const void *sendbuf, void *recvbuf, MPI_Aint count, } fallback: - return MPIR_Allreduce_allcomm_auto(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); + /* FIXME: proper error */ + return MPI_ERR_OTHER; } diff --git a/src/mpi/coll/allreduce/allreduce_intra_tree.c b/src/mpi/coll/allreduce/allreduce_intra_tree.c index df779180e4a..9a25943995d 100644 --- a/src/mpi/coll/allreduce/allreduce_intra_tree.c +++ b/src/mpi/coll/allreduce/allreduce_intra_tree.c @@ -9,6 +9,7 @@ #include "mpiimpl.h" #include "algo_common.h" #include "treealgo.h" +#include "coll_csel.h" int MPIR_Allreduce_intra_tree(const void *sendbuf, void *recvbuf, @@ -69,31 +70,11 @@ int MPIR_Allreduce_intra_tree(const void *sendbuf, MPIR_Treealgo_tree_create_topo_aware(comm_ptr, tree_type, k, root, MPIR_CVAR_ALLREDUCE_TOPO_REORDER_ENABLE, &my_tree); } else if (tree_type == MPIR_TREE_TYPE_TOPOLOGY_WAVE) { - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLREDUCE, - .comm_ptr = comm_ptr, - .u.allreduce.sendbuf = sendbuf, - .u.allreduce.recvbuf = recvbuf, - .u.allreduce.count = count, - .u.allreduce.datatype = datatype, - .u.allreduce.op = op, - }; - int overhead = MPIR_CVAR_ALLREDUCE_TOPO_OVERHEAD; int lat_diff_groups = MPIR_CVAR_ALLREDUCE_TOPO_DIFF_GROUPS; int lat_diff_switches = MPIR_CVAR_ALLREDUCE_TOPO_DIFF_SWITCHES; int lat_same_switches = MPIR_CVAR_ALLREDUCE_TOPO_SAME_SWITCHES; - MPII_Csel_container_s *cnt = MPIR_Csel_search(comm_ptr->csel_comm, coll_sig); - MPIR_Assert(cnt); - - if (cnt->id == MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_tree) { - overhead = cnt->u.allreduce.intra_tree.topo_overhead; - lat_diff_groups = cnt->u.allreduce.intra_tree.topo_diff_groups; - lat_diff_switches = cnt->u.allreduce.intra_tree.topo_diff_switches; - lat_same_switches = cnt->u.allreduce.intra_tree.topo_same_switches; - } - mpi_errno = MPIR_Treealgo_tree_create_topo_wave(comm_ptr, k, root, MPIR_CVAR_ALLREDUCE_TOPO_REORDER_ENABLE, diff --git a/src/mpi/coll/alltoall/Makefile.mk b/src/mpi/coll/alltoall/Makefile.mk index 00e91bc51ec..6ee85b5645a 100644 --- a/src/mpi/coll/alltoall/Makefile.mk +++ b/src/mpi/coll/alltoall/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/alltoall/alltoall_allcomm_nb.c \ src/mpi/coll/alltoall/alltoall_intra_brucks.c \ src/mpi/coll/alltoall/alltoall_intra_k_brucks.c \ src/mpi/coll/alltoall/alltoall_intra_scattered.c \ diff --git a/src/mpi/coll/alltoall/alltoall_allcomm_nb.c b/src/mpi/coll/alltoall/alltoall_allcomm_nb.c deleted file mode 100644 index 22d4aaa9f5f..00000000000 --- a/src/mpi/coll/alltoall/alltoall_allcomm_nb.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Alltoall_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr, - &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/alltoallv/Makefile.mk b/src/mpi/coll/alltoallv/Makefile.mk index 932b9507751..8e63bb01407 100644 --- a/src/mpi/coll/alltoallv/Makefile.mk +++ b/src/mpi/coll/alltoallv/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/alltoallv/alltoallv_allcomm_nb.c \ src/mpi/coll/alltoallv/alltoallv_intra_pairwise_sendrecv_replace.c \ src/mpi/coll/alltoallv/alltoallv_intra_scattered.c \ src/mpi/coll/alltoallv/alltoallv_inter_pairwise_exchange.c diff --git a/src/mpi/coll/alltoallv/alltoallv_allcomm_nb.c b/src/mpi/coll/alltoallv/alltoallv_allcomm_nb.c deleted file mode 100644 index 434e9c116b1..00000000000 --- a/src/mpi/coll/alltoallv/alltoallv_allcomm_nb.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Alltoallv_allcomm_nb(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ialltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, - recvtype, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/alltoallw/Makefile.mk b/src/mpi/coll/alltoallw/Makefile.mk index 1bb20b233a3..baf3e9851aa 100644 --- a/src/mpi/coll/alltoallw/Makefile.mk +++ b/src/mpi/coll/alltoallw/Makefile.mk @@ -8,6 +8,5 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/alltoallw/alltoallw_allcomm_nb.c \ src/mpi/coll/alltoallw/alltoallw_intra_scattered.c \ src/mpi/coll/alltoallw/alltoallw_inter_pairwise_exchange.c diff --git a/src/mpi/coll/alltoallw/alltoallw_allcomm_nb.c b/src/mpi/coll/alltoallw/alltoallw_allcomm_nb.c deleted file mode 100644 index 669f8517c0a..00000000000 --- a/src/mpi/coll/alltoallw/alltoallw_allcomm_nb.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Alltoallw_allcomm_nb(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ialltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, - recvtypes, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/barrier/Makefile.mk b/src/mpi/coll/barrier/Makefile.mk index 082d1f0fac4..c1e7aecbec3 100644 --- a/src/mpi/coll/barrier/Makefile.mk +++ b/src/mpi/coll/barrier/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/barrier/barrier_allcomm_nb.c \ src/mpi/coll/barrier/barrier_intra_k_dissemination.c \ src/mpi/coll/barrier/barrier_intra_recexch.c \ src/mpi/coll/barrier/barrier_intra_smp.c \ diff --git a/src/mpi/coll/barrier/barrier_allcomm_nb.c b/src/mpi/coll/barrier/barrier_allcomm_nb.c deleted file mode 100644 index f1f38075135..00000000000 --- a/src/mpi/coll/barrier/barrier_allcomm_nb.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Barrier_allcomm_nb(MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Ibarrier(comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/bcast/Makefile.mk b/src/mpi/coll/bcast/Makefile.mk index 32fb8bbe2be..989a3ea2bee 100644 --- a/src/mpi/coll/bcast/Makefile.mk +++ b/src/mpi/coll/bcast/Makefile.mk @@ -9,7 +9,6 @@ mpi_core_sources += \ src/mpi/coll/bcast/bcast_utils.c \ - src/mpi/coll/bcast/bcast_allcomm_nb.c \ src/mpi/coll/bcast/bcast_intra_binomial.c \ src/mpi/coll/bcast/bcast_intra_scatter_recursive_doubling_allgather.c \ src/mpi/coll/bcast/bcast_intra_scatter_ring_allgather.c \ diff --git a/src/mpi/coll/bcast/bcast_allcomm_nb.c b/src/mpi/coll/bcast/bcast_allcomm_nb.c deleted file mode 100644 index 6879d59886d..00000000000 --- a/src/mpi/coll/bcast/bcast_allcomm_nb.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Bcast_allcomm_nb(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Ibcast(buffer, count, datatype, root, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/bcast/bcast_inter_remote_send_local_bcast.c b/src/mpi/coll/bcast/bcast_inter_remote_send_local_bcast.c index e927cbf2814..627290aa4f8 100644 --- a/src/mpi/coll/bcast/bcast_inter_remote_send_local_bcast.c +++ b/src/mpi/coll/bcast/bcast_inter_remote_send_local_bcast.c @@ -50,7 +50,7 @@ int MPIR_Bcast_inter_remote_send_local_bcast(void *buffer, /* now do the usual broadcast on this intracommunicator * with rank 0 as root. */ - mpi_errno = MPIR_Bcast_allcomm_auto(buffer, count, datatype, 0, newcomm_ptr, coll_attr); + mpi_errno = MPIR_Bcast(buffer, count, datatype, 0, newcomm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } diff --git a/src/mpi/coll/bcast/bcast_intra_tree.c b/src/mpi/coll/bcast/bcast_intra_tree.c index f53664b61cf..3f0d9636244 100644 --- a/src/mpi/coll/bcast/bcast_intra_tree.c +++ b/src/mpi/coll/bcast/bcast_intra_tree.c @@ -4,6 +4,7 @@ */ #include "mpiimpl.h" +#include "coll_csel.h" /* Algorithm: Tree-based bcast * For short messages, we use a kary/knomial tree-based algorithm. * Cost = lgp.alpha + n.lgp.beta @@ -83,27 +84,6 @@ int MPIR_Bcast_intra_tree(void *buffer, int lat_diff_switches = MPIR_CVAR_BCAST_TOPO_DIFF_SWITCHES; int lat_same_switches = MPIR_CVAR_BCAST_TOPO_SAME_SWITCHES; - if (comm_ptr->csel_comm) { - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__BCAST, - .comm_ptr = comm_ptr, - .u.bcast.buffer = buffer, - .u.bcast.count = count, - .u.bcast.datatype = datatype, - .u.bcast.root = root, - }; - - MPII_Csel_container_s *cnt = MPIR_Csel_search(comm_ptr->csel_comm, coll_sig); - MPIR_Assert(cnt); - - if (cnt->id == MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_tree) { - overhead = cnt->u.bcast.intra_tree.topo_overhead; - lat_diff_groups = cnt->u.bcast.intra_tree.topo_diff_groups; - lat_diff_switches = cnt->u.bcast.intra_tree.topo_diff_switches; - lat_same_switches = cnt->u.bcast.intra_tree.topo_same_switches; - } - } - mpi_errno = MPIR_Treealgo_tree_create_topo_wave(comm_ptr, branching_factor, root, MPIR_CVAR_BCAST_TOPO_REORDER_ENABLE, diff --git a/src/mpi/coll/coll_algorithms.txt b/src/mpi/coll/coll_algorithms.txt index 0279538dca4..421ef0c4044 100644 --- a/src/mpi/coll/coll_algorithms.txt +++ b/src/mpi/coll/coll_algorithms.txt @@ -16,6 +16,20 @@ # [algorithm_name] # [key]: [values] # +# "func-commkind" may be "general", under which the "algorithm_name" is the full name of the +# algorithm function. General algorithm functions takes (coll_sig, cnt) parameters. +# +# In addition, "conditions" list all conditions and their corresponding checkers. There are +# a few types of conditions distinguished by the format: +# - conidtion_name: func +# The condition calls checker function with signature: bool (*func)(coll_sig). +# - condition_name(thresh): func +# This condition calls a query function that returns a value: int (*func)(coll_sig) +# The thresh marks the upper limit (inclusive) of the condition. +# The conditions are used in specifying both algorithm restrictions and CSEL conditions. +# Example usage in restriction list or JSON file: inplace, !inplace, avg_msg_size(1024), etc. +# Most of the checker functions should be inlined to minimize function call overhead. +# # Notes: # * indentations (use 4 spaces) and ':' behind keys are significant # * auto and nb algorithms are assumed and not listed @@ -25,8 +39,8 @@ # # Recognized attribute keys (by gen_coll.py): # * restrictions: -# possible values include parent-comm, power-of-two, inplace, no-inplace, -# cummutative, builtin-op, node-consecutive, displs-ordered, size-ge-pof2 +# A comma-separated list restrictions. All restrictions must be specified in the top +# conditions list. # * extra_params and cvar_params: # Additional parameters specific to the algorithm functions. Most algorithm functions # use the same arguments (e.g. as MPIR_Bcast), but some may require additional @@ -37,9 +51,28 @@ # the same order. In addition, some of the extra parameter can be constant by specify # a initialization, e.g. param1=val. Rather than repeating the same constant in the # cvar_params, we can use `-` as placeholder for the corresponding constant param. -# * func_name: -# Some algorithm use another algorithm or use a different function name. +# ---- +conditions: + inplace: MPIR_Csel_sendbuf_inplace + pof2: MPIR_Csel_comm_size_is_pof2 + commutative: MPIR_Csel_op_is_commutative + builtin_op: MPIR_Csel_op_is_builtin + hierarchical: MPIR_Csel_is_hierarchical + node_consecutive: MPIR_Csel_is_node_consecutive + node_regular: MPIR_Csel_is_node_canonical + count_ge_pof2: MPIR_Csel_count_ge_pof2 + displs_ordered: MPIR_Csel_displs_ordered + block_regular: MPIR_Csel_block_regular + comm_size(thresh): MPIR_Csel_comm_size + avg_msg_size(thresh): MPIR_Csel_avg_msg_size + total_msg_size(thresh): MPIR_Csel_total_msg_size + +# ---- +general: + MPIR_Coll_nb + +# ---- barrier-intra: k_dissemination extra_params: k @@ -48,7 +81,7 @@ barrier-intra: extra_params: k, single_phase_recv cvar_params: RECEXCH_KVAL, RECEXCH_SINGLE_PHASE_RECV smp - restrictions: parent-comm + restrictions: hierarchical barrier-inter: bcast ibarrier-intra: @@ -67,7 +100,7 @@ bcast-intra: scatter_recursive_doubling_allgather scatter_ring_allgather smp - restrictions: parent-comm + restrictions: hierarchical tree extra_params: tree_type, k, is_non_blocking cvar_params: TREE_TYPE, TREE_KVAL, IS_NON_BLOCKING @@ -77,24 +110,25 @@ bcast-intra: ibcast-intra: sched_binomial sched_smp - restrictions: parent-comm + restrictions: hierarchical sched_scatter_recursive_doubling_allgather - restrictions: power-of-two + restrictions: pof2 sched_scatter_ring_allgather tsp_tree extra_params: tree_type, k, chunk_size cvar_params: TREE_TYPE, TREE_KVAL, TREE_PIPELINE_CHUNK_SIZE - tsp_scatterv_recexch_allgatherv - func_name: scatterv_allgatherv - extra_params: allgatherv_algo=MPIR_CVAR_IALLGATHERV_INTRA_ALGORITHM_tsp_recexch_doubling, scatterv_k, allgatherv_k + tsp_scatterv_allgatherv + extra_params: use_ring=0, scatterv_k, allgatherv_k cvar_params: -, SCATTERV_KVAL, ALLGATHERV_RECEXCH_KVAL + tsp_scatterv_recexch_allgatherv + extra_params: scatterv_k, allgatherv_k + cvar_params: SCATTERV_KVAL, ALLGATHERV_RECEXCH_KVAL tsp_scatterv_ring_allgatherv extra_params: scatterv_k cvar_params: SCATTERV_KVAL tsp_ring - func_name: tree - extra_params: tree_type=MPIR_TREE_TYPE_KARY, k=1, chunk_size - cvar_params: -, -, RING_CHUNK_SIZE + extra_params: chunk_size + cvar_params: RING_CHUNK_SIZE bcast-inter: remote_send_local_bcast ibcast-inter: @@ -168,60 +202,54 @@ allgather-intra: extra_params: k cvar_params: BRUCKS_KVAL recursive_doubling - restrictions: power-of-two + restrictions: pof2 ring recexch_doubling - func_name: recexch - extra_params: recexch_type=MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_DOUBLING, k, single_phase_recv - cvar_params: -, RECEXCH_KVAL, RECEXCH_SINGLE_PHASE_RECV + extra_params: k, single_phase_recv + cvar_params: RECEXCH_KVAL, RECEXCH_SINGLE_PHASE_RECV recexch_halving - func_name: recexch - extra_params: recexch_type=MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING, k, single_phase_recv - cvar_params: -, RECEXCH_KVAL, RECEXCH_SINGLE_PHASE_RECV + extra_params: k, single_phase_recv + cvar_params: RECEXCH_KVAL, RECEXCH_SINGLE_PHASE_RECV allgather-inter: local_gather_remote_bcast iallgather-intra: sched_ring sched_brucks sched_recursive_doubling - restrictions: power-of-two + restrictions: pof2 tsp_ring tsp_brucks extra_params: k cvar_params: BRUCKS_KVAL tsp_recexch_doubling - func_name: recexch - extra_params: recexch_type=MPIR_IALLGATHER_RECEXCH_TYPE_DISTANCE_DOUBLING, k - cvar_params: -, RECEXCH_KVAL + extra_params: k + cvar_params: RECEXCH_KVAL tsp_recexch_halving - func_name: recexch - extra_params: recexch_type=MPIR_IALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING, k - cvar_params: -, RECEXCH_KVAL + extra_params: k + cvar_params: RECEXCH_KVAL iallgather-inter: sched_local_gather_remote_bcast allgatherv-intra: brucks recursive_doubling - restrictions: power-of-two + restrictions: pof2 ring allgatherv-inter: remote_gather_local_bcast iallgatherv-intra: sched_brucks sched_recursive_doubling - restrictions: power-of-two + restrictions: pof2 sched_ring tsp_recexch_doubling - restrictions: displs-ordered - func_name: recexch - extra_params: recexch_type=MPIR_IALLGATHERV_RECEXCH_TYPE_DISTANCE_DOUBLING, k - cvar_params: -, RECEXCH_KVAL + restrictions: displs_ordered + extra_params: k + cvar_params: RECEXCH_KVAL tsp_recexch_halving - restrictions: displs-ordered - func_name: recexch - extra_params: recexch_type=MPIR_IALLGATHERV_RECEXCH_TYPE_DISTANCE_HALVING, k - cvar_params: -, RECEXCH_KVAL + restrictions: displs_ordered + extra_params: k + cvar_params: RECEXCH_KVAL tsp_ring tsp_brucks extra_params: k @@ -231,28 +259,28 @@ iallgatherv-inter: alltoall-intra: brucks - restrictions: noinplace + restrictions: !inplace k_brucks - restrictions: noinplace + restrictions: !inplace extra_params: k cvar_params: BRUCKS_KVAL pairwise - restrictions: noinplace + restrictions: !inplace pairwise_sendrecv_replace restrictions: inplace scattered - restrictions: noinplace + restrictions: !inplace alltoall-inter: pairwise_exchange ialltoall-intra: sched_brucks - restrictions: noinplace + restrictions: !inplace sched_inplace restrictions: inplace sched_pairwise - restrictions: noinplace + restrictions: !inplace sched_permuted_sendrecv - restrictions: noinplace + restrictions: !inplace tsp_ring tsp_brucks extra_params: k, buffer_per_phase @@ -267,20 +295,20 @@ alltoallv-intra: pairwise_sendrecv_replace restrictions: inplace scattered - restrictions: noinplace + restrictions: !inplace alltoallv-inter: pairwise_exchange ialltoallv-intra: sched_blocked - restrictions: noinplace + restrictions: !inplace sched_inplace restrictions: inplace tsp_scattered - restrictions: noinplace + restrictions: !inplace extra_params: batch_size, bblock cvar_params: SCATTERED_BATCH_SIZE, SCATTERED_OUTSTANDING_TASKS tsp_blocked - restrictions: noinplace + restrictions: !inplace extra_params: bblock cvar_params: THROTTLE tsp_inplace @@ -292,16 +320,16 @@ alltoallw-intra: pairwise_sendrecv_replace restrictions: inplace scattered - restrictions: noinplace + restrictions: !inplace alltoallw-inter: pairwise_exchange ialltoallw-intra: sched_blocked - restrictions: noinplace + restrictions: !inplace sched_inplace restrictions: inplace tsp_blocked - restrictions: noinplace + restrictions: !inplace extra_params: bblock cvar_params: THROTTLE tsp_inplace @@ -312,38 +340,37 @@ ialltoallw-inter: reduce-intra: binomial smp - restrictions: commutative, parent-comm + restrictions: commutative, hierarchical reduce_scatter_gather - restrictions: size-ge-pof2, builtin-op + restrictions: count_ge_pof2, builtin_op reduce-inter: local_reduce_remote_send ireduce-intra: sched_smp - restrictions: commutative, parent-comm + restrictions: commutative, hierarchical sched_binomial sched_reduce_scatter_gather - restrictions: size-ge-pof2, builtin-op + restrictions: count_ge_pof2, builtin_op tsp_tree restrictions: commutative extra_params: tree_type, k, chunk_size, buffer_per_child cvar_params: TREE_TYPE, TREE_KVAL, TREE_PIPELINE_CHUNK_SIZE, TREE_BUFFER_PER_CHILD tsp_ring - func_name: tree - extra_params: tree_type=MPIR_TREE_TYPE_KARY, k=1, chunk_size, buffer_per_child - cvar_params: -, -, RING_CHUNK_SIZE, TREE_BUFFER_PER_CHILD + extra_params: chunk_size, buffer_per_child + cvar_params: RING_CHUNK_SIZE, TREE_BUFFER_PER_CHILD ireduce-inter: sched_local_reduce_remote_send allreduce-intra: smp - restrictions: commutative, parent-comm + restrictions: commutative, hierarchical recursive_doubling recursive_multiplying extra_params: k cvar_params: RECURSIVE_MULTIPLYING_KVAL restrictions: commutative reduce_scatter_allgather - restrictions: size-ge-pof2, builtin-op + restrictions: count_ge_pof2, builtin_op tree extra_params: tree_type, k, chunk_size, buffer_per_child cvar_params: TREE_TYPE, TREE_KVAL, TREE_PIPELINE_CHUNK_SIZE, TREE_BUFFER_PER_CHILD @@ -364,17 +391,18 @@ allreduce-inter: iallreduce-intra: sched_naive sched_smp - restrictions: commutative, parent-comm + restrictions: commutative, hierarchical sched_recursive_doubling sched_reduce_scatter_allgather - restrictions: size-ge-pof2, builtin-op + restrictions: count_ge_pof2, builtin_op tsp_recexch_single_buffer - func_name: recexch - extra_params: recexch_type=MPIR_IALLREDUCE_RECEXCH_TYPE_SINGLE_BUFFER, k - cvar_params: -, RECEXCH_KVAL + extra_params: k + cvar_params: RECEXCH_KVAL tsp_recexch_multiple_buffer - func_name: recexch - extra_params: recexch_type=MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER, k + extra_params: k + cvar_params: RECEXCH_KVAL + tsp_recexch + extra_params: per_nbr_buffer=1, k cvar_params: -, RECEXCH_KVAL tsp_tree extra_params: tree_type, k, chunk_size, buffer_per_child @@ -399,7 +427,7 @@ reduce_scatter-inter: remote_reduce_local_scatter ireduce_scatter-intra: sched_noncommutative - restrictions: power-of-two + restrictions: pof2 sched_recursive_doubling sched_pairwise restrictions: commutative @@ -407,14 +435,14 @@ ireduce_scatter-intra: restrictions: commutative tsp_recexch restrictions: commutative - extra_params: recexch_type=IREDUCE_SCATTER_RECEXCH_TYPE_DISTANCE_DOUBLING, k + extra_params: is_dist_halving=0, k cvar_params: -, RECEXCH_KVAL ireduce_scatter-inter: sched_remote_reduce_local_scatterv reduce_scatter_block-intra: noncommutative - restrictions: power-of-two + restrictions: pof2 recursive_doubling pairwise restrictions: commutative @@ -424,7 +452,7 @@ reduce_scatter_block-inter: remote_reduce_local_scatter ireduce_scatter_block-intra: sched_noncommutative - restrictions: power-of-two + restrictions: pof2 sched_recursive_doubling sched_pairwise restrictions: commutative @@ -439,11 +467,11 @@ ireduce_scatter_block-inter: scan-intra: smp - restrictions: commutative, node-consecutive + restrictions: commutative, node_consecutive recursive_doubling iscan-intra: sched_smp - restrictions: commutative, node-consecutive + restrictions: commutative, node_consecutive sched_recursive_doubling tsp_recursive_doubling diff --git a/src/mpi/coll/coll_selection.json b/src/mpi/coll/coll_selection.json new file mode 100644 index 00000000000..d62952b9baf --- /dev/null +++ b/src/mpi/coll/coll_selection.json @@ -0,0 +1,824 @@ +{ + "name=main": "call=auto", + "name=auto": { + "collective=bcast-intra": "call=bcast-intra-auto", + "collective=bcast-inter": "call=bcast-inter-auto", + "collective=allreduce-intra": "call=allreduce-intra-auto", + "collective=allreduce-inter": "call=allreduce-inter-auto", + "collective=allgather-intra": "call=allgather-intra-auto", + "collective=allgather-inter": "call=allgather-inter-auto", + "collective=allgatherv-intra": "call=allgatherv-intra-auto", + "collective=allgatherv-inter": "call=allgatherv-inter-auto", + "collective=reduce-intra": "call=reduce-intra-auto", + "collective=reduce-inter": "call=reduce-inter-auto", + "collective=reduce_scatter-intra": "call=reduce_scatter-intra-auto", + "collective=reduce_scatter-inter": "call=reduce_scatter-inter-auto", + "collective=reduce_scatter_block-intra": "call=reduce_scatter_block-intra-auto", + "collective=reduce_scatter_block-inter": "call=reduce_scatter_block-inter-auto", + "collective=gather-intra": "call=gather-intra-auto", + "collective=gather-inter": "call=gather-inter-auto", + "collective=scatter-intra": "call=scatter-intra-auto", + "collective=scatter-inter": "call=scatter-inter-auto", + "collective=gatherv-intra": "call=gatherv-intra-auto", + "collective=gatherv-inter": "call=gatherv-inter-auto", + "collective=scatterv-intra": "call=scatterv-intra-auto", + "collective=scatterv-inter": "call=scatterv-inter-auto", + "collective=scan-intra": "call=scan-intra-auto", + "collective=exscan-intra": "call=exscan-intra-auto", + "collective=barrier-intra": "call=barrier-intra-auto", + "collective=barrier-inter": "call=barrier-inter-auto", + "collective=alltoall-intra": "call=alltoall-intra-auto", + "collective=alltoall-inter": "call=alltoall-inter-auto", + "collective=alltoallv-intra": "call=alltoallv-intra-auto", + "collective=alltoallv-inter": "call=alltoallv-inter-auto", + "collective=alltoallw-intra": "call=alltoallw-intra-auto", + "collective=alltoallw-inter": "call=alltoallw-inter-auto", + "collective=neighbor_allgather-intra": "call=neighbor_allgather-intra-auto", + "collective=neighbor_allgatherv-intra": "call=neighbor_allgatherv-intra-auto", + "collective=neighbor_alltoall-intra": "call=neighbor_alltoall-intra-auto", + "collective=neighbor_alltoallv-intra": "call=neighbor_alltoallv-intra-auto", + "collective=neighbor_alltoallw-intra": "call=neighbor_alltoallw-intra-auto", + "collective=ibarrier-intra": "call=ibarrier-intra-auto", + "collective=ibarrier-inter": "call=ibarrier-inter-auto", + "collective=iallreduce-intra": "call=iallreduce-intra-auto", + "collective=iallreduce-inter": "call=iallreduce-inter-auto", + "collective=iallgather-intra": "call=iallgather-intra-auto", + "collective=iallgather-inter": "call=iallgather-inter-auto", + "collective=iallgatherv-intra": "call=iallgatherv-intra-auto", + "collective=iallgatherv-inter": "call=iallgatherv-inter-auto", + "collective=ibcast-intra": "call=ibcast-intra-auto", + "collective=ibcast-inter": "call=ibcast-inter-auto", + "collective=ireduce-intra": "call=ireduce-intra-auto", + "collective=ireduce-inter": "call=ireduce-inter-auto", + "collective=ireduce_scatter-intra": "call=ireduce_scatter-intra-auto", + "collective=ireduce_scatter-inter": "call=ireduce_scatter-inter-auto", + "collective=igather-intra": "call=igather-intra-auto", + "collective=igather-inter": "call=igather-inter-auto", + "collective=iscatter-intra": "call=iscatter-intra-auto", + "collective=iscatter-inter": "call=iscatter-inter-auto", + "collective=igatherv-intra": "call=igatherv-intra-auto", + "collective=igatherv-inter": "call=igatherv-inter-auto", + "collective=iscatterv-intra": "call=iscatterv-intra-auto", + "collective=iscatterv-inter": "call=iscatterv-inter-auto", + "collective=iexscan-intra": "call=iexscan-intra-auto", + "collective=ialltoall-intra": "call=ialltoall-intra-auto", + "collective=ialltoall-inter": "call=ialltoall-inter-auto", + "collective=ialltoallv-intra": "call=ialltoallv-intra-auto", + "collective=ialltoallv-inter": "call=ialltoallv-inter-auto", + "collective=ialltoallw-intra": "call=ialltoallw-intra-auto", + "collective=ialltoallw-inter": "call=ialltoallw-inter-auto", + "collective=ireduce_scatter_block-intra": "call=ireduce_scatter_block-intra-auto", + "collective=ireduce_scatter_block-inter": "call=ireduce_scatter_block-inter-auto", + "collective=iscan-intra": "call=iscan-intra-auto", + "collective=ineighbor_allgather-intra": "call=ineighbor_allgather-intra-auto", + "collective=ineighbor_allgatherv-intra": "call=ineighbor_allgatherv-intra-auto", + "collective=ineighbor_alltoall-intra": "call=ineighbor_alltoall-intra-auto", + "collective=ineighbor_alltoallv-intra": "call=ineighbor_alltoallv-intra-auto", + "collective=ineighbor_alltoallw-intra": "call=ineighbor_alltoallw-intra-auto" + }, + "name=bcast-intra-auto": + { + "comm_size(8)": + { + "algorithm=MPIR_Bcast_intra_binomial":{} + }, + "pof2": + { + "avg_msg_size(12288)": + { + "algorithm=MPIR_Bcast_intra_binomial":{} + }, + "avg_msg_size(524288)": + { + "algorithm=MPIR_Bcast_intra_scatter_recursive_doubling_allgather":{} + }, + "any": + { + "algorithm=MPIR_Bcast_intra_scatter_ring_allgather":{} + } + }, + "comm_size(any)": + { + "avg_msg_size(12288)": + { + "algorithm=MPIR_Bcast_intra_binomial":{} + }, + "any": + { + "algorithm=MPIR_Bcast_intra_scatter_ring_allgather":{} + } + } + }, + "name=bcast-inter-auto": + { + "algorithm=MPIR_Bcast_inter_remote_send_local_bcast":{} + }, + "name=allreduce-intra-auto": + { + "avg_msg_size(8)": + { + "algorithm=MPIR_Allreduce_intra_recursive_doubling":{} + }, + "any": + { + "!builtin_op": + { + "algorithm=MPIR_Allreduce_intra_recursive_doubling":{} + }, + "builtin_op": + { + "!count_ge_pof2": + { + "algorithm=MPIR_Allreduce_intra_recursive_doubling":{} + }, + "any": + { + "algorithm=MPIR_Allreduce_intra_reduce_scatter_allgather":{} + } + } + } + }, + "name=allreduce-inter-auto": + { + "algorithm=MPIR_Allreduce_inter_reduce_exchange_bcast":{} + }, + "name=allgather-intra-auto": + { + "pof2": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Allgather_intra_recursive_doubling":{} + }, + "any": + { + "algorithm=MPIR_Allgather_intra_ring":{} + } + }, + "any": + { + "total_msg_size(81920)": + { + "algorithm=MPIR_Allgather_intra_brucks":{} + }, + "any": + { + "algorithm=MPIR_Allgather_intra_ring":{} + } + } + }, + "name=allgather-inter-auto": + { + "algorithm=MPIR_Allgather_inter_local_gather_remote_bcast":{} + }, + "name=allgatherv-intra-auto": + { + "pof2": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Allgatherv_intra_recursive_doubling":{} + }, + "any": + { + "algorithm=MPIR_Allgatherv_intra_ring":{} + } + }, + "any": + { + "total_msg_size(81920)": + { + "algorithm=MPIR_Allgatherv_intra_brucks":{} + }, + "any": + { + "algorithm=MPIR_Allgatherv_intra_ring":{} + } + } + }, + "name=allgatherv-inter-auto": + { + "algorithm=MPIR_Allgatherv_inter_remote_gather_local_bcast":{} + }, + "name=reduce-intra-auto": + { + "avg_msg_size(2048)": + { + "algorithm=MPIR_Reduce_intra_binomial":{} + }, + "any": + { + "builtin_op": + { + "!count_ge_pof2": + { + "algorithm=MPIR_Reduce_intra_binomial":{} + }, + "any": + { + "algorithm=MPIR_Reduce_intra_reduce_scatter_gather":{} + } + }, + "any": + { + "algorithm=MPIR_Reduce_intra_binomial":{} + } + } + }, + "name=reduce-inter-auto": + { + "algorithm=MPIR_Reduce_inter_local_reduce_remote_send":{} + }, + "name=reduce_scatter-intra-auto": + { + "commutative": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Reduce_scatter_intra_recursive_halving":{} + }, + "any": + { + "algorithm=MPIR_Reduce_scatter_intra_pairwise":{} + } + }, + "any": + { + "block_regular": + { + "pof2": + { + "algorithm=MPIR_Reduce_scatter_intra_noncommutative":{} + }, + "any": + { + "algorithm=MPIR_Reduce_scatter_intra_recursive_doubling":{} + } + }, + "any": + { + "algorithm=MPIR_Reduce_scatter_intra_recursive_doubling":{} + } + } + }, + "name=reduce_scatter-inter-auto": + { + "algorithm=MPIR_Reduce_scatter_inter_remote_reduce_local_scatter":{} + }, + "name=reduce_scatter_block-intra-auto": + { + "commutative": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Reduce_scatter_block_intra_recursive_halving":{} + }, + "any": + { + "algorithm=MPIR_Reduce_scatter_block_intra_pairwise":{} + } + }, + "any": + { + "pof2": + { + "algorithm=MPIR_Reduce_scatter_block_intra_noncommutative":{} + }, + "any": + { + "algorithm=MPIR_Reduce_scatter_block_intra_recursive_doubling":{} + } + } + }, + "name=reduce_scatter_block-inter-auto": + { + "algorithm=MPIR_Reduce_scatter_block_inter_remote_reduce_local_scatter":{} + }, + "name=gather-intra-auto": + { + "algorithm=MPIR_Gather_intra_binomial":{} + }, + "name=gather-inter-auto": + { + "total_msg_size(2048)": + { + "algorithm=MPIR_Gather_inter_local_gather_remote_send":{} + }, + "any": + { + "algorithm=MPIR_Gather_inter_linear":{} + } + }, + "name=scatter-intra-auto": + { + "algorithm=MPIR_Scatter_intra_binomial":{} + }, + "name=scatter-inter-auto": + { + "total_msg_size(2048)": + { + "algorithm=MPIR_Scatter_inter_remote_send_local_scatter":{} + }, + "any": + { + "algorithm=MPIR_Scatter_inter_linear":{} + } + }, + "name=gatherv-intra-auto": + { + "algorithm=MPIR_Gatherv_allcomm_linear":{} + }, + "name=gatherv-inter-auto": + { + "algorithm=MPIR_Gatherv_allcomm_linear":{} + }, + "name=scatterv-intra-auto": + { + "algorithm=MPIR_Scatterv_allcomm_linear":{} + }, + "name=scatterv-inter-auto": + { + "algorithm=MPIR_Scatterv_allcomm_linear":{} + }, + "name=scan-intra-auto": + { + "algorithm=MPIR_Scan_intra_recursive_doubling":{} + }, + "name=exscan-intra-auto": + { + "algorithm=MPIR_Exscan_intra_recursive_doubling":{} + }, + "name=barrier-intra-auto": + { + "algorithm=MPIR_Barrier_intra_k_dissemination":{ + "k=2": {} + } + }, + "name=barrier-inter-auto": + { + "algorithm=MPIR_Barrier_inter_bcast":{} + }, + "name=alltoall-intra-auto": + { + "inplace": + { + "algorithm=MPIR_Alltoall_intra_pairwise_sendrecv_replace":{} + }, + "any": + { + "comm_size(8)": + { + "avg_msg_size(32768)": + { + "algorithm=MPIR_Alltoall_intra_scattered":{} + }, + "any": + { + "algorithm=MPIR_Alltoall_intra_pairwise":{} + } + }, + "any": + { + "avg_msg_size(256)": + { + "algorithm=MPIR_Alltoall_intra_brucks":{} + }, + "avg_msg_size(32768)": + { + "algorithm=MPIR_Alltoall_intra_scattered":{} + }, + "any": + { + "algorithm=MPIR_Alltoall_intra_pairwise":{} + } + } + } + }, + "name=alltoall-inter-auto": + { + "algorithm=MPIR_Alltoall_inter_pairwise_exchange":{} + }, + "name=alltoallv-intra-auto": + { + "inplace": + { + "algorithm=MPIR_Alltoallv_intra_pairwise_sendrecv_replace":{} + }, + "any": + { + "algorithm=MPIR_Alltoallv_intra_scattered":{} + } + }, + "name=alltoallv-inter-auto": + { + "algorithm=MPIR_Alltoallv_inter_pairwise_exchange":{} + }, + "name=alltoallw-intra-auto": + { + "inplace": + { + "algorithm=MPIR_Alltoallw_intra_pairwise_sendrecv_replace":{} + }, + "any": + { + "algorithm=MPIR_Alltoallw_intra_scattered":{} + } + }, + "name=alltoallw-inter-auto": + { + "algorithm=MPIR_Alltoallw_inter_pairwise_exchange":{} + }, + "name=neighbor_allgather-intra-auto": + { + "algorithm=MPIR_Coll_nb":{} + }, + "name=neighbor_allgatherv-intra-auto": + { + "algorithm=MPIR_Coll_nb":{} + }, + "name=neighbor_alltoall-intra-auto": + { + "algorithm=MPIR_Coll_nb":{} + }, + "name=neighbor_alltoallv-intra-auto": + { + "algorithm=MPIR_Coll_nb":{} + }, + "name=neighbor_alltoallw-intra-auto": + { + "algorithm=MPIR_Coll_nb":{} + }, + "name=ibarrier-intra-auto": + { + "algorithm=MPIR_Ibarrier_intra_sched_recursive_doubling":{} + }, + "name=ibarrier-inter-auto": + { + "algorithm=MPIR_Ibarrier_inter_sched_bcast":{} + }, + "name=iallreduce-intra-auto": + { + "avg_msg_size(2048)": + { + "algorithm=MPIR_Iallreduce_intra_sched_recursive_doubling":{} + }, + "any": + { + "!builtin_op": + { + "algorithm=MPIR_Iallreduce_intra_sched_recursive_doubling":{} + }, + "builtin_op": + { + "!count_ge_pof2": + { + "algorithm=MPIR_Iallreduce_intra_sched_recursive_doubling":{} + }, + "any": + { + "algorithm=MPIR_Iallreduce_intra_sched_reduce_scatter_allgather":{} + } + } + } + }, + "name=iallreduce-inter-auto": + { + "algorithm=MPIR_Iallreduce_inter_sched_remote_reduce_local_bcast":{} + }, + "name=iallgather-intra-auto": + { + "pof2": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Iallgather_intra_sched_recursive_doubling":{} + }, + "any": + { + "algorithm=MPIR_Iallgather_intra_sched_ring":{} + } + }, + "any": + { + "total_msg_size(81920)": + { + "algorithm=MPIR_Iallgather_intra_sched_brucks":{} + }, + "any": + { + "algorithm=MPIR_Iallgather_intra_sched_ring":{} + } + } + }, + "name=iallgather-inter-auto": + { + "algorithm=MPIR_Iallgather_inter_sched_local_gather_remote_bcast":{} + }, + "name=iallgatherv-intra-auto": + { + "pof2": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Iallgatherv_intra_sched_recursive_doubling":{} + }, + "any": + { + "algorithm=MPIR_Iallgatherv_intra_sched_ring":{} + } + }, + "any": + { + "total_msg_size(81920)": + { + "algorithm=MPIR_Iallgatherv_intra_sched_brucks":{} + }, + "any": + { + "algorithm=MPIR_Iallgatherv_intra_sched_ring":{} + } + } + }, + "name=iallgatherv-inter-auto": + { + "algorithm=MPIR_Iallgatherv_inter_sched_remote_gather_local_bcast":{} + }, + "name=ibcast-intra-auto": + { + "comm_size(8)": + { + "any": + { + "algorithm=MPIR_Ibcast_intra_sched_binomial":{} + } + }, + "pof2": + { + "avg_msg_size(12288)": + { + "algorithm=MPIR_Ibcast_intra_sched_binomial":{} + }, + "avg_msg_size(524288)": + { + "algorithm=MPIR_Ibcast_intra_sched_scatter_recursive_doubling_allgather":{} + }, + "any": + { + "algorithm=MPIR_Ibcast_intra_sched_scatter_ring_allgather":{} + } + }, + "any": + { + "avg_msg_size(12288)": + { + "algorithm=MPIR_Ibcast_intra_sched_binomial":{} + }, + "any": + { + "algorithm=MPIR_Ibcast_intra_sched_scatter_ring_allgather":{} + } + } + }, + "name=ibcast-inter-auto": + { + "algorithm=MPIR_Ibcast_inter_sched_flat":{} + }, + "name=ireduce-intra-auto": + { + "avg_msg_size(2048)": + { + "algorithm=MPIR_Ireduce_intra_sched_binomial":{} + }, + "any": + { + "builtin_op": + { + "!count_ge_pof2": + { + "algorithm=MPIR_Ireduce_intra_sched_binomial":{} + }, + "any": + { + "algorithm=MPIR_Ireduce_intra_sched_reduce_scatter_gather":{} + } + }, + "!builtin_op": + { + "algorithm=MPIR_Ireduce_intra_sched_binomial":{} + } + } + }, + "name=ireduce-inter-auto": + { + "algorithm=MPIR_Ireduce_inter_sched_local_reduce_remote_send":{} + }, + "name=ireduce_scatter-intra-auto": + { + "commutative": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Ireduce_scatter_intra_sched_recursive_halving":{} + }, + "any": + { + "algorithm=MPIR_Ireduce_scatter_intra_sched_pairwise":{} + } + }, + "any": + { + "block_regular": + { + "pof2": + { + "algorithm=MPIR_Ireduce_scatter_intra_sched_noncommutative":{} + }, + "any": + { + "algorithm=MPIR_Ireduce_scatter_intra_sched_recursive_doubling":{} + } + }, + "!block_regular": + { + "algorithm=MPIR_Ireduce_scatter_intra_sched_recursive_doubling":{} + } + } + }, + "name=ireduce_scatter-inter-auto": + { + "algorithm=MPIR_Ireduce_scatter_inter_sched_remote_reduce_local_scatterv":{} + }, + "name=igather-intra-auto": + { + "algorithm=MPIR_Igather_intra_sched_binomial":{} + }, + "name=igather-inter-auto": + { + "total_msg_size(2048)": + { + "algorithm=MPIR_Igather_inter_sched_short":{} + }, + "any": + { + "algorithm=MPIR_Igather_inter_sched_long":{} + } + }, + "name=iscatter-intra-auto": + { + "algorithm=MPIR_Iscatter_intra_sched_binomial":{} + }, + "name=iscatter-inter-auto": + { + "total_msg_size(2048)": + { + "algorithm=MPIR_Iscatter_inter_sched_remote_send_local_scatter":{} + }, + "any": + { + "algorithm=MPIR_Iscatter_inter_sched_linear":{} + } + }, + "name=igatherv-intra-auto": + { + "algorithm=MPIR_Igatherv_allcomm_sched_linear":{} + }, + "name=igatherv-inter-auto": + { + "algorithm=MPIR_Igatherv_allcomm_sched_linear":{} + }, + "name=iscatterv-intra-auto": + { + "algorithm=MPIR_Iscatterv_allcomm_sched_linear":{} + }, + "name=iscatterv-inter-auto": + { + "algorithm=MPIR_Iscatterv_allcomm_sched_linear":{} + }, + "name=iexscan-intra-auto": + { + "algorithm=MPIR_Iexscan_intra_sched_recursive_doubling":{} + }, + "name=ialltoall-intra-auto": + { + "inplace": + { + "algorithm=MPIR_Ialltoall_intra_sched_inplace":{} + }, + "any": + { + "comm_size(8)": + { + "avg_msg_size(32768)": + { + "algorithm=MPIR_Ialltoall_intra_sched_permuted_sendrecv":{} + }, + "any": + { + "algorithm=MPIR_Ialltoall_intra_sched_pairwise":{} + } + }, + "any": + { + "avg_msg_size(2048)": + { + "algorithm=MPIR_Ialltoall_intra_sched_brucks":{} + }, + "avg_msg_size(32768)": + { + "algorithm=MPIR_Ialltoall_intra_sched_permuted_sendrecv":{} + }, + "any": + { + "algorithm=MPIR_Ialltoall_intra_sched_pairwise":{} + } + } + } + }, + "name=ialltoall-inter-auto": + { + "algorithm=MPIR_Ialltoall_inter_sched_pairwise_exchange":{} + }, + "name=ialltoallv-intra-auto": + { + "inplace": + { + "algorithm=MPIR_Ialltoallv_intra_sched_inplace":{} + }, + "any": + { + "algorithm=MPIR_Ialltoallv_intra_sched_blocked":{} + } + }, + "name=ialltoallv-inter-auto": + { + "algorithm=MPIR_Ialltoallv_inter_sched_pairwise_exchange":{} + }, + "name=ialltoallw-intra-auto": + { + "inplace": + { + "algorithm=MPIR_Ialltoallw_intra_sched_inplace":{} + }, + "any": + { + "algorithm=MPIR_Ialltoallw_intra_sched_blocked":{} + } + }, + "name=ialltoallw-inter-auto": + { + "algorithm=MPIR_Ialltoallw_inter_sched_pairwise_exchange":{} + }, + "name=ireduce_scatter_block-intra-auto": + { + "commutative": + { + "total_msg_size(524288)": + { + "algorithm=MPIR_Ireduce_scatter_block_intra_sched_recursive_halving":{} + }, + "any": + { + "algorithm=MPIR_Ireduce_scatter_block_intra_sched_pairwise":{} + } + }, + "any": + { + "pof2": + { + "algorithm=MPIR_Ireduce_scatter_block_intra_sched_noncommutative":{} + }, + "any": + { + "algorithm=MPIR_Ireduce_scatter_block_intra_sched_recursive_doubling":{} + } + } + }, + "name=ireduce_scatter_block-inter-auto": + { + "algorithm=MPIR_Ireduce_scatter_block_inter_sched_remote_reduce_local_scatterv":{} + }, + "name=iscan-intra-auto": + { + "algorithm=MPIR_Iscan_intra_sched_recursive_doubling":{} + }, + "name=ineighbor_allgather-intra-auto": + { + "algorithm=MPIR_Ineighbor_allgather_allcomm_sched_linear":{} + }, + "name=ineighbor_allgatherv-intra-auto": + { + "algorithm=MPIR_Ineighbor_allgatherv_allcomm_sched_linear":{} + }, + "name=ineighbor_alltoall-intra-auto": + { + "algorithm=MPIR_Ineighbor_alltoall_allcomm_sched_linear":{} + }, + "name=ineighbor_alltoallv-intra-auto": + { + "algorithm=MPIR_Ineighbor_alltoallv_allcomm_sched_linear":{} + }, + "name=ineighbor_alltoallw-intra-auto": + { + "algorithm=MPIR_Ineighbor_alltoallw_allcomm_sched_linear":{} + } +} diff --git a/src/mpi/coll/cvars.txt b/src/mpi/coll/cvars.txt index 40c150d5e88..f7c0db2d6a1 100644 --- a/src/mpi/coll/cvars.txt +++ b/src/mpi/coll/cvars.txt @@ -16,7 +16,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select barrier algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm smp - Force smp algorithm k_dissemination - Force high radix dissemination algorithm @@ -31,7 +31,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select barrier algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) bcast - Force bcast algorithm nb - Force nonblocking algorithm @@ -95,7 +95,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ibarrier algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_recursive_doubling - Force recursive doubling algorithm tsp_recexch - Force generic transport based recursive exchange algorithm @@ -110,7 +110,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ibarrier algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_bcast - Force bcast algorithm @@ -183,7 +183,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select bcast algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) binomial - Force Binomial Tree nb - Force nonblocking algorithm smp - Force smp algorithm @@ -308,7 +308,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select bcast algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm remote_send_local_bcast - Force remote-send-local-bcast algorithm @@ -366,13 +366,14 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ibcast algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_binomial - Force Binomial algorithm sched_smp - Force smp algorithm sched_scatter_recursive_doubling_allgather - Force Scatter Recursive Doubling Allgather algorithm sched_scatter_ring_allgather - Force Scatter Ring Allgather algorithm tsp_tree - Force Generic Transport Tree algorithm + tsp_scatterv_allgatherv - Force Generic Transport Scatterv followed by Allgatherv algorithm tsp_scatterv_recexch_allgatherv - Force Generic Transport Scatterv followed by Recursive Exchange Allgatherv algorithm tsp_scatterv_ring_allgatherv - Force Generic Transport Scatterv followed by Ring Allgatherv algorithm tsp_ring - Force Generic Transport Ring algorithm @@ -406,7 +407,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ibcast algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_flat - Force flat algorithm @@ -444,7 +445,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select gather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) binomial - Force binomial algorithm nb - Force nonblocking algorithm @@ -457,7 +458,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select gather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) linear - Force linear algorithm local_gather_remote_send - Force local-gather-remote-send algorithm nb - Force nonblocking algorithm @@ -471,7 +472,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select igather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_binomial - Force binomial algorithm tsp_tree - Force genetric transport based tree algorithm @@ -495,7 +496,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select igather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_long - Force long inter algorithm sched_short - Force short inter algorithm @@ -509,7 +510,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select gatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) linear - Force linear algorithm nb - Force nonblocking algorithm @@ -522,7 +523,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select gatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) linear - Force linear algorithm nb - Force nonblocking algorithm @@ -535,7 +536,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select igatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -549,7 +550,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select igatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -574,7 +575,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select scatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) binomial - Force binomial algorithm nb - Force nonblocking algorithm @@ -587,7 +588,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select scatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) linear - Force linear algorithm nb - Force nonblocking algorithm remote_send_local_scatter - Force remote-send-local-scatter algorithm @@ -601,7 +602,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iscatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_binomial - Force binomial algorithm tsp_tree - Force genetric transport based tree algorithm @@ -625,7 +626,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iscatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm sched_remote_send_local_scatter - Force remote-send-local-scatter algorithm @@ -639,7 +640,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select scatterv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) linear - Force linear algorithm nb - Force nonblocking algorithm @@ -652,7 +653,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select scatterv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) linear - Force linear algorithm nb - Force nonblocking algorithm @@ -665,7 +666,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iscatterv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -679,7 +680,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iscatterv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -717,7 +718,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) brucks - Force brucks algorithm k_brucks - Force brucks algorithm nb - Force nonblocking algorithm @@ -766,7 +767,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) local_gather_remote_bcast - Force local-gather-remote-bcast algorithm nb - Force nonblocking algorithm @@ -799,7 +800,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_ring - Force ring algorithm sched_brucks - Force brucks algorithm @@ -818,7 +819,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_local_gather_remote_bcast - Force local-gather-remote-bcast algorithm @@ -842,7 +843,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) brucks - Force brucks algorithm nb - Force nonblocking algorithm recursive_doubling - Force recursive doubling algorithm @@ -857,7 +858,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm remote_gather_local_bcast - Force remote-gather-local-bcast algorithm @@ -890,7 +891,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_brucks - Force brucks algorithm sched_recursive_doubling - Force recursive doubling algorithm @@ -909,7 +910,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_remote_gather_local_bcast - Force remote-gather-local-bcast algorithm @@ -959,7 +960,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select alltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) brucks - Force brucks algorithm k_brucks - Force Force radix k brucks algorithm nb - Force nonblocking algorithm @@ -986,7 +987,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select alltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm pairwise_exchange - Force pairwise exchange algorithm @@ -999,7 +1000,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ialltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_brucks - Force brucks algorithm sched_inplace - Force inplace algorithm @@ -1018,7 +1019,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ialltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_pairwise_exchange - Force pairwise exchange algorithm @@ -1031,7 +1032,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select alltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm pairwise_sendrecv_replace - Force pairwise_sendrecv_replace algorithm scattered - Force scattered algorithm @@ -1045,7 +1046,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select alltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) pairwise_exchange - Force pairwise exchange algorithm nb - Force nonblocking algorithm @@ -1058,7 +1059,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ialltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_blocked - Force blocked algorithm sched_inplace - Force inplace algorithm @@ -1075,7 +1076,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ialltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_pairwise_exchange - Force pairwise exchange algorithm @@ -1109,7 +1110,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select alltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm pairwise_sendrecv_replace - Force pairwise sendrecv replace algorithm scattered - Force scattered algorithm @@ -1123,7 +1124,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select alltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm pairwise_exchange - Force pairwise exchange algorithm @@ -1136,7 +1137,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ialltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_blocked - Force blocked algorithm sched_inplace - Force inplace algorithm @@ -1152,7 +1153,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ialltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_pairwise_exchange - Force pairwise exchange algorithm @@ -1176,7 +1177,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select reduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) binomial - Force binomial algorithm nb - Force nonblocking algorithm smp - Force smp algorithm @@ -1191,7 +1192,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select reduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) local_reduce_remote_send - Force local-reduce-remote-send algorithm nb - Force nonblocking algorithm @@ -1318,7 +1319,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ireduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_smp - Force smp algorithm sched_binomial - Force binomial algorithm @@ -1335,7 +1336,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ireduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_local_reduce_remote_send - Force local-reduce-remote-send algorithm @@ -1359,7 +1360,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allreduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm smp - Force smp algorithm recursive_doubling - Force recursive doubling algorithm @@ -1528,7 +1529,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allreduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm reduce_exchange_bcast - Force reduce-exchange-bcast algorithm @@ -1600,12 +1601,13 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallreduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_naive - Force naive algorithm sched_smp - Force smp algorithm sched_recursive_doubling - Force recursive doubling algorithm sched_reduce_scatter_allgather - Force reduce scatter allgather algorithm + tsp_recexch - Force generic transport recursive exchange algorithm tsp_recexch_single_buffer - Force generic transport recursive exchange with single buffer for receives tsp_recexch_multiple_buffer - Force generic transport recursive exchange with multiple buffers for receives tsp_tree - Force generic transport tree algorithm @@ -1621,7 +1623,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iallreduce algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_remote_reduce_local_bcast - Force remote-reduce-local-bcast algorithm @@ -1645,7 +1647,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select reduce_scatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm noncommutative - Force noncommutative algorithm pairwise - Force pairwise algorithm @@ -1661,7 +1663,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select reduce_scatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm remote_reduce_local_scatter - Force remote-reduce-local-scatter algorithm @@ -1684,7 +1686,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ireduce_scatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_noncommutative - Force noncommutative algorithm sched_recursive_doubling - Force recursive doubling algorithm @@ -1701,7 +1703,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ireduce_scatter algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_remote_reduce_local_scatterv - Force remote-reduce-local-scatterv algorithm @@ -1714,7 +1716,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select reduce_scatter_block algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) noncommutative - Force noncommutative algorithm recursive_doubling - Force recursive doubling algorithm pairwise - Force pairwise algorithm @@ -1730,7 +1732,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select reduce_scatter_block algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm remote_reduce_local_scatter - Force remote-reduce-local-scatter algorithm @@ -1753,7 +1755,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ireduce_scatter_block algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_noncommutative - Force noncommutative algorithm sched_recursive_doubling - Force recursive doubling algorithm @@ -1770,7 +1772,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ireduce_scatter_block algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_remote_reduce_local_scatterv - Force remote-reduce-local-scatterv algorithm @@ -1783,7 +1785,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm smp - Force smp algorithm recursive_doubling - Force recursive doubling algorithm @@ -1797,7 +1799,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_smp - Force smp algorithm sched_recursive_doubling - Force recursive doubling algorithm @@ -1812,7 +1814,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm recursive_doubling - Force recursive doubling algorithm @@ -1825,7 +1827,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select iexscan algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_recursive_doubling - Force recursive doubling algorithm @@ -1838,7 +1840,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm - name : MPIR_CVAR_NEIGHBOR_ALLGATHER_INTER_ALGORITHM @@ -1850,7 +1852,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nonblocking algorithm - name : MPIR_CVAR_INEIGHBOR_ALLGATHER_INTRA_ALGORITHM @@ -1862,7 +1864,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -1876,7 +1878,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_allgather algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -1890,7 +1892,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_allgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_NEIGHBOR_ALLGATHERV_INTER_ALGORITHM @@ -1902,7 +1904,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_allgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_INEIGHBOR_ALLGATHERV_INTRA_ALGORITHM @@ -1914,7 +1916,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_allgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -1928,7 +1930,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_allgatherv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -1942,7 +1944,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_alltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_NEIGHBOR_ALLTOALL_INTER_ALGORITHM @@ -1954,7 +1956,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_alltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_INEIGHBOR_ALLTOALL_INTRA_ALGORITHM @@ -1966,7 +1968,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_alltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -1981,7 +1983,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_alltoall algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -1995,7 +1997,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_alltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_NEIGHBOR_ALLTOALLV_INTER_ALGORITHM @@ -2007,7 +2009,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_alltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_INEIGHBOR_ALLTOALLV_INTRA_ALGORITHM @@ -2019,7 +2021,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_alltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -2033,7 +2035,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_alltoallv algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -2047,7 +2049,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_alltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_NEIGHBOR_ALLTOALLW_INTER_ALGORITHM @@ -2059,7 +2061,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select neighbor_alltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) nb - Force nb algorithm - name : MPIR_CVAR_INEIGHBOR_ALLTOALLW_INTRA_ALGORITHM @@ -2071,7 +2073,7 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_alltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm @@ -2085,1001 +2087,11 @@ cvars: scope : MPI_T_SCOPE_ALL_EQ description : |- Variable to select ineighbor_alltoallw algorithm - auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE) + auto - Internal algorithm selection (can be overridden with MPIR_CVAR_COLL_SELECTION_JSON_FILE) sched_auto - Internal algorithm selection for sched-based algorithms sched_linear - Force linear algorithm tsp_linear - Force generic transport based linear algorithm - - name : MPIR_CVAR_BARRIER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Barrier will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IBARRIER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ibarrier will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_BARRIER_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Barrier will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_BCAST_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Bcast will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IBCAST_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ibcast will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_BCAST_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Bcast_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_GATHER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Gather will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IGATHER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Igather will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_GATHER_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Gather_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_GATHERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Gatherv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IGATHERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Igatherv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_GATHERV_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Gatherv_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_SCATTER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Scatter will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ISCATTER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iscatter will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_SCATTER_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Scatter_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_SCATTERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Scatterv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ISCATTERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iscatterv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_SCATTERV_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Scatterv_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLGATHER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Allgather will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IALLGATHER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iallgather will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLGATHER_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Allgather_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLGATHERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Allgatherv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IALLGATHERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iallgatherv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLGATHERV_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Allgatherv_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLTOALL_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Alltoall will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IALLTOALL_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ialltoall will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLTOALL_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Alltoall_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLTOALLV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Alltoallv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IALLTOALLV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ialltoallv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLTOALLV_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Alltoallv_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLTOALLW_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Alltoallw will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IALLTOALLW_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ialltoallw will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLTOALLW_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Alltoallw_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_REDUCE_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Reduce will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IREDUCE_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ireduce will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_REDUCE_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Reduce_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLREDUCE_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Allreduce will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IALLREDUCE_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iallreduce will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ALLREDUCE_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Allreduce_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_REDUCE_SCATTER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Reduce_scatter will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IREDUCE_SCATTER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ireduce_scatter will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_REDUCE_SCATTER_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Reduce_scatter_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_REDUCE_SCATTER_BLOCK_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Reduce_scatter_block will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IREDUCE_SCATTER_BLOCK_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ireduce_scatter_block will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_REDUCE_SCATTER_BLOCK_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Reduce_scatter_block_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_SCAN_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Scan will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_ISCAN_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iscan will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_SCAN_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Scan_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_EXSCAN_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Exscan will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_IEXSCAN_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Iexscan will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_EXSCAN_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Exscan_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLGATHER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_allgather will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_INEIGHBOR_ALLGATHER_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ineighbor_allgather will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLGATHER_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_allgather_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLGATHERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_allgatherv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_INEIGHBOR_ALLGATHERV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ineighbor_allgatherv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLGATHERV_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_allgatherv_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLTOALL_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_alltoall will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_INEIGHBOR_ALLTOALL_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ineighbor_alltoall will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLTOALL_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_alltoall_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLTOALLV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_alltoallv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_INEIGHBOR_ALLTOALLV_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ineighbor_alltoallv will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLTOALLV_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_alltoallv_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLTOALLW_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_alltoallw will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_INEIGHBOR_ALLTOALLW_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Ineighbor_alltoallw will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - - name : MPIR_CVAR_NEIGHBOR_ALLTOALLW_INIT_DEVICE_COLLECTIVE - category : COLLECTIVE - type : boolean - default : true - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : >- - This CVAR is only used when MPIR_CVAR_DEVICE_COLLECTIVES - is set to "percoll". If set to true, MPI_Neighbor_alltoallw_init will - allow the device to override the MPIR-level collective - algorithms. The device might still call the MPIR-level - algorithms manually. If set to false, the device-override - will be disabled. - - name : MPIR_CVAR_COLL_HYBRID_MEMORY category : COLLECTIVE type : boolean diff --git a/src/mpi/coll/exscan/Makefile.mk b/src/mpi/coll/exscan/Makefile.mk index a2ee7989ecf..5efd175be22 100644 --- a/src/mpi/coll/exscan/Makefile.mk +++ b/src/mpi/coll/exscan/Makefile.mk @@ -8,5 +8,4 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/exscan/exscan_allcomm_nb.c \ src/mpi/coll/exscan/exscan_intra_recursive_doubling.c diff --git a/src/mpi/coll/exscan/exscan_allcomm_nb.c b/src/mpi/coll/exscan/exscan_allcomm_nb.c deleted file mode 100644 index c015af7b101..00000000000 --- a/src/mpi/coll/exscan/exscan_allcomm_nb.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Exscan_allcomm_nb(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Iexscan(sendbuf, recvbuf, count, datatype, op, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/gather/Makefile.mk b/src/mpi/coll/gather/Makefile.mk index f6cd7ea1ac5..3846c425629 100644 --- a/src/mpi/coll/gather/Makefile.mk +++ b/src/mpi/coll/gather/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/gather/gather_allcomm_nb.c \ src/mpi/coll/gather/gather_intra_binomial.c \ src/mpi/coll/gather/gather_inter_linear.c \ src/mpi/coll/gather/gather_inter_local_gather_remote_send.c diff --git a/src/mpi/coll/gather/gather_allcomm_nb.c b/src/mpi/coll/gather/gather_allcomm_nb.c deleted file mode 100644 index 9817b26f5b7..00000000000 --- a/src/mpi/coll/gather/gather_allcomm_nb.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Gather_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, - &req_ptr); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/gatherv/Makefile.mk b/src/mpi/coll/gatherv/Makefile.mk index 6d7221fa917..bf2e48dddf3 100644 --- a/src/mpi/coll/gatherv/Makefile.mk +++ b/src/mpi/coll/gatherv/Makefile.mk @@ -8,5 +8,4 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/gatherv/gatherv_allcomm_nb.c \ src/mpi/coll/gatherv/gatherv_allcomm_linear.c diff --git a/src/mpi/coll/gatherv/gatherv_allcomm_nb.c b/src/mpi/coll/gatherv/gatherv_allcomm_nb.c deleted file mode 100644 index c8a90e5fa3f..00000000000 --- a/src/mpi/coll/gatherv/gatherv_allcomm_nb.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Gatherv_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, - comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/iallgather/iallgather_tsp_recexch.c b/src/mpi/coll/iallgather/iallgather_tsp_recexch.c index 5b849db5fd7..1e309b52f2b 100644 --- a/src/mpi/coll/iallgather/iallgather_tsp_recexch.c +++ b/src/mpi/coll/iallgather/iallgather_tsp_recexch.c @@ -70,7 +70,7 @@ static int MPIR_TSP_Iallgather_sched_intra_recexch_step1(int step1_sendto, int * int vtx_id; MPIR_FUNC_ENTER; - if (step1_sendto != -1) { /* non-participating rank sends the data to a partcipating rank */ + if (step1_sendto != -1) { /* non-participating rank sends the data to a participating rank */ void *buf_to_send; MPI_Aint send_offset = rank * recv_extent * recvcount; if (is_inplace) @@ -219,11 +219,11 @@ static int MPIR_TSP_Iallgather_sched_intra_recexch_step3(int step1_sendto, int * * paper, Sack et al, "Faster topology-aware collective algorithms through * non-minimal communication", 2012. * */ -int MPIR_TSP_Iallgather_sched_intra_recexch(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int is_dist_halving, int k, - MPIR_TSP_sched_t sched) +static int MPIR_TSP_Iallgather_sched_intra_recexch(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + MPI_Aint recvcount, MPI_Datatype recvtype, + MPIR_Comm * comm, int is_dist_halving, int k, + MPIR_TSP_sched_t sched) { int mpi_errno = MPI_SUCCESS; int is_inplace, i; @@ -321,3 +321,22 @@ int MPIR_TSP_Iallgather_sched_intra_recexch(const void *sendbuf, MPI_Aint sendco fn_fail: goto fn_exit; } + +int MPIR_TSP_Iallgather_sched_intra_recexch_doubling(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + MPI_Aint recvcount, MPI_Datatype recvtype, + MPIR_Comm * comm, int k, + MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Iallgather_sched_intra_recexch(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, 0, k, sched); +} + +int MPIR_TSP_Iallgather_sched_intra_recexch_halving(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + MPI_Aint recvcount, MPI_Datatype recvtype, + MPIR_Comm * comm, int k, MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Iallgather_sched_intra_recexch(sendbuf, sendcount, sendtype, + recvbuf, recvcount, recvtype, comm, 1, k, sched); +} diff --git a/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c b/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c index f228b787192..cf242a1498e 100644 --- a/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c +++ b/src/mpi/coll/iallgatherv/iallgatherv_tsp_recexch.c @@ -78,7 +78,7 @@ static int MPIR_TSP_Iallgatherv_sched_intra_recexch_step1(int step1_sendto, int int i, vtx_id; MPIR_FUNC_ENTER; - if (step1_sendto != -1) { /* non-participating rank sends the data to a partcipating rank */ + if (step1_sendto != -1) { /* non-participating rank sends the data to a participating rank */ void *buf_to_send; MPI_Aint send_offset = displs[rank] * recv_extent; if (is_inplace) @@ -232,11 +232,12 @@ static int MPIR_TSP_Iallgatherv_sched_intra_recexch_step3(int step1_sendto, int } /* Routine to schedule a recursive exchange based allgather */ -int MPIR_TSP_Iallgatherv_sched_intra_recexch(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - int is_dist_halving, int k, MPIR_TSP_sched_t sched) +static int MPIR_TSP_Iallgatherv_sched_intra_recexch(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + const MPI_Aint * recvcounts, + const MPI_Aint * displs, MPI_Datatype recvtype, + MPIR_Comm * comm, int is_dist_halving, int k, + MPIR_TSP_sched_t sched) { int mpi_errno = MPI_SUCCESS; int is_inplace, i; @@ -334,3 +335,29 @@ int MPIR_TSP_Iallgatherv_sched_intra_recexch(const void *sendbuf, MPI_Aint sendc fn_fail: goto fn_exit; } + +int MPIR_TSP_Iallgatherv_sched_intra_recexch_doubling(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + const MPI_Aint * recvcounts, + const MPI_Aint * displs, + MPI_Datatype recvtype, MPIR_Comm * comm, + int k, MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Iallgatherv_sched_intra_recexch(sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, comm, + 0, k, sched); + +} + +int MPIR_TSP_Iallgatherv_sched_intra_recexch_halving(const void *sendbuf, MPI_Aint sendcount, + MPI_Datatype sendtype, void *recvbuf, + const MPI_Aint * recvcounts, + const MPI_Aint * displs, MPI_Datatype recvtype, + MPIR_Comm * comm, int k, + MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Iallgatherv_sched_intra_recexch(sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, comm, + 1, k, sched); + +} diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_auto.c b/src/mpi/coll/iallreduce/iallreduce_tsp_auto.c index e8ca1600030..20149af8dc7 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_auto.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_auto.c @@ -6,6 +6,7 @@ #include "mpiimpl.h" #include "algo_common.h" #include "treealgo.h" +#include "coll_csel.h" /* Routine to schedule a pipelined tree based allreduce */ int MPIR_TSP_Iallreduce_sched_intra_tsp_auto(const void *sendbuf, void *recvbuf, MPI_Aint count, @@ -13,155 +14,13 @@ int MPIR_TSP_Iallreduce_sched_intra_tsp_auto(const void *sendbuf, void *recvbuf, MPIR_Comm * comm, MPIR_TSP_sched_t sched) { int mpi_errno = MPI_SUCCESS; - int is_commutative = MPIR_Op_is_commutative(op); - int nranks; - int rank; - MPIR_COMM_RANK_SIZE(comm, rank, nranks); MPIR_Assert(comm->comm_kind == MPIR_COMM_KIND__INTRACOMM); - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__IALLREDUCE, - .comm_ptr = comm, - - .u.iallreduce.sendbuf = sendbuf, - .u.iallreduce.recvbuf = recvbuf, - .u.iallreduce.count = count, - .u.iallreduce.datatype = datatype, - .u.iallreduce.op = op, - }; - - MPII_Csel_container_s *cnt; - - switch (MPIR_CVAR_IALLREDUCE_INTRA_ALGORITHM) { - case MPIR_CVAR_IALLREDUCE_INTRA_ALGORITHM_tsp_recexch_single_buffer: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, - datatype, op, comm, - MPIR_IALLREDUCE_RECEXCH_TYPE_SINGLE_BUFFER, - MPIR_CVAR_IALLREDUCE_RECEXCH_KVAL, sched); - break; - - case MPIR_CVAR_IALLREDUCE_INTRA_ALGORITHM_tsp_recexch_multiple_buffer: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, - datatype, op, comm, - MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER, - MPIR_CVAR_IALLREDUCE_RECEXCH_KVAL, sched); - break; - - case MPIR_CVAR_IALLREDUCE_INTRA_ALGORITHM_tsp_tree: - /*Only knomial_1 tree supports non-commutative operations */ - MPII_COLLECTIVE_FALLBACK_CHECK(rank, is_commutative || - MPIR_Iallreduce_tree_type == - MPIR_TREE_TYPE_KNOMIAL_1, mpi_errno, - "Iallreduce gentran_tree cannot be applied.\n"); - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, - comm, MPIR_Iallreduce_tree_type, - MPIR_CVAR_IALLREDUCE_TREE_KVAL, - MPIR_CVAR_IALLREDUCE_TREE_PIPELINE_CHUNK_SIZE, - MPIR_CVAR_IALLREDUCE_TREE_BUFFER_PER_CHILD, - sched); - break; - - case MPIR_CVAR_IALLREDUCE_INTRA_ALGORITHM_tsp_ring: - MPII_COLLECTIVE_FALLBACK_CHECK(rank, is_commutative, mpi_errno, - "Iallreduce gentran_ring cannot be applied.\n"); - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_ring(sendbuf, recvbuf, count, datatype, - op, comm, sched); - break; - case MPIR_CVAR_IALLREDUCE_INTRA_ALGORITHM_tsp_recexch_reduce_scatter_recexch_allgatherv: - /* This algorithm will work for commutative - * operations and if the count is bigger than total - * number of ranks. If it not commutative or if the - * count < nranks, MPIR_Iallreduce_sched algorithm - * will be run */ - MPII_COLLECTIVE_FALLBACK_CHECK(rank, is_commutative && - count >= nranks, mpi_errno, - "Iallreduce gentran_recexch_reduce_scatter_recexch_allgatherv cannot be applied.\n"); - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv(sendbuf, - recvbuf, - count, - datatype, - op, - comm, - MPIR_CVAR_IALLREDUCE_RECEXCH_KVAL, - sched); - break; - default: - cnt = MPIR_Csel_search(comm->csel_comm, coll_sig); - MPIR_Assert(cnt); - - switch (cnt->id) { - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_single_buffer: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, - datatype, op, comm, - MPIR_IALLREDUCE_RECEXCH_TYPE_SINGLE_BUFFER, - cnt->u. - iallreduce.intra_tsp_recexch_single_buffer. - k, sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_multiple_buffer: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, - datatype, op, comm, - MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER, - cnt->u. - iallreduce.intra_tsp_recexch_single_buffer. - k, sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_tree: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, - comm, - cnt->u.iallreduce. - intra_tsp_tree.tree_type, - cnt->u.iallreduce.intra_tsp_tree.k, - cnt->u.iallreduce. - intra_tsp_tree.chunk_size, - cnt->u.iallreduce. - intra_tsp_tree.buffer_per_child, - sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_ring: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_ring(sendbuf, recvbuf, count, datatype, op, - comm, sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_reduce_scatter_recexch_allgatherv: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv - (sendbuf, recvbuf, count, datatype, op, comm, - cnt->u.iallreduce.intra_tsp_recexch_reduce_scatter_recexch_allgatherv.k, - sched); - break; - - default: - /* Replace this call with MPIR_Assert(0) when json files have gentran algos */ - goto fallback; - break; - } - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: mpi_errno = MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, datatype, op, comm, MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER, MPIR_CVAR_IALLREDUCE_RECEXCH_KVAL, sched); - fn_exit: return mpi_errno; - fn_fail: - goto fn_exit; } diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c index 724d27d14b1..85bc5080fab 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch.c @@ -9,8 +9,8 @@ #include "iallreduce_tsp_recursive_exchange_common.h" /* Routine to schedule a recursive exchange based allreduce */ -int MPIR_TSP_Iallreduce_sched_intra_recexch(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, +int MPIR_TSP_Iallreduce_sched_intra_recexch(const void *sendbuf, void *recvbuf, + MPI_Aint count, MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, int per_nbr_buffer, int k, MPIR_TSP_sched_t sched) { @@ -275,3 +275,21 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch(const void *sendbuf, void *recvbuf, fn_fail: goto fn_exit; } + +int MPIR_TSP_Iallreduce_sched_intra_recexch_single_buffer(const void *sendbuf, void *recvbuf, + MPI_Aint count, MPI_Datatype datatype, + MPI_Op op, MPIR_Comm * comm, int k, + MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, datatype, op, comm, + 0, k, sched); +} + +int MPIR_TSP_Iallreduce_sched_intra_recexch_multiple_buffer(const void *sendbuf, void *recvbuf, + MPI_Aint count, MPI_Datatype datatype, + MPI_Op op, MPIR_Comm * comm, int k, + MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Iallreduce_sched_intra_recexch(sendbuf, recvbuf, count, datatype, op, comm, + 1, k, sched); +} diff --git a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c index bd7bf934414..76e1dbdc942 100644 --- a/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c +++ b/src/mpi/coll/iallreduce/iallreduce_tsp_recexch_reduce_scatter_recexch_allgatherv.c @@ -37,8 +37,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv(co void *tmp_recvbuf; void **step1_recvbuf = NULL; int tag, vtx_id; - int allgather_algo_type = MPIR_IALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING; - int redscat_algo_type = IREDUCE_SCATTER_RECEXCH_TYPE_DISTANCE_HALVING; + int is_dist_halving = 1; MPIR_CHKLMEM_DECL(); MPIR_FUNC_ENTER; @@ -114,7 +113,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv(co MPIR_TSP_Ireduce_scatter_sched_intra_recexch_step2(recvbuf, tmp_recvbuf, cnts, displs, datatype, op, extent, tag, - comm, k, redscat_algo_type, + comm, k, is_dist_halving, step2_nphases, step2_nbrs, rank, nranks, sink_id, 0, NULL, sched); @@ -123,7 +122,7 @@ int MPIR_TSP_Iallreduce_sched_intra_recexch_reduce_scatter_recexch_allgatherv(co MPIR_TSP_Iallgatherv_sched_intra_recexch_step2(step1_sendto, step2_nphases, step2_nbrs, rank, nranks, k, p_of_k, log_pofk, T, &nvtcs, &recv_id, tag, recvbuf, extent, cnts, displs, - datatype, allgather_algo_type, comm, sched); + datatype, is_dist_halving, comm, sched); } diff --git a/src/mpi/coll/ibarrier/ibarrier_tsp_auto.c b/src/mpi/coll/ibarrier/ibarrier_tsp_auto.c index 0d2f14ae2a2..01f9927dc4f 100644 --- a/src/mpi/coll/ibarrier/ibarrier_tsp_auto.c +++ b/src/mpi/coll/ibarrier/ibarrier_tsp_auto.c @@ -4,75 +4,16 @@ */ #include "mpiimpl.h" +#include "coll_csel.h" /* sched version of CVAR and json based collective selection. Meant only for gentran scheduler */ int MPIR_TSP_Ibarrier_sched_intra_tsp_auto(MPIR_Comm * comm, MPIR_TSP_sched_t sched) { int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__IBARRIER, - .comm_ptr = comm, - }; - MPII_Csel_container_s *cnt; - void *recvbuf = NULL; - - MPIR_Assert(comm->comm_kind == MPIR_COMM_KIND__INTRACOMM); - - switch (MPIR_CVAR_IBARRIER_INTRA_ALGORITHM) { - case MPIR_CVAR_IBARRIER_INTRA_ALGORITHM_tsp_recexch: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch(MPI_IN_PLACE, recvbuf, 0, - MPIR_BYTE_INTERNAL, MPI_SUM, comm, - MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER, - MPIR_CVAR_IBARRIER_RECEXCH_KVAL, sched); - break; - - case MPIR_CVAR_IBARRIER_INTRA_ALGORITHM_tsp_k_dissemination: - mpi_errno = - MPIR_TSP_Ibarrier_sched_intra_k_dissemination(comm, MPIR_CVAR_IBARRIER_DISSEM_KVAL, - sched); - break; - - default: - cnt = MPIR_Csel_search(comm->csel_comm, coll_sig); - MPIR_Assert(cnt); - - switch (cnt->id) { - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_tsp_recexch: - mpi_errno = - MPIR_TSP_Iallreduce_sched_intra_recexch(MPI_IN_PLACE, recvbuf, 0, - MPIR_BYTE_INTERNAL, MPI_SUM, comm, - MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER, - cnt->u.ibarrier.intra_tsp_recexch.k, - sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_tsp_k_dissemination: - mpi_errno = - MPIR_TSP_Ibarrier_sched_intra_k_dissemination(comm, - cnt->u. - ibarrier.intra_tsp_k_dissemination. - k, sched); - break; - - default: - /* Replace this call with MPIR_Assert(0) when json files have gentran algos */ - goto fallback; - break; - } - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: mpi_errno = MPIR_TSP_Iallreduce_sched_intra_recexch(MPI_IN_PLACE, NULL, 0, MPIR_BYTE_INTERNAL, MPI_SUM, comm, 0, 2, sched); - fn_exit: return mpi_errno; - fn_fail: - goto fn_exit; } diff --git a/src/mpi/coll/ibcast/Makefile.mk b/src/mpi/coll/ibcast/Makefile.mk index 11c9cdf492d..d804bb7a40e 100644 --- a/src/mpi/coll/ibcast/Makefile.mk +++ b/src/mpi/coll/ibcast/Makefile.mk @@ -14,7 +14,6 @@ mpi_core_sources += \ src/mpi/coll/ibcast/ibcast_intra_sched_smp.c \ src/mpi/coll/ibcast/ibcast_inter_sched_flat.c \ src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c \ - src/mpi/coll/ibcast/ibcast_tsp_scatterv_ring_allgatherv.c \ src/mpi/coll/ibcast/ibcast_tsp_tree.c \ src/mpi/coll/ibcast/ibcast_tsp_auto.c \ src/mpi/coll/ibcast/ibcast_utils.c diff --git a/src/mpi/coll/ibcast/ibcast_tsp_auto.c b/src/mpi/coll/ibcast/ibcast_tsp_auto.c index 2902d8d965f..9278a5787e7 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_auto.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_auto.c @@ -6,6 +6,7 @@ #include "mpiimpl.h" #include "algo_common.h" #include "treealgo.h" +#include "coll_csel.h" /* Provides a "flat" broadcast that doesn't know anything about * hierarchy. It will choose between several different algorithms @@ -36,8 +37,7 @@ static int MPIR_Ibcast_sched_intra_tsp_flat_auto(void *buffer, MPI_Aint count, /* gentran scatterv recexch allgather with radix 2 */ mpi_errno = MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(buffer, count, datatype, root, - comm_ptr, - MPIR_CVAR_IALLGATHERV_INTRA_ALGORITHM_tsp_recexch_doubling, + comm_ptr, 0, scatterv_k, allgatherv_k, sched); } MPIR_ERR_CHECK(mpi_errno); @@ -54,105 +54,8 @@ int MPIR_TSP_Ibcast_sched_intra_tsp_auto(void *buffer, MPI_Aint count, MPI_Datat { int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__IBCAST, - .comm_ptr = comm_ptr, - - .u.ibcast.buffer = buffer, - .u.ibcast.count = count, - .u.ibcast.datatype = datatype, - .u.ibcast.root = root, - }; - MPII_Csel_container_s *cnt; - - MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM); - - switch (MPIR_CVAR_IBCAST_INTRA_ALGORITHM) { - case MPIR_CVAR_IBCAST_INTRA_ALGORITHM_tsp_tree: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_tree(buffer, count, datatype, root, comm_ptr, - MPIR_Ibcast_tree_type, - MPIR_CVAR_IBCAST_TREE_KVAL, - MPIR_CVAR_IBCAST_TREE_PIPELINE_CHUNK_SIZE, sched); - break; - - case MPIR_CVAR_IBCAST_INTRA_ALGORITHM_tsp_scatterv_recexch_allgatherv: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(buffer, count, datatype, - root, comm_ptr, - MPIR_CVAR_IALLGATHERV_INTRA_ALGORITHM_tsp_recexch_doubling, - MPIR_CVAR_IBCAST_SCATTERV_KVAL, - MPIR_CVAR_IBCAST_ALLGATHERV_RECEXCH_KVAL, - sched); - break; - - case MPIR_CVAR_IBCAST_INTRA_ALGORITHM_tsp_scatterv_ring_allgatherv: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_scatterv_ring_allgatherv(buffer, count, datatype, - root, comm_ptr, 1, sched); - break; - - case MPIR_CVAR_IBCAST_INTRA_ALGORITHM_tsp_ring: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_tree(buffer, count, datatype, root, comm_ptr, - MPIR_TREE_TYPE_KARY, 1, - MPIR_CVAR_IBCAST_RING_CHUNK_SIZE, sched); - break; - - default: - cnt = MPIR_Csel_search(comm_ptr->csel_comm, coll_sig); - MPIR_Assert(cnt); - - switch (cnt->id) { - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_tree: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_tree(buffer, count, datatype, root, comm_ptr, - cnt->u.ibcast.intra_tsp_tree.tree_type, - cnt->u.ibcast.intra_tsp_tree.k, - cnt->u.ibcast.intra_tsp_tree.chunk_size, - sched); - break; - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_scatterv_recexch_allgatherv: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(buffer, count, datatype, - root, comm_ptr, - MPIR_CVAR_IALLGATHERV_INTRA_ALGORITHM_tsp_recexch_doubling, - cnt->u. - ibcast.intra_tsp_scatterv_recexch_allgatherv.scatterv_k, - cnt->u. - ibcast.intra_tsp_scatterv_recexch_allgatherv.allgatherv_k, - sched); - break; - - case MPIR_CVAR_IBCAST_INTRA_ALGORITHM_tsp_scatterv_ring_allgatherv: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_scatterv_ring_allgatherv(buffer, count, - datatype, root, - comm_ptr, 1, sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_ring: - mpi_errno = - MPIR_TSP_Ibcast_sched_intra_tree(buffer, count, datatype, root, comm_ptr, - MPIR_TREE_TYPE_KARY, 1, - cnt->u.ibcast.intra_tsp_tree.chunk_size, - sched); - break; - default: - /* Replace this call with MPIR_Assert(0) when json files have gentran algos */ - goto fallback; - break; - } - } - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: mpi_errno = MPIR_Ibcast_sched_intra_tsp_flat_auto(buffer, count, datatype, root, comm_ptr, sched); - fn_exit: return mpi_errno; - fn_fail: - goto fn_exit; } diff --git a/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c b/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c index b1fb176a50c..8b98132840c 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_scatterv_allgatherv.c @@ -10,7 +10,7 @@ /* Routine to schedule a scatter followed by recursive exchange based broadcast */ int MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, - MPIR_Comm * comm, int allgatherv_algo, + MPIR_Comm * comm, int use_ring, int scatterv_k, int allgatherv_k, MPIR_TSP_sched_t sched) { @@ -181,7 +181,7 @@ int MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(void *buffer, MPI_Aint count mpi_errno = MPIR_TSP_sched_fence(sched); /* wait for scatter to complete */ MPIR_ERR_CHECK(mpi_errno); - if (allgatherv_algo == MPIR_CVAR_IALLGATHERV_INTRA_ALGORITHM_tsp_ring) + if (use_ring) /* Schedule Allgatherv ring */ mpi_errno = MPIR_TSP_Iallgatherv_sched_intra_ring(MPI_IN_PLACE, cnts[rank], MPIR_BYTE_INTERNAL, @@ -190,9 +190,10 @@ int MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(void *buffer, MPI_Aint count else /* Schedule Allgatherv recexch */ mpi_errno = - MPIR_TSP_Iallgatherv_sched_intra_recexch(MPI_IN_PLACE, cnts[rank], MPIR_BYTE_INTERNAL, - tmp_buf, cnts, displs, MPIR_BYTE_INTERNAL, - comm, 0, allgatherv_k, sched); + MPIR_TSP_Iallgatherv_sched_intra_recexch_doubling(MPI_IN_PLACE, cnts[rank], + MPIR_BYTE_INTERNAL, tmp_buf, cnts, + displs, MPIR_BYTE_INTERNAL, comm, + allgatherv_k, sched); MPIR_ERR_CHECK(mpi_errno); if (!is_contig) { @@ -214,3 +215,22 @@ int MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(void *buffer, MPI_Aint count fn_fail: goto fn_exit; } + +int MPIR_TSP_Ibcast_sched_intra_scatterv_ring_allgatherv(void *buffer, MPI_Aint count, + MPI_Datatype datatype, int root, + MPIR_Comm * comm, int scatterv_k, + MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(buffer, count, datatype, root, comm, + 1, scatterv_k, 0, sched); +} + +int MPIR_TSP_Ibcast_sched_intra_scatterv_recexch_allgatherv(void *buffer, MPI_Aint count, + MPI_Datatype datatype, int root, + MPIR_Comm * comm, int scatterv_k, + int allgatherv_k, + MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Ibcast_sched_intra_scatterv_allgatherv(buffer, count, datatype, root, comm, + 0, scatterv_k, allgatherv_k, sched); +} diff --git a/src/mpi/coll/ibcast/ibcast_tsp_tree.c b/src/mpi/coll/ibcast/ibcast_tsp_tree.c index 1a720bccb00..130d447c6e9 100644 --- a/src/mpi/coll/ibcast/ibcast_tsp_tree.c +++ b/src/mpi/coll/ibcast/ibcast_tsp_tree.c @@ -98,3 +98,10 @@ int MPIR_TSP_Ibcast_sched_intra_tree(void *buffer, MPI_Aint count, MPI_Datatype fn_fail: goto fn_exit; } + +int MPIR_TSP_Ibcast_sched_intra_ring(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, + MPIR_Comm * comm, int chunk_size, MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Ibcast_sched_intra_tree(buffer, count, datatype, root, comm, + MPIR_TREE_TYPE_KARY, 1, chunk_size, sched); +} diff --git a/src/mpi/coll/include/coll_csel.h b/src/mpi/coll/include/coll_csel.h new file mode 100644 index 00000000000..fbe5f477033 --- /dev/null +++ b/src/mpi/coll/include/coll_csel.h @@ -0,0 +1,314 @@ +/* + * Copyright (C) by Argonne National Laboratory + * See COPYRIGHT in top-level directory + */ + +#ifndef COLL_CSEL_H_INCLUDED +#define COLL_CSEL_H_INCLUDED + +#include "json.h" + +typedef struct csel_node { + MPIR_Csel_node_type_e type; + union { + struct { + int idx; + } call; + struct { + MPIR_Csel_coll_type_e coll_type; + } collective; + struct { + bool negate; + int thresh; + } condition; + MPII_Csel_container_s *container; + } u; + struct csel_node *success; + struct csel_node *failure; +} MPIR_Csel_node_s; + +extern MPIR_Coll_algo_fn *MPIR_Coll_algo_table; +extern int *MPIR_Coll_cvar_table; +extern const char **MPIR_Coll_type_names; +extern const char **MPIR_Coll_algo_names; +extern const char **MPIR_Csel_condition_names; + +int MPIR_Csel_load_file(const char *json_file); +int MPIR_Csel_load_buf(const char *json_str); +int MPIR_Csel_free(void); +MPIR_Csel_node_s *MPIR_Csel_get_tree(const char *name); +MPII_Csel_container_s *MPIR_Csel_search(void *csel, MPIR_Csel_coll_sig_s * coll_sig); +void MPIR_Csel_print_tree(MPIR_Csel_node_s * node, int level); +int MPIR_Coll_run_tree(MPIR_Csel_node_s * tree, MPIR_Csel_coll_sig_s * coll_sig); + +MPL_STATIC_INLINE_PREFIX int MPIR_Csel_comm_size(MPIR_Csel_coll_sig_s * coll_sig) +{ + /* FIXME: update when we have intercomm algorithms that need select on comm_size */ + MPIR_Assert(coll_sig->comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM); + return coll_sig->comm_ptr->local_size; +} + +#define COLL_TYPE_ALL_CASE(NAME) \ + case MPIR_CSEL_COLL_TYPE__INTRA_ ## NAME: \ + case MPIR_CSEL_COLL_TYPE__INTRA_I ## NAME: \ + case MPIR_CSEL_COLL_TYPE__INTER_ ## NAME: \ + case MPIR_CSEL_COLL_TYPE__INTER_I ## NAME + +MPL_STATIC_INLINE_PREFIX MPI_Aint MPIR_Csel_avg_msg_size(MPIR_Csel_coll_sig_s * coll_sig) +{ + MPI_Aint msgsize = 0; + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(ALLREDUCE): + MPIR_Datatype_get_size_macro(coll_sig->u.allreduce.datatype, msgsize); + msgsize *= coll_sig->u.allreduce.count; + break; + COLL_TYPE_ALL_CASE(BCAST): + MPIR_Datatype_get_size_macro(coll_sig->u.bcast.datatype, msgsize); + msgsize *= coll_sig->u.bcast.count; + break; + COLL_TYPE_ALL_CASE(REDUCE): + MPIR_Datatype_get_size_macro(coll_sig->u.reduce.datatype, msgsize); + msgsize *= coll_sig->u.reduce.count; + break; + COLL_TYPE_ALL_CASE(ALLTOALL): + MPIR_Datatype_get_size_macro(coll_sig->u.alltoall.sendtype, msgsize); + msgsize *= coll_sig->u.alltoall.sendcount; + break; + default: + fprintf(stderr, "avg_msg_size not defined for coll_type %d\n", coll_sig->coll_type); + MPIR_Assert(0); + break; + } + return msgsize; +} + +MPL_STATIC_INLINE_PREFIX MPI_Aint MPIR_Csel_total_msg_size(MPIR_Csel_coll_sig_s * coll_sig) +{ + MPI_Aint total_bytes = 0; + int comm_size = coll_sig->comm_ptr->local_size; + + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(ALLREDUCE): + MPIR_Datatype_get_size_macro(coll_sig->u.allreduce.datatype, total_bytes); + total_bytes *= coll_sig->u.allreduce.count * comm_size; + break; + COLL_TYPE_ALL_CASE(BCAST): + MPIR_Datatype_get_size_macro(coll_sig->u.bcast.datatype, total_bytes); + total_bytes *= coll_sig->u.bcast.count * comm_size; + break; + COLL_TYPE_ALL_CASE(REDUCE): + MPIR_Datatype_get_size_macro(coll_sig->u.reduce.datatype, total_bytes); + total_bytes *= coll_sig->u.reduce.count * comm_size; + break; + COLL_TYPE_ALL_CASE(ALLTOALL): + MPIR_Datatype_get_size_macro(coll_sig->u.alltoall.sendtype, total_bytes); + total_bytes *= coll_sig->u.alltoall.sendcount * comm_size; + break; + COLL_TYPE_ALL_CASE(ALLTOALLV): + MPIR_Datatype_get_size_macro(coll_sig->u.alltoallv.sendtype, total_bytes); + { + MPI_Aint count = 0; + for (int i = 0; i < comm_size; i++) { + count += coll_sig->u.alltoallv.sendcounts[i]; + } + total_bytes *= count; + } + break; + COLL_TYPE_ALL_CASE(ALLTOALLW): + for (int i = 0; i < comm_size; i++) { + MPI_Aint typesize; + MPIR_Datatype_get_size_macro(coll_sig->u.alltoallw.sendtypes[i], typesize); + total_bytes += (coll_sig->u.alltoallw.sendcounts[i] * typesize); + } + break; + COLL_TYPE_ALL_CASE(ALLGATHER): + MPIR_Datatype_get_size_macro(coll_sig->u.allgather.recvtype, total_bytes); + total_bytes *= coll_sig->u.allgather.recvcount * comm_size; + break; + COLL_TYPE_ALL_CASE(ALLGATHERV): + MPIR_Datatype_get_size_macro(coll_sig->u.allgatherv.recvtype, total_bytes); + { + MPI_Aint count = 0; + for (int i = 0; i < comm_size; i++) { + count += coll_sig->u.allgatherv.recvcounts[i]; + } + total_bytes *= count; + } + break; + COLL_TYPE_ALL_CASE(GATHER): + if (coll_sig->u.gather.root == MPI_ROOT) { + MPIR_Datatype_get_size_macro(coll_sig->u.gather.recvtype, total_bytes); + /* use remote size for intercomm root */ + if (coll_sig->comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) + total_bytes = coll_sig->u.gather.recvcount * (coll_sig->comm_ptr->remote_size); + else + total_bytes = coll_sig->u.gather.recvcount * comm_size; + } else { + MPIR_Datatype_get_size_macro(coll_sig->u.gather.sendtype, total_bytes); + total_bytes = coll_sig->u.gather.sendcount * comm_size; + } + break; + COLL_TYPE_ALL_CASE(SCATTER): + if (coll_sig->u.scatter.root == MPI_ROOT) { + MPIR_Datatype_get_size_macro(coll_sig->u.scatter.sendtype, total_bytes); + /* use remote size for intercomm root */ + if (coll_sig->comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) + total_bytes = coll_sig->u.scatter.sendcount * (coll_sig->comm_ptr->remote_size); + else + total_bytes = coll_sig->u.scatter.sendcount * comm_size; + } else { + MPIR_Datatype_get_size_macro(coll_sig->u.scatter.recvtype, total_bytes); + total_bytes = coll_sig->u.scatter.recvcount * comm_size; + } + break; + COLL_TYPE_ALL_CASE(REDUCE_SCATTER): + MPIR_Datatype_get_size_macro(coll_sig->u.reduce_scatter.datatype, total_bytes); + { + MPI_Aint count = 0; + for (int i = 0; i < comm_size; i++) { + count += coll_sig->u.reduce_scatter.recvcounts[i]; + } + total_bytes *= count; + } + break; + COLL_TYPE_ALL_CASE(REDUCE_SCATTER_BLOCK): + MPIR_Datatype_get_size_macro(coll_sig->u.reduce_scatter_block.datatype, total_bytes); + total_bytes *= coll_sig->u.reduce_scatter_block.recvcount * comm_size; + break; + default: + MPIR_Assert(0); + break; + } + + return total_bytes; +} + +/* boolean csel checker_functions */ +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_comm_size_is_pof2(MPIR_Csel_coll_sig_s * coll_sig) +{ + return MPL_is_pof2(coll_sig->comm_ptr->local_size); +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_count_ge_pof2(MPIR_Csel_coll_sig_s * coll_sig) +{ + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(REDUCE): + return (coll_sig->u.reduce.count >= MPL_pof2(coll_sig->comm_ptr->local_size)); + COLL_TYPE_ALL_CASE(ALLREDUCE): + return (coll_sig->u.allreduce.count >= MPL_pof2(coll_sig->comm_ptr->local_size)); + default: + printf("MPIR_Csel_count_ge_pof2: unsupported coll_type: %s\n", + MPIR_Coll_type_names[coll_sig->coll_type]); + MPIR_Assert(0); + return false; + } +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_block_regular(MPIR_Csel_coll_sig_s * coll_sig) +{ + const MPI_Aint *counts = NULL; + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(REDUCE_SCATTER): + counts = coll_sig->u.reduce_scatter.recvcounts; + break; + default: + MPIR_Assert(0); + return false; + } + + for (int i = 1; i < coll_sig->comm_ptr->local_size; i++) { + if (counts[i] != counts[0]) { + return false; + } + } + return true; +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_displs_ordered(MPIR_Csel_coll_sig_s * coll_sig) +{ + const MPI_Aint *counts = NULL; + const MPI_Aint *displs = NULL; + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(ALLGATHERV): + counts = coll_sig->u.allgatherv.recvcounts; + displs = coll_sig->u.allgatherv.displs; + break; + default: + return false; + } + + MPI_Aint pos = 0; + for (int i = 0; i < coll_sig->comm_ptr->local_size; i++) { + if (pos != displs[i]) + return false; + pos += counts[i]; + } + return true; +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_sendbuf_inplace(MPIR_Csel_coll_sig_s * coll_sig) +{ + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(ALLTOALL): + return (coll_sig->u.alltoall.sendbuf == MPI_IN_PLACE); + COLL_TYPE_ALL_CASE(ALLTOALLV): + return (coll_sig->u.alltoallv.sendbuf == MPI_IN_PLACE); + COLL_TYPE_ALL_CASE(ALLTOALLW): + return (coll_sig->u.alltoallw.sendbuf == MPI_IN_PLACE); + default: + fprintf(stderr, "is_sendbuf_inplace not defined for coll_type %d\n", + coll_sig->coll_type); + MPIR_Assert(0); + return false; + } +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_op_is_builtin(MPIR_Csel_coll_sig_s * coll_sig) +{ + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(ALLREDUCE): + return HANDLE_IS_BUILTIN(coll_sig->u.allreduce.op); + COLL_TYPE_ALL_CASE(REDUCE): + return HANDLE_IS_BUILTIN(coll_sig->u.reduce.op); + break; + default: + fprintf(stderr, "is_op_builtin not defined for coll_type %d\n", coll_sig->coll_type); + MPIR_Assert(0); + return false; + } +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_op_is_commutative(MPIR_Csel_coll_sig_s * coll_sig) +{ + switch (coll_sig->coll_type) { + COLL_TYPE_ALL_CASE(ALLREDUCE): + return MPIR_Op_is_commutative(coll_sig->u.allreduce.op); + COLL_TYPE_ALL_CASE(REDUCE): + return MPIR_Op_is_commutative(coll_sig->u.reduce.op); + COLL_TYPE_ALL_CASE(REDUCE_SCATTER): + return MPIR_Op_is_commutative(coll_sig->u.reduce_scatter.op); + COLL_TYPE_ALL_CASE(REDUCE_SCATTER_BLOCK): + return MPIR_Op_is_commutative(coll_sig->u.reduce_scatter_block.op); + default: + fprintf(stderr, "is_commutative not defined for coll_type %d\n", coll_sig->coll_type); + MPIR_Assert(0); + return false; + } +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_is_hierarchical(MPIR_Csel_coll_sig_s * coll_sig) +{ + return MPIR_Comm_is_parent_comm(coll_sig->comm_ptr); +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_is_node_consecutive(MPIR_Csel_coll_sig_s * coll_sig) +{ + return MPII_Comm_is_node_consecutive(coll_sig->comm_ptr); +} + +MPL_STATIC_INLINE_PREFIX bool MPIR_Csel_is_node_canonical(MPIR_Csel_coll_sig_s * coll_sig) +{ + return MPII_Comm_is_node_canonical(coll_sig->comm_ptr); +} + +#endif /* COLL_CSEL_H_INCLUDED */ diff --git a/src/mpi/coll/include/coll_impl.h b/src/mpi/coll/include/coll_impl.h index 75eaeab6e94..41227192a3c 100644 --- a/src/mpi/coll/include/coll_impl.h +++ b/src/mpi/coll/include/coll_impl.h @@ -14,8 +14,6 @@ #include "../algorithms/treealgo/treealgo.h" #include "../algorithms/recexchalgo/recexchalgo.h" -#include "csel_container.h" - #define MPII_COLLECTIVE_FALLBACK_CHECK(rank, check, mpi_errno, ...) \ do { \ if ((check) == 0) { \ @@ -41,9 +39,7 @@ extern MPIR_Tree_type_t MPIR_Allreduce_tree_type; extern MPIR_Tree_type_t MPIR_Ireduce_tree_type; extern MPIR_Tree_type_t MPIR_Ibcast_tree_type; extern MPIR_Tree_type_t MPIR_Bcast_tree_type; -extern void *MPIR_Csel_root; -extern const char *MPIR_Csel_source; -extern char MPII_coll_generic_json[]; +extern char MPII_coll_selection_json[]; MPIR_Tree_type_t get_tree_type_from_string(const char *tree_str); int get_ccl_from_string(const char *ccl_str); @@ -61,6 +57,8 @@ int MPIR_Coll_safe_to_block(void); int MPII_Coll_finalize(void); +int MPIR_Coll_json(MPIR_Csel_coll_sig_s * coll_sig); + #define MPII_GENTRAN_CREATE_SCHED_P() \ do { \ *sched_type_p = MPIR_SCHED_GENTRAN; \ @@ -84,6 +82,37 @@ int MPII_Coll_finalize(void); *sched_p = s; \ } while (0) +#define MPII_CSEL_CREATE_TSP_SCHED(coll_sig) \ + do { \ + if (coll_sig->sched == NULL) { \ + coll_sig->sched_type = MPIR_SCHED_GENTRAN; \ + MPIR_TSP_sched_create(&coll_sig->sched, coll_sig->is_persistent); \ + } else { \ + MPIR_Assert(coll_sig->sched_type = MPIR_SCHED_GENTRAN); \ + } \ + } while (0) + +#define MPII_CSEL_CREATE_SCHED(coll_sig) \ + do { \ + if (coll_sig->sched == NULL) { \ + MPIR_Sched_t s = MPIR_SCHED_NULL; \ + enum MPIR_Sched_kind sched_kind = MPIR_SCHED_KIND_REGULAR; \ + if (coll_sig->is_persistent) { \ + sched_kind = MPIR_SCHED_KIND_PERSISTENT; \ + } \ + mpi_errno = MPIR_Sched_create(&s, sched_kind); \ + MPIR_ERR_CHECK(mpi_errno); \ + int tag = -1; \ + mpi_errno = MPIR_Sched_next_tag(coll_sig->comm_ptr, &tag); \ + MPIR_ERR_CHECK(mpi_errno); \ + MPIR_Sched_set_tag(s, tag); \ + coll_sig->sched_type = MPIR_SCHED_NORMAL; \ + coll_sig->sched = s; \ + } else { \ + MPIR_Assert(coll_sig->sched_type = MPIR_SCHED_NORMAL); \ + } \ + } while (0) + #define MPII_SCHED_START(sched_type, sched, comm_ptr, request) \ do { \ if (sched_type == MPIR_SCHED_NORMAL) { \ diff --git a/src/mpi/coll/include/coll_types.h b/src/mpi/coll/include/coll_types.h index a32ce6c551d..57e389a98c2 100644 --- a/src/mpi/coll/include/coll_types.h +++ b/src/mpi/coll/include/coll_types.h @@ -29,30 +29,6 @@ enum { MPIR_IALLREDUCE_RECEXCH_TYPE_MULTIPLE_BUFFER }; -/* enumerator for different recexch types */ -enum { - MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_DOUBLING = 0, - MPIR_ALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING -}; - -/* enumerator for different recexch types */ -enum { - MPIR_IALLGATHER_RECEXCH_TYPE_DISTANCE_DOUBLING = 0, - MPIR_IALLGATHER_RECEXCH_TYPE_DISTANCE_HALVING -}; - -/* enumerator for different recexch types */ -enum { - MPIR_IALLGATHERV_RECEXCH_TYPE_DISTANCE_DOUBLING = 0, - MPIR_IALLGATHERV_RECEXCH_TYPE_DISTANCE_HALVING -}; - -/* enumerator for different reduce scatter types */ -enum { - IREDUCE_SCATTER_RECEXCH_TYPE_DISTANCE_DOUBLING = 0, - IREDUCE_SCATTER_RECEXCH_TYPE_DISTANCE_HALVING -}; - /* Collectives request data structure */ typedef struct MPII_Coll_req_t { void *sched; /* pointer to the schedule */ diff --git a/src/mpi/coll/include/csel_container.h b/src/mpi/coll/include/csel_container.h deleted file mode 100644 index fae6ea479e4..00000000000 --- a/src/mpi/coll/include/csel_container.h +++ /dev/null @@ -1,412 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#ifndef MPIR_CSEL_CONTAINER_H_INCLUDED -#define MPIR_CSEL_CONTAINER_H_INCLUDED - -#include - -void *MPII_Create_container(struct json_object *obj); - -typedef enum { - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_k_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recexch_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recexch_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_inter_local_gather_remote_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_intra_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_intra_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_inter_remote_gather_local_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recursive_multiplying, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_reduce_scatter_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recexch, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_k_reduce_scatter_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_ccl, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_inter_reduce_exchange_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_k_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_pairwise, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_pairwise_sendrecv_replace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_scattered, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_inter_pairwise_exchange, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_intra_pairwise_sendrecv_replace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_intra_scattered, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_inter_pairwise_exchange, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_intra_pairwise_sendrecv_replace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_intra_scattered, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_inter_pairwise_exchange, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_k_dissemination, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_recexch, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_inter_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_recursive_doubling_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_ring_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_pipelined_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_inter_remote_send_local_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Exscan_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Exscan_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_intra_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_inter_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_inter_local_gather_remote_send, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gatherv_allcomm_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gatherv_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_sched_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_sched_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_recexch_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_recexch_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_inter_sched_local_gather_remote_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_sched_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_sched_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_recexch_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_recexch_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_inter_sched_remote_gather_local_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_naive, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_reduce_scatter_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_single_buffer, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_multiple_buffer, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_reduce_scatter_recexch_allgatherv, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_inter_sched_remote_reduce_local_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_tsp_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_tsp_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_tsp_scattered, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_brucks, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_inplace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_pairwise, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_permuted_sendrecv, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_inter_sched_pairwise_exchange, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_sched_blocked, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_sched_inplace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_tsp_scattered, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_tsp_blocked, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_tsp_inplace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_inter_sched_pairwise_exchange, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_tsp_blocked, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_tsp_inplace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_sched_blocked, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_sched_inplace, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_inter_sched_pairwise_exchange, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_tsp_recexch, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_tsp_k_dissemination, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_inter_sched_bcast, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_scatterv_recexch_allgatherv, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_scatterv_ring_allgatherv, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_scatter_recursive_doubling_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_scatter_ring_allgather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_inter_sched_flat, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iexscan_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_intra_tsp_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_intra_sched_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_inter_sched_long, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_inter_sched_short, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igatherv_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igatherv_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgather_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgather_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgatherv_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgatherv_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoall_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoall_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallv_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallv_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallw_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallw_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_ring, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_sched_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_sched_reduce_scatter_gather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_sched_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_inter_sched_local_reduce_remote_send, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_noncommutative, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_pairwise, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_recursive_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_tsp_recexch, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_inter_sched_remote_reduce_local_scatterv, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_tsp_recexch, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_noncommutative, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_pairwise, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_recursive_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_inter_sched_remote_reduce_local_scatterv, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscan_intra_sched_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscan_intra_sched_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscan_intra_tsp_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_intra_tsp_tree, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_intra_sched_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_inter_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_inter_sched_remote_send_local_scatter, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatterv_allcomm_tsp_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatterv_allcomm_sched_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_allgather_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_allgatherv_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_alltoall_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_alltoallv_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_alltoallw_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_intra_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_intra_reduce_scatter_gather, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_intra_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_inter_local_reduce_remote_send, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_noncommutative, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_pairwise, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_recursive_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_inter_remote_reduce_local_scatter, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_noncommutative, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_pairwise, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_recursive_halving, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_inter_remote_reduce_local_scatter, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scan_intra_recursive_doubling, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scan_intra_smp, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scan_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_intra_binomial, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_inter_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_inter_remote_send_local_scatter, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatterv_allcomm_linear, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatterv_allcomm_nb, - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__Algorithm_count, -} MPII_Csel_container_type_e; - -typedef struct { - MPII_Csel_container_type_e id; - - union { - struct { - struct { - int k; - } intra_tsp_brucks; - struct { - int k; - } intra_tsp_recexch_doubling; - struct { - int k; - } intra_tsp_recexch_halving; - } iallgather; - struct { - struct { - int k; - } intra_tsp_brucks; - struct { - int k; - } intra_tsp_recexch_doubling; - struct { - int k; - } intra_tsp_recexch_halving; - } iallgatherv; - struct { - struct { - int k; - } intra_tsp_recexch_single_buffer; - struct { - int k; - } intra_tsp_recexch_multiple_buffer; - struct { - int tree_type; - int k; - int chunk_size; - int buffer_per_child; - } intra_tsp_tree; - struct { - int k; - } intra_tsp_recexch_reduce_scatter_recexch_allgatherv; - } iallreduce; - struct { - struct { - int k; - int buffer_per_phase; - } intra_tsp_brucks; - struct { - int batch_size; - int bblock; - } intra_tsp_scattered; - } ialltoall; - struct { - struct { - int batch_size; - int bblock; - } intra_tsp_scattered; - struct { - int bblock; - } intra_tsp_blocked; - } ialltoallv; - struct { - struct { - int bblock; - } intra_tsp_blocked; - } ialltoallw; - struct { - struct { - int k; - } intra_k_dissemination; - struct { - int k; - bool single_phase_recv; - } intra_recexch; - } barrier; - struct { - struct { - int k; - } intra_tsp_recexch; - struct { - int k; - } intra_tsp_k_dissemination; - } ibarrier; - struct { - struct { - int tree_type; - int k; - int chunk_size; - } intra_tsp_tree; - struct { - int chunk_size; - } intra_tsp_ring; - struct { - int scatterv_k; - int allgatherv_k; - } intra_tsp_scatterv_recexch_allgatherv; - struct { - int scatterv_k; - } intra_tsp_scatterv_ring_allgatherv; - } ibcast; - struct { - struct { - int tree_type; - int k; - int is_non_blocking; - int topo_overhead; - int topo_diff_groups; - int topo_diff_switches; - int topo_same_switches; - } intra_tree; - struct { - int tree_type; - int k; - int is_non_blocking; - int chunk_size; - int recv_pre_posted; - } intra_pipelined_tree; - } bcast; - struct { - struct { - int k; - } intra_k_brucks; - struct { - int k; - bool single_phase_recv; - } intra_recexch_doubling; - struct { - int k; - bool single_phase_recv; - } intra_recexch_halving; - } allgather; - struct { - struct { - int k; - } intra_k_brucks; - } alltoall; - struct { - struct { - int k; - } intra_tsp_tree; - } igather; - struct { - struct { - int tree_type; - int k; - int chunk_size; - int buffer_per_child; - int topo_overhead; - int topo_diff_groups; - int topo_diff_switches; - int topo_same_switches; - } intra_tsp_tree; - struct { - int chunk_size; - int buffer_per_child; - } intra_tsp_ring; - } ireduce; - struct { - struct { - int k; - } intra_tsp_recexch; - } ireduce_scatter; - struct { - struct { - int k; - } intra_tsp_recexch; - } ireduce_scatter_block; - struct { - struct { - int k; - } intra_recursive_multiplying; - struct { - int tree_type; - int k; - int chunk_size; - int buffer_per_child; - int topo_overhead; - int topo_diff_groups; - int topo_diff_switches; - int topo_same_switches; - } intra_tree; - struct { - int k; - bool single_phase_recv; - } intra_recexch; - struct { - int k; - bool single_phase_recv; - } intra_k_reduce_scatter_allgather; - struct { - int ccl; - } intra_ccl; - } allreduce; - struct { - struct { - int k; - } intra_tsp_tree; - } iscatter; - } u; -} MPII_Csel_container_s; - -#endif /* MPIR_CSEL_CONTAINER_H_INCLUDED */ diff --git a/src/mpi/coll/ireduce/ireduce_tsp_auto.c b/src/mpi/coll/ireduce/ireduce_tsp_auto.c index 1666e2dd029..81634f552a4 100644 --- a/src/mpi/coll/ireduce/ireduce_tsp_auto.c +++ b/src/mpi/coll/ireduce/ireduce_tsp_auto.c @@ -4,15 +4,15 @@ */ #include "mpiimpl.h" +#include "coll_csel.h" /* Provides a "flat" reduce that doesn't know anything about * hierarchy. It will choose between several different algorithms * based on the given parameters. */ /* Remove this function when gentran algos are in json file */ -static int MPIR_Ireduce_sched_intra_tsp_flat_auto(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, - MPIR_TSP_sched_t sched) +int MPIR_Ireduce_sched_intra_tsp_flat_auto(const void *sendbuf, void *recvbuf, + MPI_Aint count, MPI_Datatype datatype, MPI_Op op, + int root, MPIR_Comm * comm_ptr, MPIR_TSP_sched_t sched) { int mpi_errno = MPI_SUCCESS; int tree_type = MPIR_TREE_TYPE_KNOMIAL_1; @@ -44,80 +44,6 @@ int MPIR_TSP_Ireduce_sched_intra_tsp_auto(const void *sendbuf, void *recvbuf, MP { int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__IREDUCE, - .comm_ptr = comm_ptr, - - .u.ireduce.sendbuf = sendbuf, - .u.ireduce.recvbuf = recvbuf, - .u.ireduce.count = count, - .u.ireduce.datatype = datatype, - .u.ireduce.op = op, - .u.ireduce.root = root, - }; - MPII_Csel_container_s *cnt; - - MPIR_Assert(comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM); - - switch (MPIR_CVAR_IREDUCE_INTRA_ALGORITHM) { - case MPIR_CVAR_IREDUCE_INTRA_ALGORITHM_tsp_tree: - /*Only knomial_1 tree supports non-commutative operations */ - MPII_COLLECTIVE_FALLBACK_CHECK(comm_ptr->rank, MPIR_Op_is_commutative(op) || - MPIR_Ireduce_tree_type == MPIR_TREE_TYPE_KNOMIAL_1, - mpi_errno, "Ireduce gentran_tree cannot be applied.\n"); - mpi_errno = - MPIR_TSP_Ireduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, root, - comm_ptr, MPIR_Ireduce_tree_type, - MPIR_CVAR_IREDUCE_TREE_KVAL, - MPIR_CVAR_IREDUCE_TREE_PIPELINE_CHUNK_SIZE, - MPIR_CVAR_IREDUCE_TREE_BUFFER_PER_CHILD, sched); - break; - - case MPIR_CVAR_IREDUCE_INTRA_ALGORITHM_tsp_ring: - mpi_errno = - MPIR_TSP_Ireduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, root, - comm_ptr, MPIR_TREE_TYPE_KARY, 1, - MPIR_CVAR_IREDUCE_RING_CHUNK_SIZE, - MPIR_CVAR_IREDUCE_TREE_BUFFER_PER_CHILD, sched); - break; - - default: - cnt = MPIR_Csel_search(comm_ptr->csel_comm, coll_sig); - MPIR_Assert(cnt); - - switch (cnt->id) { - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_tree: - mpi_errno = - MPIR_TSP_Ireduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, - root, comm_ptr, - cnt->u.ireduce.intra_tsp_tree.tree_type, - cnt->u.ireduce.intra_tsp_tree.k, - cnt->u.ireduce.intra_tsp_tree.chunk_size, - cnt->u.ireduce. - intra_tsp_tree.buffer_per_child, sched); - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_ring: - mpi_errno = - MPIR_TSP_Ireduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, - root, comm_ptr, MPIR_TREE_TYPE_KARY, 1, - cnt->u.ireduce.intra_tsp_ring.chunk_size, - cnt->u.ireduce. - intra_tsp_ring.buffer_per_child, sched); - break; - - default: - /* Replace this call with MPIR_Assert(0) when json files have gentran algos */ - mpi_errno = - MPIR_Ireduce_sched_intra_tsp_flat_auto(sendbuf, recvbuf, count, - datatype, op, root, comm_ptr, sched); - break; - } - } - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: mpi_errno = MPIR_TSP_Ireduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, MPIR_TREE_TYPE_KARY, 1, @@ -125,8 +51,5 @@ int MPIR_TSP_Ireduce_sched_intra_tsp_auto(const void *sendbuf, void *recvbuf, MP MPIR_CVAR_IREDUCE_TREE_BUFFER_PER_CHILD, sched); - fn_exit: return mpi_errno; - fn_fail: - goto fn_exit; } diff --git a/src/mpi/coll/ireduce/ireduce_tsp_tree.c b/src/mpi/coll/ireduce/ireduce_tsp_tree.c index c5f668ab286..bb3984b4720 100644 --- a/src/mpi/coll/ireduce/ireduce_tsp_tree.c +++ b/src/mpi/coll/ireduce/ireduce_tsp_tree.c @@ -6,6 +6,7 @@ #include "mpiimpl.h" #include "algo_common.h" #include "treealgo.h" +#include "coll_csel.h" /* Routine to schedule a pipelined tree based reduce */ int MPIR_TSP_Ireduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI_Aint count, @@ -71,32 +72,11 @@ int MPIR_TSP_Ireduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI_Ai MPIR_Treealgo_tree_create_topo_aware(comm, tree_type, k, tree_root, MPIR_CVAR_IREDUCE_TOPO_REORDER_ENABLE, &my_tree); } else if (tree_type == MPIR_TREE_TYPE_TOPOLOGY_WAVE) { - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__IREDUCE, - .comm_ptr = comm, - .u.ireduce.sendbuf = sendbuf, - .u.ireduce.recvbuf = recvbuf, - .u.ireduce.count = count, - .u.ireduce.datatype = datatype, - .u.ireduce.op = op, - .u.ireduce.root = tree_root, - }; - int overhead = MPIR_CVAR_IREDUCE_TOPO_OVERHEAD; int lat_diff_groups = MPIR_CVAR_IREDUCE_TOPO_DIFF_GROUPS; int lat_diff_switches = MPIR_CVAR_IREDUCE_TOPO_DIFF_SWITCHES; int lat_same_switches = MPIR_CVAR_IREDUCE_TOPO_SAME_SWITCHES; - MPII_Csel_container_s *cnt = MPIR_Csel_search(comm->csel_comm, coll_sig); - MPIR_Assert(cnt); - - if (cnt->id == MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_tree) { - overhead = cnt->u.ireduce.intra_tsp_tree.topo_overhead; - lat_diff_groups = cnt->u.ireduce.intra_tsp_tree.topo_diff_groups; - lat_diff_switches = cnt->u.ireduce.intra_tsp_tree.topo_diff_switches; - lat_same_switches = cnt->u.ireduce.intra_tsp_tree.topo_same_switches; - } - mpi_errno = MPIR_Treealgo_tree_create_topo_wave(comm, k, tree_root, MPIR_CVAR_IREDUCE_TOPO_REORDER_ENABLE, @@ -285,3 +265,13 @@ int MPIR_TSP_Ireduce_sched_intra_tree(const void *sendbuf, void *recvbuf, MPI_Ai fn_fail: goto fn_exit; } + +int MPIR_TSP_Ireduce_sched_intra_ring(const void *sendbuf, void *recvbuf, MPI_Aint count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm, int chunk_size, + int buffer_per_child, MPIR_TSP_sched_t sched) +{ + return MPIR_TSP_Ireduce_sched_intra_tree(sendbuf, recvbuf, count, datatype, op, root, comm, + MPIR_TREE_TYPE_KARY, 1, chunk_size, buffer_per_child, + sched); +} diff --git a/src/mpi/coll/neighbor_allgather/Makefile.mk b/src/mpi/coll/neighbor_allgather/Makefile.mk index 30c4f43957d..a8003424c82 100644 --- a/src/mpi/coll/neighbor_allgather/Makefile.mk +++ b/src/mpi/coll/neighbor_allgather/Makefile.mk @@ -7,5 +7,4 @@ # The code for the MPI operations (e.g., MPI_SUM) is not included in # mpi_sources -mpi_core_sources += \ - src/mpi/coll/neighbor_allgather/neighbor_allgather_allcomm_nb.c +# Currently the only algorithm for neighbor_allgather is MPIR_Coll_nb. diff --git a/src/mpi/coll/neighbor_allgather/neighbor_allgather_allcomm_nb.c b/src/mpi/coll/neighbor_allgather/neighbor_allgather_allcomm_nb.c deleted file mode 100644 index 30df508c89d..00000000000 --- a/src/mpi/coll/neighbor_allgather/neighbor_allgather_allcomm_nb.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Neighbor_allgather_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/neighbor_allgatherv/Makefile.mk b/src/mpi/coll/neighbor_allgatherv/Makefile.mk index b2b92fb8c4b..241af00feb2 100644 --- a/src/mpi/coll/neighbor_allgatherv/Makefile.mk +++ b/src/mpi/coll/neighbor_allgatherv/Makefile.mk @@ -7,5 +7,4 @@ # The code for the MPI operations (e.g., MPI_SUM) is not included in # mpi_sources -mpi_core_sources += \ - src/mpi/coll/neighbor_allgatherv/neighbor_allgatherv_allcomm_nb.c +# Currently the only algorithm for neighbor_allgatherv is MPIR_Coll_nb. diff --git a/src/mpi/coll/neighbor_allgatherv/neighbor_allgatherv_allcomm_nb.c b/src/mpi/coll/neighbor_allgatherv/neighbor_allgatherv_allcomm_nb.c deleted file mode 100644 index 471bd766f50..00000000000 --- a/src/mpi/coll/neighbor_allgatherv/neighbor_allgatherv_allcomm_nb.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Neighbor_allgatherv_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], - MPI_Datatype recvtype, MPIR_Comm * comm_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Ineighbor_allgatherv(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm_ptr, - &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/neighbor_alltoall/Makefile.mk b/src/mpi/coll/neighbor_alltoall/Makefile.mk index adacd6c806f..a086126e18c 100644 --- a/src/mpi/coll/neighbor_alltoall/Makefile.mk +++ b/src/mpi/coll/neighbor_alltoall/Makefile.mk @@ -7,5 +7,4 @@ # The code for the MPI operations (e.g., MPI_SUM) is not included in # mpi_sources -mpi_core_sources += \ - src/mpi/coll/neighbor_alltoall/neighbor_alltoall_allcomm_nb.c +# Currently the only algorithm for neighbor_alltoall is MPIR_Coll_nb. diff --git a/src/mpi/coll/neighbor_alltoall/neighbor_alltoall_allcomm_nb.c b/src/mpi/coll/neighbor_alltoall/neighbor_alltoall_allcomm_nb.c deleted file mode 100644 index 4fda947004f..00000000000 --- a/src/mpi/coll/neighbor_alltoall/neighbor_alltoall_allcomm_nb.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Neighbor_alltoall_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Ineighbor_alltoall(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/neighbor_alltoallv/Makefile.mk b/src/mpi/coll/neighbor_alltoallv/Makefile.mk index 2a70776f2cb..e50aeecbb8b 100644 --- a/src/mpi/coll/neighbor_alltoallv/Makefile.mk +++ b/src/mpi/coll/neighbor_alltoallv/Makefile.mk @@ -7,5 +7,4 @@ # The code for the MPI operations (e.g., MPI_SUM) is not included in # mpi_sources -mpi_core_sources += \ - src/mpi/coll/neighbor_alltoallv/neighbor_alltoallv_allcomm_nb.c +# Currently the only algorithm for neighbor_alltoallv is MPIR_Coll_nb. diff --git a/src/mpi/coll/neighbor_alltoallv/neighbor_alltoallv_allcomm_nb.c b/src/mpi/coll/neighbor_alltoallv/neighbor_alltoallv_allcomm_nb.c deleted file mode 100644 index af010b378c7..00000000000 --- a/src/mpi/coll/neighbor_alltoallv/neighbor_alltoallv_allcomm_nb.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Neighbor_alltoallv_allcomm_nb(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ineighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/neighbor_alltoallw/Makefile.mk b/src/mpi/coll/neighbor_alltoallw/Makefile.mk index addd9aa0751..463dad0fe31 100644 --- a/src/mpi/coll/neighbor_alltoallw/Makefile.mk +++ b/src/mpi/coll/neighbor_alltoallw/Makefile.mk @@ -7,5 +7,4 @@ # The code for the MPI operations (e.g., MPI_SUM) is not included in # mpi_sources -mpi_core_sources += \ - src/mpi/coll/neighbor_alltoallw/neighbor_alltoallw_allcomm_nb.c +# Currently the only algorithm for neighbor_alltoallw is MPIR_Coll_nb. diff --git a/src/mpi/coll/neighbor_alltoallw/neighbor_alltoallw_allcomm_nb.c b/src/mpi/coll/neighbor_alltoallw/neighbor_alltoallw_allcomm_nb.c deleted file mode 100644 index e98e9cd7bd6..00000000000 --- a/src/mpi/coll/neighbor_alltoallw/neighbor_alltoallw_allcomm_nb.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Neighbor_alltoallw_allcomm_nb(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ineighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/reduce/Makefile.mk b/src/mpi/coll/reduce/Makefile.mk index e686543e43a..6be89ca43d3 100644 --- a/src/mpi/coll/reduce/Makefile.mk +++ b/src/mpi/coll/reduce/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/reduce/reduce_allcomm_nb.c \ src/mpi/coll/reduce/reduce_intra_binomial.c \ src/mpi/coll/reduce/reduce_intra_reduce_scatter_gather.c \ src/mpi/coll/reduce/reduce_intra_smp.c \ diff --git a/src/mpi/coll/reduce/reduce_allcomm_nb.c b/src/mpi/coll/reduce/reduce_allcomm_nb.c deleted file mode 100644 index 5bfbdbb973e..00000000000 --- a/src/mpi/coll/reduce/reduce_allcomm_nb.c +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Reduce_allcomm_nb(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Ireduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/reduce_scatter/Makefile.mk b/src/mpi/coll/reduce_scatter/Makefile.mk index 96c9263b6a8..2bb1b69ef73 100644 --- a/src/mpi/coll/reduce_scatter/Makefile.mk +++ b/src/mpi/coll/reduce_scatter/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/reduce_scatter/reduce_scatter_allcomm_nb.c \ src/mpi/coll/reduce_scatter/reduce_scatter_intra_recursive_halving.c \ src/mpi/coll/reduce_scatter/reduce_scatter_intra_pairwise.c \ src/mpi/coll/reduce_scatter/reduce_scatter_intra_recursive_doubling.c \ diff --git a/src/mpi/coll/reduce_scatter/reduce_scatter_allcomm_nb.c b/src/mpi/coll/reduce_scatter/reduce_scatter_allcomm_nb.c deleted file mode 100644 index 674babdc285..00000000000 --- a/src/mpi/coll/reduce_scatter/reduce_scatter_allcomm_nb.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Reduce_scatter_allcomm_nb(const void *sendbuf, void *recvbuf, const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/reduce_scatter/reduce_scatter_inter_remote_reduce_local_scatter.c b/src/mpi/coll/reduce_scatter/reduce_scatter_inter_remote_reduce_local_scatter.c index 9b79908fe1f..e19a6b1217e 100644 --- a/src/mpi/coll/reduce_scatter/reduce_scatter_inter_remote_reduce_local_scatter.c +++ b/src/mpi/coll/reduce_scatter/reduce_scatter_inter_remote_reduce_local_scatter.c @@ -58,26 +58,26 @@ int MPIR_Reduce_scatter_inter_remote_reduce_local_scatter(const void *sendbuf, v if (comm_ptr->is_low_group) { /* reduce from right group to rank 0 */ root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* reduce to rank 0 of right group */ root = 0; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } else { /* reduce to rank 0 of left group */ root = 0; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* reduce from right group to rank 0 */ root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } diff --git a/src/mpi/coll/reduce_scatter_block/Makefile.mk b/src/mpi/coll/reduce_scatter_block/Makefile.mk index 86f19e28c64..4d493454707 100644 --- a/src/mpi/coll/reduce_scatter_block/Makefile.mk +++ b/src/mpi/coll/reduce_scatter_block/Makefile.mk @@ -7,8 +7,7 @@ # The code for the MPI operations (e.g., MPI_SUM) is not included in # mpi_sources -mpi_core_sources += \ - src/mpi/coll/reduce_scatter_block/reduce_scatter_block_allcomm_nb.c \ +mpi_core_sources += \ src/mpi/coll/reduce_scatter_block/reduce_scatter_block_intra_recursive_halving.c \ src/mpi/coll/reduce_scatter_block/reduce_scatter_block_intra_pairwise.c \ src/mpi/coll/reduce_scatter_block/reduce_scatter_block_intra_recursive_doubling.c \ diff --git a/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_allcomm_nb.c b/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_allcomm_nb.c deleted file mode 100644 index a64b7d949d8..00000000000 --- a/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_allcomm_nb.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Reduce_scatter_block_allcomm_nb(const void *sendbuf, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_inter_remote_reduce_local_scatter.c b/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_inter_remote_reduce_local_scatter.c index 06fd800823b..c088c3cf74b 100644 --- a/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_inter_remote_reduce_local_scatter.c +++ b/src/mpi/coll/reduce_scatter_block/reduce_scatter_block_inter_remote_reduce_local_scatter.c @@ -49,26 +49,26 @@ int MPIR_Reduce_scatter_block_inter_remote_reduce_local_scatter(const void *send if (comm_ptr->is_low_group) { /* reduce from right group to rank 0 */ root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* reduce to rank 0 of right group */ root = 0; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } else { /* reduce to rank 0 of left group */ root = 0; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* reduce from right group to rank 0 */ root = (rank == 0) ? MPI_ROOT : MPI_PROC_NULL; - mpi_errno = MPIR_Reduce_allcomm_auto(sendbuf, tmp_buf, total_count, datatype, op, - root, comm_ptr, coll_attr); + mpi_errno = MPIR_Reduce(sendbuf, tmp_buf, total_count, datatype, op, + root, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } diff --git a/src/mpi/coll/scan/Makefile.mk b/src/mpi/coll/scan/Makefile.mk index bd3326bb471..d559778ed7d 100644 --- a/src/mpi/coll/scan/Makefile.mk +++ b/src/mpi/coll/scan/Makefile.mk @@ -8,6 +8,5 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/scan/scan_allcomm_nb.c \ src/mpi/coll/scan/scan_intra_recursive_doubling.c \ src/mpi/coll/scan/scan_intra_smp.c diff --git a/src/mpi/coll/scan/scan_allcomm_nb.c b/src/mpi/coll/scan/scan_allcomm_nb.c deleted file mode 100644 index 314335ffbdc..00000000000 --- a/src/mpi/coll/scan/scan_allcomm_nb.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Scan_allcomm_nb(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = MPIR_Iscan(sendbuf, recvbuf, count, datatype, op, comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/scatter/Makefile.mk b/src/mpi/coll/scatter/Makefile.mk index 9ba53c1a05a..474b4adfb9b 100644 --- a/src/mpi/coll/scatter/Makefile.mk +++ b/src/mpi/coll/scatter/Makefile.mk @@ -8,7 +8,6 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/scatter/scatter_allcomm_nb.c \ src/mpi/coll/scatter/scatter_intra_binomial.c \ src/mpi/coll/scatter/scatter_inter_linear.c \ src/mpi/coll/scatter/scatter_inter_remote_send_local_scatter.c diff --git a/src/mpi/coll/scatter/scatter_allcomm_nb.c b/src/mpi/coll/scatter/scatter_allcomm_nb.c deleted file mode 100644 index 0ae69e48206..00000000000 --- a/src/mpi/coll/scatter/scatter_allcomm_nb.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Scatter_allcomm_nb(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, - &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/scatterv/Makefile.mk b/src/mpi/coll/scatterv/Makefile.mk index 89ddc847dc6..06a3d1d7ff5 100644 --- a/src/mpi/coll/scatterv/Makefile.mk +++ b/src/mpi/coll/scatterv/Makefile.mk @@ -8,5 +8,4 @@ # mpi_sources mpi_core_sources += \ - src/mpi/coll/scatterv/scatterv_allcomm_nb.c \ src/mpi/coll/scatterv/scatterv_allcomm_linear.c diff --git a/src/mpi/coll/scatterv/scatterv_allcomm_nb.c b/src/mpi/coll/scatterv/scatterv_allcomm_nb.c deleted file mode 100644 index bcbb8424878..00000000000 --- a/src/mpi/coll/scatterv/scatterv_allcomm_nb.c +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" - -int MPIR_Scatterv_allcomm_nb(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Request *req_ptr = NULL; - - /* just call the nonblocking version and wait on it */ - mpi_errno = - MPIR_Iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, - comm_ptr, &req_ptr); - MPIR_ERR_CHECK(mpi_errno); - - mpi_errno = MPIC_Wait(req_ptr); - MPIR_ERR_CHECK(mpi_errno); - MPIR_Request_free(req_ptr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpi/coll/src/Makefile.mk b/src/mpi/coll/src/Makefile.mk index 1dd1b78b204..91ee6b595b6 100644 --- a/src/mpi/coll/src/Makefile.mk +++ b/src/mpi/coll/src/Makefile.mk @@ -10,5 +10,4 @@ mpi_core_sources += \ src/mpi/coll/src/coll_impl.c \ src/mpi/coll/src/csel.c \ - src/mpi/coll/src/csel_container.c \ src/mpi/coll/src/csel_json_autogen.c diff --git a/src/mpi/coll/src/coll_impl.c b/src/mpi/coll/src/coll_impl.c index 9066f82c871..014d0d906d6 100644 --- a/src/mpi/coll/src/coll_impl.c +++ b/src/mpi/coll/src/coll_impl.c @@ -5,6 +5,7 @@ #include "mpiimpl.h" #include "coll_impl.h" +#include "coll_csel.h" /* === BEGIN_MPI_T_CVAR_INFO_BLOCK === @@ -14,20 +15,6 @@ categories : description : A category for collective communication variables. cvars: - - name : MPIR_CVAR_DEVICE_COLLECTIVES - category : COLLECTIVE - type : enum - default : percoll - class : none - verbosity : MPI_T_VERBOSITY_USER_BASIC - scope : MPI_T_SCOPE_ALL_EQ - description : |- - Variable to select whether the device can override the - MPIR-level collective algorithms. - all - Always prefer the device collectives - none - Never pick the device collectives - percoll - Use the per-collective CVARs to decide - - name : MPIR_CVAR_COLLECTIVE_FALLBACK category : COLLECTIVE type : enum @@ -43,7 +30,7 @@ categories : print - print an error message and fallback to the internally selected algorithm silent - silently fallback to the internally selected algorithm - - name : MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE + - name : MPIR_CVAR_COLL_SELECTION_JSON_FILE category : COLLECTIVE type : string default : "" @@ -51,7 +38,7 @@ categories : verbosity : MPI_T_VERBOSITY_USER_BASIC scope : MPI_T_SCOPE_ALL_EQ description : >- - Defines the location of tuning file. + Defines the location of tuning file that selects basic collective algorithms. - name : MPIR_CVAR_HIERARCHY_DUMP category : COLLECTIVE @@ -109,6 +96,20 @@ MPIR_Tree_type_t MPIR_Ireduce_tree_type = MPIR_TREE_TYPE_KARY; void *MPIR_Csel_root = NULL; const char *MPIR_Csel_source; +MPIR_Csel_node_s *csel_tree_main; +MPIR_Csel_node_s *csel_tree_auto; + +/* table of all collective algorithms */ +MPIR_Coll_algo_fn *MPIR_Coll_algo_table; + +/* table of collective algorithm cvars */ +int *MPIR_Coll_cvar_table; + +/* string tables to facilitate parsing and debugging */ +const char **MPIR_Coll_type_names; +const char **MPIR_Coll_algo_names; +const char **MPIR_Csel_condition_names; + MPIR_Tree_type_t get_tree_type_from_string(const char *tree_str) { MPIR_Tree_type_t tree_type = MPIR_TREE_TYPE_KARY; @@ -190,24 +191,45 @@ int MPII_Coll_init(void) mpi_errno = MPII_Recexchalgo_init(); MPIR_ERR_CHECK(mpi_errno); + /* FIXME: this is hackish. Define the "num" constants in coll_algos.h */ + MPIR_Coll_cvar_table = MPL_malloc(MPIR_CSEL_NUM_COLL_TYPES * sizeof(int), MPL_MEM_COLL); + MPIR_Coll_type_names = MPL_malloc(MPIR_CSEL_NUM_COLL_TYPES * sizeof(char *), MPL_MEM_COLL); + MPIR_Coll_algo_table = + MPL_malloc(MPIR_CSEL_NUM_ALGORITHMS * sizeof(MPIR_Coll_algo_fn), MPL_MEM_COLL); + MPIR_Coll_algo_names = MPL_malloc(MPIR_CSEL_NUM_ALGORITHMS * sizeof(char *), MPL_MEM_COLL); + MPIR_Csel_condition_names = MPL_malloc(MPIR_CSEL_NUM_CONDITIONS * sizeof(char *), MPL_MEM_COLL); + + MPII_Coll_type_init(); + MPII_Coll_algo_init(); + MPII_Csel_init_condition_names(); + /* initialize selection tree */ - if (!strcmp(MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE, "")) { - mpi_errno = MPIR_Csel_create_from_buf(MPII_coll_generic_json, - MPII_Create_container, &MPIR_Csel_root); - MPIR_Csel_source = "MPII_coll_generic_json"; - } else { - mpi_errno = MPIR_Csel_create_from_file(MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE, - MPII_Create_container, &MPIR_Csel_root); - MPIR_Csel_source = MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE; - } + mpi_errno = MPIR_Csel_load_buf(MPII_coll_selection_json); MPIR_ERR_CHECK(mpi_errno); + if (strcmp(MPIR_CVAR_COLL_SELECTION_JSON_FILE, "") != 0) { + mpi_errno = MPIR_Csel_load_file(MPIR_CVAR_COLL_SELECTION_JSON_FILE); + MPIR_ERR_CHECK(mpi_errno); + } + + csel_tree_main = MPIR_Csel_get_tree("main"); + MPIR_Assert(csel_tree_main); + + csel_tree_auto = MPIR_Csel_get_tree("auto"); + if (!csel_tree_auto) { + csel_tree_auto = csel_tree_main; + } + fn_exit: return mpi_errno; fn_fail: goto fn_exit; } +void MPIR_Init_coll_sig(MPIR_Csel_coll_sig_s * coll_sig) +{ +} + int MPII_Coll_finalize(void) { int mpi_errno = MPI_SUCCESS; @@ -218,9 +240,18 @@ int MPII_Coll_finalize(void) mpi_errno = MPII_TSP_finalize(); MPIR_ERR_CHECK(mpi_errno); - mpi_errno = MPIR_Csel_free(MPIR_Csel_root); + mpi_errno = MPIR_Csel_free(); MPIR_ERR_CHECK(mpi_errno); + MPL_free(MPIR_Coll_algo_table); + MPL_free(MPIR_Coll_cvar_table); + MPL_free(MPIR_Coll_algo_names); + MPL_free(MPIR_Coll_type_names); + MPL_free(MPIR_Csel_condition_names); + + csel_tree_main = NULL; + csel_tree_auto = NULL; + fn_exit: return mpi_errno; fn_fail: @@ -255,9 +286,6 @@ int MPIR_Coll_comm_init(MPIR_Comm * comm) if (mpi_errno) MPIR_ERR_POP(mpi_errno); - mpi_errno = MPIR_Csel_prune(MPIR_Csel_root, comm, &comm->csel_comm); - MPIR_ERR_CHECK(mpi_errno); - fn_exit: return mpi_errno; fn_fail: @@ -269,9 +297,6 @@ int MPII_Coll_comm_cleanup(MPIR_Comm * comm) { int mpi_errno = MPI_SUCCESS; - mpi_errno = MPIR_Csel_free(comm->csel_comm); - MPIR_ERR_CHECK(mpi_errno); - /* cleanup all collective communicators */ mpi_errno = MPII_Stubalgo_comm_cleanup(comm); MPIR_ERR_CHECK(mpi_errno); @@ -372,3 +397,79 @@ void MPIR_Coll_host_buffer_persist_set(void *host_sendbuf, void *host_recvbuf, v MPIR_Datatype_add_ref_if_not_builtin(datatype); } } + +int MPIR_Coll_run_tree(MPIR_Csel_node_s * tree, MPIR_Csel_coll_sig_s * coll_sig) +{ + int mpi_errno = MPI_SUCCESS; + + MPII_Csel_container_s *cnt = MPIR_Csel_search(tree, coll_sig); + MPIR_ERR_CHKANDJUMP(!cnt, mpi_errno, MPI_ERR_OTHER, "**csel_noresult"); + + mpi_errno = MPIR_Coll_algo_table[cnt->id] (coll_sig, cnt); + MPIR_ERR_CHECK(mpi_errno); + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +int MPIR_Coll_json(MPIR_Csel_coll_sig_s * coll_sig) +{ + int mpi_errno = MPI_SUCCESS; + + /* First check whether user has set an algorithm CVAR */ + int coll_type = coll_sig->coll_type; + int cvar_val = MPIR_Coll_cvar_table[coll_type]; + if (cvar_val) { + int algo_id = MPIR_Coll_cvar_to_algo_id(coll_type, cvar_val); + bool restriction_ok = MPIR_Coll_check_algo_restriction(coll_sig, algo_id); + + if (restriction_ok) { + MPII_Csel_container_s algo_cnt; + MPIR_Coll_init_algo_container(coll_sig, algo_id, &algo_cnt); + mpi_errno = MPIR_Coll_algo_table[algo_id] (coll_sig, &algo_cnt); + MPIR_ERR_CHECK(mpi_errno); + goto fn_exit; + } else { + /* Error or Fall-thru */ + } + } + + mpi_errno = MPIR_Coll_run_tree(csel_tree_main, coll_sig); + MPIR_ERR_CHECK(mpi_errno); + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} + +/* blocking collectives by calling its nonblocking forms */ +int MPIR_Coll_nb(MPIR_Csel_coll_sig_s * coll_sig, MPII_Csel_container_s * me) +{ + int mpi_errno = MPI_SUCCESS; + + /* Trick: blocking coll_type is even. Its nonblocking type is +1 */ + MPIR_Assert(coll_sig->coll_type % 2 == 0); + coll_sig->coll_type += 1; + + mpi_errno = MPIR_Coll_run_tree(csel_tree_auto, coll_sig); + MPIR_ERR_CHECK(mpi_errno); + + MPIR_Request *req; + MPII_SCHED_START(coll_sig->sched_type, coll_sig->sched, coll_sig->comm_ptr, &req); + + mpi_errno = MPIC_Wait(req); + MPIR_ERR_CHECK(mpi_errno); + MPIR_Request_free(req); + + /* clean up coll_sig just in case */ + coll_sig->coll_type -= 1; + coll_sig->sched = NULL; + + fn_exit: + return mpi_errno; + fn_fail: + goto fn_exit; +} diff --git a/src/mpi/coll/src/csel.c b/src/mpi/coll/src/csel.c index b2712350764..8cb375dbc7c 100644 --- a/src/mpi/coll/src/csel.c +++ b/src/mpi/coll/src/csel.c @@ -5,343 +5,108 @@ #include "mpiimpl.h" #include "mpl.h" -#include "mpir_csel.h" +#include "coll_csel.h" #include /* open */ #include /* mmap */ #include #include -typedef enum { - /* global operator types */ - CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED = 0, - - /* comm-specific operator types */ - CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTRA, - CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTER, - - CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LE, - CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LT, - CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE, - CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_POW2, - - CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY, - CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE, - - CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE, - CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT, - - /* collective selection operator */ - CSEL_NODE_TYPE__OPERATOR__COLLECTIVE, - - /* message-specific operator types */ - CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LE, - CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LT, - CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LE, - CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LT, - - CSEL_NODE_TYPE__OPERATOR__COUNT_LE, - CSEL_NODE_TYPE__OPERATOR__COUNT_LT_POW2, - - CSEL_NODE_TYPE__OPERATOR__IS_SBUF_INPLACE, - CSEL_NODE_TYPE__OPERATOR__IS_BLOCK_REGULAR, - CSEL_NODE_TYPE__OPERATOR__IS_COMMUTATIVE, - CSEL_NODE_TYPE__OPERATOR__IS_OP_BUILT_IN, - - /* any - has to be the last branch in an array */ - CSEL_NODE_TYPE__OPERATOR__ANY, - - /* container type */ - CSEL_NODE_TYPE__CONTAINER, -} csel_node_type_e; - -typedef struct csel_node { - csel_node_type_e type; - - union { - /* global types */ - struct { - int val; - } is_multi_threaded; - - /* comm-specific operator types */ - struct { - int val; - } comm_size_le; - struct { - int val; - } comm_size_lt; - - /* collective selection operator */ - struct { - MPIR_Csel_coll_type_e coll_type; - } collective; - - /* message-specific operator types */ - struct { - int val; - } avg_msg_size_le; - struct { - int val; - } avg_msg_size_lt; - struct { - int val; - } total_msg_size_le; - struct { - int val; - } total_msg_size_lt; - struct { - int val; - } count_le; - struct { - bool val; - } is_commutative; - struct { - bool val; - } is_sbuf_inplace; - struct { - bool val; - } is_op_built_in; - struct { - bool val; - } is_block_regular; - struct { - bool val; - } is_node_consecutive; - struct { - int val; - } comm_avg_ppn_le; - struct { - int val; - } comm_avg_ppn_lt; - struct { - bool val; - } comm_hierarchy; - struct { - void *container; - } cnt; - } u; - - struct csel_node *success; - struct csel_node *failure; -} csel_node_s; - -typedef enum { - CSEL_TYPE__ROOT, - CSEL_TYPE__PRUNED, -} csel_type_e; - -typedef struct { - csel_type_e type; - - union { - struct { - csel_node_s *tree; - } root; - struct { - /* one tree for each collective */ - csel_node_s *coll_trees[MPIR_CSEL_COLL_TYPE__END]; - } pruned; - } u; -} csel_s; - -static int nesting = -1; -#define nprintf(...) \ - do { \ - for (int i = 0; i < nesting; i++) \ - printf(" "); \ - printf(__VA_ARGS__); \ - } while (0) - -static void print_tree(csel_node_s * node) ATTRIBUTE((unused)); -static void print_tree(csel_node_s * node) -{ - nesting++; - - if (node == NULL) - return; +#include "uthash.h" +#include "utarray.h" + +#define CSEL_MAX_NAME 50 +struct csel_subtree_hash_item { + char name[CSEL_MAX_NAME]; /* key */ + int idx; + UT_hash_handle hh; +}; + +struct csel_subtree { + MPIR_Csel_node_s *node; + struct json_object *json_obj; +}; + +/* a dictionary maps subtree name to index into the utarray csel_subtree */ +struct csel_subtree_hash_item *csel_subtree_hash; +/* a dynamic array of (struct csel_subtree) subtrees */ +UT_array *csel_subtrees = NULL; +/* index to the main tree */ +int csel_main_idx = -1; + +static int csel_name_to_idx(const char *name); +static MPIR_Csel_node_s *csel_get_tree_by_idx(int idx); +static void add_named_tree(const char *name, struct json_object *obj); +static void replace_named_tree(int idx, struct json_object *obj); +static int parse_named_trees(void); +static void free_tree(MPIR_Csel_node_s * node); - switch (node->type) { - case CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED: - nprintf("MPI library is multithreaded\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTRA: - nprintf("comm_type is intra\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTER: - nprintf("comm_type is inter\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COLLECTIVE: - nprintf("collective: %d\n", node->u.collective.coll_type); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LE: - nprintf("comm_size <= %d\n", node->u.comm_size_le.val); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LT: - nprintf("comm_size < %d\n", node->u.comm_size_lt.val); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_POW2: - nprintf("comm_size is power-of-two\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE: - nprintf("comm_size is the same as node_comm_size\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LE: - nprintf("avg_msg_size <= %d\n", node->u.avg_msg_size_le.val); - break; - case CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LT: - nprintf("avg_msg_size < %d\n", node->u.avg_msg_size_lt.val); - break; - case CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LE: - nprintf("total_msg_size <= %d\n", node->u.total_msg_size_le.val); - break; - case CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LT: - nprintf("total_msg_size < %d\n", node->u.total_msg_size_lt.val); - break; - case CSEL_NODE_TYPE__CONTAINER: - nprintf("container\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COUNT_LE: - nprintf("count <= %d\n", node->u.count_le.val); - break; - case CSEL_NODE_TYPE__OPERATOR__COUNT_LT_POW2: - nprintf("count < nearest power-of-two less than comm size\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__IS_SBUF_INPLACE: - nprintf("source buffer is MPI_IN_PLACE\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__IS_BLOCK_REGULAR: - nprintf("all blocks have the same count\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY: - if (node->u.comm_hierarchy.val) - nprintf("communicator has hierarchical structure\n"); - else - nprintf("communicator does not have hierarchical structure\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE: - nprintf("process ranks are consecutive on the node\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE: - nprintf("communicator's avg ppn <= %d\n", node->u.comm_avg_ppn_le.val); - break; - case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT: - nprintf("communicator's avg ppn < %d\n", node->u.comm_avg_ppn_lt.val); - break; - case CSEL_NODE_TYPE__OPERATOR__IS_COMMUTATIVE: - if (node->u.is_commutative.val == true) - nprintf("operation is commutative\n"); - else - nprintf("operation is not commutative\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__IS_OP_BUILT_IN: - nprintf("other operators\n"); - break; - case CSEL_NODE_TYPE__OPERATOR__ANY: - nprintf("any\n"); - break; - default: - nprintf("unknown operator\n"); - MPIR_Assert(0); - } +static bool key_is_any(const char *ckey) +{ + int len = strlen(ckey); - if (node->type != CSEL_NODE_TYPE__CONTAINER) { - print_tree(node->success); - if (node->failure) { - nesting--; - print_tree(node->failure); - nesting++; - } + if (strcmp(ckey, "any") == 0 || strcmp(ckey + len - 5, "(any)") == 0) { + return true; + } else { + return false; } - - nesting--; } -static void validate_tree(csel_node_s * node) +static MPIR_Csel_node_s *parse_json_tree(struct json_object *obj) { - static int coll = -1; - - /* if we reached a leaf node, we are done */ - if (node->type == CSEL_NODE_TYPE__CONTAINER) - return; + enum json_type type; + type = json_object_get_type(obj); - /* if we see the collective type, store it */ - if (node->type == CSEL_NODE_TYPE__OPERATOR__COLLECTIVE) - coll = node->u.collective.coll_type; + if (type == json_type_string) { + const char *ckey = json_object_get_string(obj); - /* success path should never be NULL */ - if (node->success == NULL) { - fprintf(stderr, "unexpected NULL success path for coll %d\n", coll); - MPIR_Assert(0); - } else { - validate_tree(node->success); - } + MPIR_Csel_node_s *tmp; + tmp = MPL_calloc(sizeof(MPIR_Csel_node_s), 1, MPL_MEM_COLL); - if (node->type == CSEL_NODE_TYPE__OPERATOR__ANY) { - /* for "ANY"-style operators, the failure path must be NULL */ - if (node->failure) { - fprintf(stderr, "unexpected non-NULL failure path for coll %d\n", coll); - MPIR_Assert(0); - } - } else if (node->type != CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED && - node->type != CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTRA && - node->type != CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTER && - node->type != CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY && - node->type != CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE && - node->type != CSEL_NODE_TYPE__OPERATOR__IS_SBUF_INPLACE && - node->type != CSEL_NODE_TYPE__OPERATOR__IS_BLOCK_REGULAR && - node->type != CSEL_NODE_TYPE__OPERATOR__IS_COMMUTATIVE && - node->type != CSEL_NODE_TYPE__OPERATOR__IS_OP_BUILT_IN && - node->type != CSEL_NODE_TYPE__OPERATOR__COLLECTIVE) { - /* for boolean types, the failure path might or might not be - * NULL, but for everything else, the failure path must not be - * NULL */ - if (node->failure == NULL) { - fprintf(stderr, "unexpected NULL failure path for coll %d\n", coll); + if (!strncmp(ckey, "call=", 5)) { + tmp->type = CSEL_NODE_TYPE__OPERATOR__CALL; + tmp->u.call.idx = csel_name_to_idx(ckey + 5); + MPIR_Assert(tmp->u.call.idx >= 0); + return tmp; + } else { MPIR_Assert(0); + return NULL; } } - if (node->success) - validate_tree(node->success); - if (node->failure) - validate_tree(node->failure); -} + MPIR_Assert(type == json_type_object); + MPIR_Csel_node_s *prevnode = NULL, *node = NULL; -static bool key_is_any(const char *ckey) -{ - int len = strlen(ckey); - - if (strcmp(ckey, "any") == 0 || strcmp(ckey + len - 4, "=any") == 0) { - return true; - } else { - return false; - } -} + json_object_object_foreach(obj, key, val) { + const char *ckey = key; -static csel_node_s *parse_json_tree(struct json_object *obj, - void *(*create_container) (struct json_object *)) -{ - enum json_type type ATTRIBUTE((unused)); - csel_node_s *prevnode = NULL, *tmp, *node = NULL; + MPIR_Csel_node_s *tmp; + tmp = MPL_calloc(sizeof(MPIR_Csel_node_s), 1, MPL_MEM_COLL); - json_object_object_foreach(obj, key, val) { - type = json_object_get_type(val); - MPIR_Assert(type == json_type_object); + if (!strncmp(ckey, "algorithm=", strlen("algorithm="))) { + const char *s = ckey + strlen("algorithm="); - char *ckey = MPL_strdup_no_spaces(key); + MPII_Csel_container_s *cnt = MPL_calloc(sizeof(MPII_Csel_container_s), 1, MPL_MEM_COLL); + cnt->id = MPIR_CSEL_NUM_ALGORITHMS; + for (int i = 0; i < MPIR_CSEL_NUM_ALGORITHMS; i++) { + if (!strcmp(s, MPIR_Coll_algo_names[i])) { + cnt->id = i; + break; + } + } + if (cnt->id == MPIR_CSEL_NUM_ALGORITHMS) { + printf("parse_json_tree: unrecognized algorithm %s\n", s); + return NULL; + } - tmp = MPL_malloc(sizeof(csel_node_s), MPL_MEM_COLL); + MPII_Csel_parse_container_params(val, cnt); - if (!strncmp(ckey, "composition=", strlen("composition=")) || - !strncmp(ckey, "algorithm=", strlen("algorithm="))) { tmp->type = CSEL_NODE_TYPE__CONTAINER; - tmp->u.cnt.container = create_container(obj); - MPL_free(ckey); + tmp->u.container = cnt; return tmp; } /* this node must be an operator type */ - tmp->success = parse_json_tree(json_object_object_get(obj, key), create_container); + tmp->success = parse_json_tree(json_object_object_get(obj, key)); tmp->failure = NULL; if (node == NULL) @@ -350,225 +115,106 @@ static csel_node_s *parse_json_tree(struct json_object *obj, prevnode->failure = tmp; prevnode = tmp; - if (!strcmp(ckey, "is_multi_threaded=yes")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED; - tmp->u.is_multi_threaded.val = true; - } else if (!strcmp(ckey, "is_multi_threaded=no")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED; - tmp->u.is_multi_threaded.val = false; - } else if (!strcmp(ckey, "comm_type=intra")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTRA; - } else if (!strcmp(ckey, "comm_type=inter")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTER; - } else if (!strncmp(ckey, "collective=", strlen("collective="))) { + if (!strncmp(ckey, "collective=", strlen("collective="))) { tmp->type = CSEL_NODE_TYPE__OPERATOR__COLLECTIVE; - - char *str = ckey + strlen("collective="); - if (!strcmp(str, "allgather")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ALLGATHER; - else if (!strcmp(str, "allgatherv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ALLGATHERV; - else if (!strcmp(str, "allreduce")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ALLREDUCE; - else if (!strcmp(str, "alltoall")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ALLTOALL; - else if (!strcmp(str, "alltoallv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ALLTOALLV; - else if (!strcmp(str, "alltoallw")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ALLTOALLW; - else if (!strcmp(str, "barrier")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__BARRIER; - else if (!strcmp(str, "bcast")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__BCAST; - else if (!strcmp(str, "exscan")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__EXSCAN; - else if (!strcmp(str, "gather")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__GATHER; - else if (!strcmp(str, "gatherv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__GATHERV; - else if (!strcmp(str, "iallgather")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IALLGATHER; - else if (!strcmp(str, "iallgatherv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IALLGATHERV; - else if (!strcmp(str, "iallreduce")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IALLREDUCE; - else if (!strcmp(str, "ialltoall")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IALLTOALL; - else if (!strcmp(str, "ialltoallv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IALLTOALLV; - else if (!strcmp(str, "ialltoallw")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IALLTOALLW; - else if (!strcmp(str, "ibarrier")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IBARRIER; - else if (!strcmp(str, "ibcast")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IBCAST; - else if (!strcmp(str, "iexscan")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IEXSCAN; - else if (!strcmp(str, "igather")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IGATHER; - else if (!strcmp(str, "igatherv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IGATHERV; - else if (!strcmp(str, "ineighbor_allgather")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLGATHER; - else if (!strcmp(str, "ineighbor_allgatherv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLGATHERV; - else if (!strcmp(str, "ineighbor_alltoall")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLTOALL; - else if (!strcmp(str, "ineighbor_alltoallv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLTOALLV; - else if (!strcmp(str, "ineighbor_alltoallw")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__INEIGHBOR_ALLTOALLW; - else if (!strcmp(str, "ireduce")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IREDUCE; - else if (!strcmp(str, "ireduce_scatter")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER; - else if (!strcmp(str, "ireduce_scatter_block")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER_BLOCK; - else if (!strcmp(str, "iscan")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ISCAN; - else if (!strcmp(str, "iscatter")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ISCATTER; - else if (!strcmp(str, "iscatterv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__ISCATTERV; - else if (!strcmp(str, "neighbor_allgather")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLGATHER; - else if (!strcmp(str, "neighbor_allgatherv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLGATHERV; - else if (!strcmp(str, "neighbor_alltoall")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLTOALL; - else if (!strcmp(str, "neighbor_alltoallv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLTOALLV; - else if (!strcmp(str, "neighbor_alltoallw")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__NEIGHBOR_ALLTOALLW; - else if (!strcmp(str, "reduce")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__REDUCE; - else if (!strcmp(str, "reduce_scatter")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER; - else if (!strcmp(str, "reduce_scatter_block")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER_BLOCK; - else if (!strcmp(str, "scan")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__SCAN; - else if (!strcmp(str, "scatter")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__SCATTER; - else if (!strcmp(str, "scatterv")) - tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__SCATTERV; - else { - MPIR_Assert(0); + const char *s = ckey + strlen("collective="); + tmp->u.collective.coll_type = MPIR_CSEL_COLL_TYPE__END; + for (int i = 0; i < MPIR_CSEL_NUM_COLL_TYPES; i++) { + if (!strcmp(s, MPIR_Coll_type_names[i])) { + tmp->u.collective.coll_type = i; + break; + } } - } else if (!strcmp(ckey, "comm_size=pow2")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_POW2; - } else if (!strcmp(ckey, "comm_size=node_comm_size")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE; - } else if (!strncmp(ckey, "comm_size<=", strlen("comm_size<="))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LE; - tmp->u.comm_size_le.val = atoi(ckey + strlen("comm_size<=")); - } else if (!strncmp(ckey, "comm_size<", strlen("comm_size<"))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LT; - tmp->u.comm_size_lt.val = atoi(ckey + strlen("comm_size<")); - } else if (!strncmp(ckey, "count<=", strlen("count<="))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COUNT_LE; - tmp->u.count_le.val = atoi(ckey + strlen("count<=")); - } else if (!strcmp(ckey, "counttype = CSEL_NODE_TYPE__OPERATOR__COUNT_LT_POW2; - } else if (!strncmp(ckey, "avg_msg_size<=", strlen("avg_msg_size<="))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LE; - tmp->u.avg_msg_size_le.val = atoi(ckey + strlen("avg_msg_size<=")); - } else if (!strncmp(ckey, "avg_msg_size<", strlen("avg_msg_size<"))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LT; - tmp->u.avg_msg_size_lt.val = atoi(ckey + strlen("avg_msg_size<")); - } else if (!strncmp(ckey, "total_msg_size<=", strlen("total_msg_size<="))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LE; - tmp->u.total_msg_size_le.val = atoi(ckey + strlen("total_msg_size<=")); - } else if (!strncmp(ckey, "total_msg_size<", strlen("total_msg_size<"))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LT; - tmp->u.total_msg_size_lt.val = atoi(ckey + strlen("total_msg_size<")); - } else if (!strcmp(ckey, "is_commutative=yes")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_COMMUTATIVE; - tmp->u.is_commutative.val = true; - } else if (!strcmp(ckey, "is_commutative=no")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_COMMUTATIVE; - tmp->u.is_commutative.val = false; - } else if (!strcmp(ckey, "is_sendbuf_inplace=yes")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_SBUF_INPLACE; - tmp->u.is_sbuf_inplace.val = true; - } else if (!strcmp(ckey, "is_sendbuf_inplace=no")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_SBUF_INPLACE; - tmp->u.is_sbuf_inplace.val = false; - } else if (!strcmp(ckey, "is_op_built_in=yes")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_OP_BUILT_IN; - tmp->u.is_op_built_in.val = true; - } else if (!strcmp(ckey, "is_op_built_in=no")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_OP_BUILT_IN; - tmp->u.is_op_built_in.val = false; - } else if (!strcmp(ckey, "is_block_regular=yes")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_BLOCK_REGULAR; - tmp->u.is_block_regular.val = true; - } else if (!strcmp(ckey, "is_block_regular=no")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_BLOCK_REGULAR; - tmp->u.is_block_regular.val = false; - } else if (!strcmp(ckey, "is_node_consecutive=yes")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE; - tmp->u.is_node_consecutive.val = true; - } else if (!strcmp(ckey, "is_node_consecutive=no")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE; - tmp->u.is_node_consecutive.val = false; - } else if (!strncmp(ckey, "comm_avg_ppn<=", strlen("comm_avg_ppn<="))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE; - tmp->u.comm_avg_ppn_le.val = atoi(ckey + strlen("comm_avg_ppn<=")); - } else if (!strncmp(ckey, "comm_avg_ppn<", strlen("comm_avg_ppn<"))) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT; - tmp->u.comm_avg_ppn_le.val = atoi(ckey + strlen("comm_avg_ppn<")); - } else if (!strcmp(ckey, "comm_hierarchy=parent")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY; - tmp->u.comm_hierarchy.val = true; - } else if (!strcmp(ckey, "comm_hierarchy=node_roots")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY; - tmp->u.comm_hierarchy.val = false; - } else if (!strcmp(ckey, "comm_hierarchy=node")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY; - tmp->u.comm_hierarchy.val = false; - } else if (!strcmp(ckey, "comm_hierarchy=flat")) { - tmp->type = CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY; - tmp->u.comm_hierarchy.val = false; + MPIR_Assert(tmp->u.collective.coll_type != MPIR_CSEL_COLL_TYPE__END); } else if (key_is_any(ckey)) { tmp->type = CSEL_NODE_TYPE__OPERATOR__ANY; } else { - fprintf(stderr, "unknown key %s\n", key); - fflush(stderr); - MPIR_Assert(0); + int mpi_errno = MPII_Csel_parse_operator(ckey, tmp); + if (mpi_errno != MPI_SUCCESS) { + printf("parse_json_tree: unknown key %s\n", ckey); + MPIR_Assert(0); + } } - - MPL_free(ckey); } return node; } -int MPIR_Csel_create_from_buf(const char *json, - void *(*create_container) (struct json_object *), void **csel_) +/* parse the json_object assuming it is a list of "name=xxx": subtree. + * Otherwise, parse it as a single tree and set the name to "main". + */ +static void csel_subtree_dtor(void *item) { - csel_s *csel = NULL; - struct json_object *tree; + struct csel_subtree *p = item; + MPIR_Assert(p->json_obj == NULL); + if (p->node) { + free_tree(p->node); + p->node = NULL; + } +} + +static int parse_json_names(struct json_object *obj) +{ + int mpi_errno = MPI_SUCCESS; + + if (!csel_subtrees) { + static UT_icd csel_subtree_icd = + { sizeof(struct csel_subtree), NULL, NULL, csel_subtree_dtor }; + utarray_new(csel_subtrees, &csel_subtree_icd, MPL_MEM_COLL); + } + + int num_names = 0; + bool unexpected = false; + json_object_object_foreach(obj, key, val) { + if (strncmp(key, "name=", 5) == 0) { + const char *name = key + 5; + + int idx = csel_name_to_idx(name); + if (idx >= 0) { + replace_named_tree(idx, json_object_get(val)); + } else { + add_named_tree(name, json_object_get(val)); + } + num_names++; + } else { + unexpected = true; + break; + } + } + if (unexpected) { + MPIR_Assert(num_names == 0); + /* pass the whole tree as "name=main" */ + add_named_tree("main", json_object_get(obj)); + } + + MPIR_Assert(csel_main_idx >= 0); + /* FIXME: error handling */ - csel = (csel_s *) MPL_malloc(sizeof(csel_s), MPL_MEM_COLL); - csel->type = CSEL_TYPE__ROOT; - tree = json_tokener_parse(json); + mpi_errno = parse_named_trees(); + MPIR_ERR_CHECK(mpi_errno); + + fn_fail: + return mpi_errno; +} + +int MPIR_Csel_load_buf(const char *json_str) +{ + int mpi_errno = MPI_SUCCESS; + + struct json_object *tree; + tree = json_tokener_parse(json_str); if (tree == NULL) goto fn_exit; - csel->u.root.tree = parse_json_tree(tree, create_container); - if (csel->u.root.tree) - validate_tree(csel->u.root.tree); + mpi_errno = parse_json_names(tree); json_object_put(tree); fn_exit: - *csel_ = csel; - return 0; + return mpi_errno; } -int MPIR_Csel_create_from_file(const char *json_file, - void *(*create_container) (struct json_object *), void **csel_) +int MPIR_Csel_load_file(const char *json_file) { int mpi_errno = MPI_SUCCESS; @@ -583,85 +229,38 @@ int MPIR_Csel_create_from_file(const char *json_file, char *json = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); close(fd); - MPIR_Csel_create_from_buf(json, create_container, csel_); + mpi_errno = MPIR_Csel_load_buf(json); fn_fail: return mpi_errno; } -static csel_node_s *prune_tree(csel_node_s * root, MPIR_Comm * comm_ptr) +int MPIR_Csel_free(void) { - /* Do not prune tree based on CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED, as during init - * MPIR_IS_THREADED is set to 0 temporarily, which results in having incorrect pruned tree */ - for (csel_node_s * node = root; node;) { - switch (node->type) { - case CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTRA: - if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTER: - if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LE: - if (comm_ptr->local_size <= node->u.comm_size_le.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LT: - if (comm_ptr->local_size < node->u.comm_size_lt.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE: - if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && comm_ptr->num_external == 1) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_POW2: - if (comm_ptr->local_size & (comm_ptr->local_size - 1)) - node = node->failure; - else - node = node->success; - break; + int mpi_errno = MPI_SUCCESS; - case CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY: - if (MPIR_Comm_is_parent_comm(comm_ptr) == node->u.comm_hierarchy.val) - node = node->success; - else - node = node->failure; - break; + utarray_clear(csel_subtrees); + utarray_free(csel_subtrees); + csel_subtrees = NULL; - case CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE: - if (MPII_Comm_is_node_consecutive(comm_ptr) == node->u.is_node_consecutive.val) - node = node->success; - else - node = node->failure; - break; + return mpi_errno; +} - case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE: - if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && - comm_ptr->local_size <= node->u.comm_avg_ppn_le.val * comm_ptr->num_external) - node = node->success; - else - node = node->failure; - break; +MPII_Csel_container_s *MPIR_Csel_search(void *csel_, MPIR_Csel_coll_sig_s * coll_sig) +{ + MPIR_Assert(csel_); + MPIR_Csel_node_s *node = csel_; + while (node) { + switch (node->type) { + case CSEL_NODE_TYPE__OPERATOR__CALL: + MPIR_Csel_node_s * tree = csel_get_tree_by_idx(node->u.call.idx); + if (!tree) { + goto fn_fail; + } + return MPIR_Csel_search(tree, coll_sig); - case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT: - if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && - comm_ptr->local_size < node->u.comm_avg_ppn_le.val * comm_ptr->num_external) + case CSEL_NODE_TYPE__OPERATOR__COLLECTIVE: + if (node->u.collective.coll_type == coll_sig->coll_type) node = node->success; else node = node->failure; @@ -671,57 +270,36 @@ static csel_node_s *prune_tree(csel_node_s * root, MPIR_Comm * comm_ptr) node = node->success; break; + case CSEL_NODE_TYPE__CONTAINER: + return node->u.container; + default: - return node; + node = MPII_Csel_run_condition(node, coll_sig); + MPIR_Assert(node); + break; } } - return root; + + fn_fail: + MPIR_Assert(0 && "MPIR_Csel_search failed to find an algorithm"); + return NULL; } -/* The prune function allows us to simplify the tree for specific - * communicators using comm-specific information (such as size and - * intra/inter comm type. */ -int MPIR_Csel_prune(void *root_csel, MPIR_Comm * comm_ptr, void **comm_csel_) +MPIR_Csel_node_s *MPIR_Csel_get_tree(const char *name) { - int mpi_errno = MPI_SUCCESS; - csel_s *csel = (csel_s *) root_csel; - csel_s *comm_csel = NULL; - - MPIR_Assert(root_csel); - MPIR_Assert(comm_ptr); - - comm_csel = (csel_s *) MPL_malloc(sizeof(csel_s), MPL_MEM_COLL); - MPIR_Assert(comm_csel); - - comm_csel->type = CSEL_TYPE__PRUNED; - for (int i = 0; i < MPIR_CSEL_COLL_TYPE__END; i++) - comm_csel->u.pruned.coll_trees[i] = NULL; - - /* prune the tree as far as possible */ - csel_node_s *node = prune_tree(csel->u.root.tree, comm_ptr); - - /* if the tree is not NULL, we should be at a collective branch at - * this point */ - if (node) { - MPIR_Assert(node->type == CSEL_NODE_TYPE__OPERATOR__COLLECTIVE); - } - - while (node) { - /* see if any additional pruning is possible once the - * collective type is removed from the tree */ - comm_csel->u.pruned.coll_trees[node->u.collective.coll_type] = - prune_tree(node->success, comm_ptr); - node = node->failure; + int idx = csel_name_to_idx(name); + if (idx >= 0) { + return csel_get_tree_by_idx(idx); } - - *comm_csel_ = comm_csel; - return mpi_errno; + return NULL; } -static void free_tree(csel_node_s * node) +/* -- internal static routines -- */ + +static void free_tree(MPIR_Csel_node_s * node) { if (node->type == CSEL_NODE_TYPE__CONTAINER) { - MPL_free(node->u.cnt.container); + MPL_free(node->u.container); MPL_free(node); } else { if (node->success) @@ -732,633 +310,102 @@ static void free_tree(csel_node_s * node) } } -int MPIR_Csel_free(void *csel_) -{ - int mpi_errno = MPI_SUCCESS; - csel_s *csel = (csel_s *) csel_; - - if (csel->type == CSEL_TYPE__ROOT && csel->u.root.tree) - free_tree(csel->u.root.tree); - - MPL_free(csel); - return mpi_errno; -} - -static inline bool is_sendbuf_inplace(MPIR_Csel_coll_sig_s coll_info) +void MPIR_Csel_print_tree(MPIR_Csel_node_s * node, int level) { - bool sendbuf_inplace = false; - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__ALLTOALL: - sendbuf_inplace = (coll_info.u.alltoall.sendbuf == MPI_IN_PLACE); - break; - case MPIR_CSEL_COLL_TYPE__IALLTOALL: - sendbuf_inplace = (coll_info.u.ialltoall.sendbuf == MPI_IN_PLACE); - break; - case MPIR_CSEL_COLL_TYPE__ALLTOALLV: - sendbuf_inplace = (coll_info.u.alltoallv.sendbuf == MPI_IN_PLACE); - break; - case MPIR_CSEL_COLL_TYPE__IALLTOALLV: - sendbuf_inplace = (coll_info.u.ialltoallv.sendbuf == MPI_IN_PLACE); - break; - case MPIR_CSEL_COLL_TYPE__ALLTOALLW: - sendbuf_inplace = (coll_info.u.alltoallw.sendbuf == MPI_IN_PLACE); - break; - case MPIR_CSEL_COLL_TYPE__IALLTOALLW: - sendbuf_inplace = (coll_info.u.ialltoallw.sendbuf == MPI_IN_PLACE); - break; - default: - fprintf(stderr, "is_sendbuf_inplace not defined for coll_type %d\n", - coll_info.coll_type); - MPIR_Assert(0); - break; + for (int i = 0; i < level; i++) { + printf(" "); } - return sendbuf_inplace; -} - -static inline bool is_commutative(MPIR_Csel_coll_sig_s coll_info) -{ - bool commutative = false; - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__ALLREDUCE: - commutative = MPIR_Op_is_commutative(coll_info.u.allreduce.op); - break; - case MPIR_CSEL_COLL_TYPE__IALLREDUCE: - commutative = MPIR_Op_is_commutative(coll_info.u.iallreduce.op); - break; - case MPIR_CSEL_COLL_TYPE__REDUCE: - commutative = MPIR_Op_is_commutative(coll_info.u.reduce.op); - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE: - commutative = MPIR_Op_is_commutative(coll_info.u.ireduce.op); - break; - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER: - commutative = MPIR_Op_is_commutative(coll_info.u.reduce_scatter.op); - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER: - commutative = MPIR_Op_is_commutative(coll_info.u.ireduce_scatter.op); - break; - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER_BLOCK: - commutative = MPIR_Op_is_commutative(coll_info.u.reduce_scatter_block.op); - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER_BLOCK: - commutative = MPIR_Op_is_commutative(coll_info.u.ireduce_scatter_block.op); - break; - default: - fprintf(stderr, "is_commutative not defined for coll_type %d\n", coll_info.coll_type); - MPIR_Assert(0); - break; + if (node->type == CSEL_NODE_TYPE__CONTAINER) { + printf("Algorithm: %s\n", MPIR_Coll_algo_names[node->u.container->id]); + } else if (node->type == CSEL_NODE_TYPE__OPERATOR__COLLECTIVE) { + printf("Collective: %s\n", MPIR_Coll_type_names[node->u.collective.coll_type]); + } else if (node->type == CSEL_NODE_TYPE__OPERATOR__ANY) { + printf("ANY\n"); + } else { + if (!node->u.condition.negate) { + if (!node->u.condition.thresh) { + printf("condition: %s\n", MPIR_Csel_condition_names[node->type]); + } else { + printf("condition: %s(%d)\n", MPIR_Csel_condition_names[node->type], + node->u.condition.thresh); + } + } else { + if (!node->u.condition.thresh) { + printf("condition: !%s\n", MPIR_Csel_condition_names[node->type]); + } else { + printf("condition: !%s(%d)\n", MPIR_Csel_condition_names[node->type], + node->u.condition.thresh); + } + } } - return commutative; -} -static inline bool is_op_built_in(MPIR_Csel_coll_sig_s coll_info) -{ - bool op_built_in = false; - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__ALLREDUCE: - op_built_in = HANDLE_GET_KIND(coll_info.u.allreduce.op) == HANDLE_KIND_BUILTIN; - break; - case MPIR_CSEL_COLL_TYPE__IALLREDUCE: - op_built_in = HANDLE_GET_KIND(coll_info.u.iallreduce.op) == HANDLE_KIND_BUILTIN; - break; - case MPIR_CSEL_COLL_TYPE__REDUCE: - op_built_in = HANDLE_GET_KIND(coll_info.u.reduce.op) == HANDLE_KIND_BUILTIN; - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE: - op_built_in = HANDLE_GET_KIND(coll_info.u.ireduce.op) == HANDLE_KIND_BUILTIN; - break; - default: - fprintf(stderr, "is_op_builtin not defined for coll_type %d\n", coll_info.coll_type); - MPIR_Assert(0); - break; + if (node->success) { + MPIR_Csel_print_tree(node->success, level + 1); + } + if (node->failure) { + MPIR_Csel_print_tree(node->failure, level + 1); } - return op_built_in; } -static inline bool is_block_regular(MPIR_Csel_coll_sig_s coll_info) +static int csel_name_to_idx(const char *name) { - bool is_regular = true; - int i = 0; - const MPI_Aint *recvcounts = NULL; - - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER: - recvcounts = coll_info.u.reduce_scatter.recvcounts; - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER: - recvcounts = coll_info.u.ireduce_scatter.recvcounts; - break; - default: - MPIR_Assert(0); - break; - } - for (i = 0; i < (coll_info.comm_ptr->local_size - 1); ++i) { - if (recvcounts[i] != recvcounts[i + 1]) { - is_regular = false; - break; - } + struct csel_subtree_hash_item *h; + HASH_FIND_STR(csel_subtree_hash, name, h); + if (h) { + return h->idx; + } else { + return -1; } - return is_regular; } -static inline MPI_Aint get_avg_msgsize(MPIR_Csel_coll_sig_s coll_info) +static MPIR_Csel_node_s *csel_get_tree_by_idx(int idx) { - MPI_Aint msgsize = 0; - - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__ALLREDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.allreduce.datatype, msgsize); - msgsize *= coll_info.u.allreduce.count; - break; - - case MPIR_CSEL_COLL_TYPE__IALLREDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.iallreduce.datatype, msgsize); - msgsize *= coll_info.u.iallreduce.count; - break; - - case MPIR_CSEL_COLL_TYPE__BCAST: - MPIR_Datatype_get_size_macro(coll_info.u.bcast.datatype, msgsize); - msgsize *= coll_info.u.bcast.count; - break; - - case MPIR_CSEL_COLL_TYPE__IBCAST: - MPIR_Datatype_get_size_macro(coll_info.u.ibcast.datatype, msgsize); - msgsize *= coll_info.u.ibcast.count; - break; - - case MPIR_CSEL_COLL_TYPE__REDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.reduce.datatype, msgsize); - msgsize *= coll_info.u.reduce.count; - break; - - case MPIR_CSEL_COLL_TYPE__IREDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.ireduce.datatype, msgsize); - msgsize *= coll_info.u.ireduce.count; - break; - - case MPIR_CSEL_COLL_TYPE__ALLTOALL: - MPIR_Datatype_get_size_macro(coll_info.u.alltoall.sendtype, msgsize); - msgsize *= coll_info.u.alltoall.sendcount; - break; - - case MPIR_CSEL_COLL_TYPE__IALLTOALL: - MPIR_Datatype_get_size_macro(coll_info.u.ialltoall.sendtype, msgsize); - msgsize *= coll_info.u.ialltoall.sendcount; - break; - - default: - fprintf(stderr, "avg_msg_size not defined for coll_type %d\n", coll_info.coll_type); - MPIR_Assert(0); - break; - } - - return msgsize; + struct csel_subtree *p = (void *) utarray_eltptr(csel_subtrees, idx); + MPIR_Assert(p); + MPIR_Assert(p->node); + return p->node; } -static inline MPI_Aint get_count(MPIR_Csel_coll_sig_s coll_info) +static void add_named_tree(const char *name, struct json_object *obj) { - MPI_Aint count = 0; - int i = 0; - int comm_size = coll_info.comm_ptr->local_size; - - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__BCAST: - count = coll_info.u.bcast.count; - break; - case MPIR_CSEL_COLL_TYPE__ALLREDUCE: - count = coll_info.u.allreduce.count; - break; - case MPIR_CSEL_COLL_TYPE__IALLREDUCE: - count = coll_info.u.iallreduce.count; - break; - case MPIR_CSEL_COLL_TYPE__REDUCE: - count = coll_info.u.reduce.count; - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE: - count = coll_info.u.ireduce.count; - break; - case MPIR_CSEL_COLL_TYPE__ALLGATHER: - count = coll_info.u.allgather.recvcount; - break; - case MPIR_CSEL_COLL_TYPE__ALLGATHERV: - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.allgatherv.recvcounts[i]; - break; - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER: - for (i = 0; i < comm_size; i++) - count += coll_info.u.reduce_scatter.recvcounts[i]; - break; - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER_BLOCK: - count = coll_info.u.reduce_scatter_block.recvcount; - break; - case MPIR_CSEL_COLL_TYPE__IALLGATHER: - count = coll_info.u.iallgather.recvcount; - break; - case MPIR_CSEL_COLL_TYPE__IALLGATHERV: - for (i = 0; i < comm_size; i++) - count += coll_info.u.iallgatherv.recvcounts[i]; - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER: - for (i = 0; i < comm_size; i++) - count += coll_info.u.ireduce_scatter.recvcounts[i]; - break; - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER_BLOCK: - count = coll_info.u.ireduce_scatter_block.recvcount; - break; - default: - MPIR_Assert(0); - break; + struct csel_subtree item = { NULL, obj }; + utarray_push_back(csel_subtrees, &item, MPL_MEM_COLL); + + /* map the name to idx */ + struct csel_subtree_hash_item *h; + h = MPL_malloc(sizeof(*h), MPL_MEM_OTHER); + strncpy(h->name, name, CSEL_MAX_NAME - 1); + h->idx = utarray_len(csel_subtrees) - 1; + HASH_ADD_STR(csel_subtree_hash, name, h, MPL_MEM_COLL); + + /* update csel_main_idx if name == "main" */ + if (strcmp(name, "main") == 0) { + csel_main_idx = h->idx; } - return count; } -static inline MPI_Aint get_total_msgsize(MPIR_Csel_coll_sig_s coll_info) +static void replace_named_tree(int idx, struct json_object *obj) { - MPI_Aint total_bytes = 0, i = 0, count = 0, typesize = 0; - int comm_size = coll_info.comm_ptr->local_size; - - switch (coll_info.coll_type) { - case MPIR_CSEL_COLL_TYPE__ALLREDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.allreduce.datatype, total_bytes); - total_bytes *= coll_info.u.allreduce.count * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__BCAST: - MPIR_Datatype_get_size_macro(coll_info.u.bcast.datatype, total_bytes); - total_bytes *= coll_info.u.bcast.count * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__REDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.reduce.datatype, total_bytes); - total_bytes *= coll_info.u.reduce.count * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__ALLTOALL: - MPIR_Datatype_get_size_macro(coll_info.u.alltoall.sendtype, total_bytes); - total_bytes *= coll_info.u.alltoall.sendcount * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__ALLTOALLV: - MPIR_Datatype_get_size_macro(coll_info.u.alltoallv.sendtype, total_bytes); - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.alltoallv.sendcounts[i]; - total_bytes *= count; - break; - - case MPIR_CSEL_COLL_TYPE__ALLTOALLW: - count = 0; - typesize = 0; - for (i = 0; i < comm_size; i++) { - MPIR_Datatype_get_size_macro(coll_info.u.alltoallw.sendtypes[i], typesize); - count = coll_info.u.alltoallw.sendcounts[i]; - total_bytes += (count * typesize); - } - break; - - case MPIR_CSEL_COLL_TYPE__ALLGATHER: - MPIR_Datatype_get_size_macro(coll_info.u.allgather.recvtype, total_bytes); - total_bytes *= coll_info.u.allgather.recvcount * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__ALLGATHERV: - MPIR_Datatype_get_size_macro(coll_info.u.allgatherv.recvtype, total_bytes); - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.allgatherv.recvcounts[i]; - total_bytes *= count; - break; - - case MPIR_CSEL_COLL_TYPE__GATHER: - if (coll_info.u.gather.root == MPI_ROOT) { - MPIR_Datatype_get_size_macro(coll_info.u.gather.recvtype, total_bytes); - /* use remote size for intercomm root */ - if (coll_info.comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) - total_bytes = coll_info.u.gather.recvcount * (coll_info.comm_ptr->remote_size); - else - total_bytes = coll_info.u.gather.recvcount * comm_size; - } else { - MPIR_Datatype_get_size_macro(coll_info.u.gather.sendtype, total_bytes); - total_bytes = coll_info.u.gather.sendcount * comm_size; - } - break; - - case MPIR_CSEL_COLL_TYPE__SCATTER: - if (coll_info.u.scatter.root == MPI_ROOT) { - MPIR_Datatype_get_size_macro(coll_info.u.scatter.sendtype, total_bytes); - /* use remote size for intercomm root */ - if (coll_info.comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) - total_bytes = coll_info.u.scatter.sendcount * (coll_info.comm_ptr->remote_size); - else - total_bytes = coll_info.u.scatter.sendcount * comm_size; - } else { - MPIR_Datatype_get_size_macro(coll_info.u.scatter.recvtype, total_bytes); - total_bytes = coll_info.u.scatter.recvcount * comm_size; - } - break; - - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER: - MPIR_Datatype_get_size_macro(coll_info.u.reduce_scatter.datatype, total_bytes); - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.reduce_scatter.recvcounts[i]; - total_bytes *= count; - break; - - case MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER_BLOCK: - MPIR_Datatype_get_size_macro(coll_info.u.reduce_scatter_block.datatype, total_bytes); - total_bytes *= coll_info.u.reduce_scatter_block.recvcount * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IALLREDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.iallreduce.datatype, total_bytes); - total_bytes *= coll_info.u.iallreduce.count * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IBCAST: - MPIR_Datatype_get_size_macro(coll_info.u.ibcast.datatype, total_bytes); - total_bytes *= coll_info.u.ibcast.count * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IREDUCE: - MPIR_Datatype_get_size_macro(coll_info.u.ireduce.datatype, total_bytes); - total_bytes *= coll_info.u.ireduce.count * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IALLTOALL: - MPIR_Datatype_get_size_macro(coll_info.u.ialltoall.sendtype, total_bytes); - total_bytes *= coll_info.u.ialltoall.sendcount * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IALLTOALLV: - MPIR_Datatype_get_size_macro(coll_info.u.ialltoallv.sendtype, total_bytes); - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.ialltoallv.sendcounts[i]; - total_bytes *= count; - break; - - case MPIR_CSEL_COLL_TYPE__IALLTOALLW: - count = 0; - typesize = 0; - for (i = 0; i < comm_size; i++) { - MPIR_Datatype_get_size_macro(coll_info.u.ialltoallw.sendtypes[i], typesize); - count = coll_info.u.ialltoallw.sendcounts[i]; - total_bytes += (count * typesize); - } - break; - - case MPIR_CSEL_COLL_TYPE__IALLGATHER: - MPIR_Datatype_get_size_macro(coll_info.u.iallgather.recvtype, total_bytes); - total_bytes *= coll_info.u.iallgather.recvcount * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IALLGATHERV: - MPIR_Datatype_get_size_macro(coll_info.u.iallgatherv.recvtype, total_bytes); - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.iallgatherv.recvcounts[i]; - total_bytes *= count; - break; - - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER: - MPIR_Datatype_get_size_macro(coll_info.u.ireduce_scatter.datatype, total_bytes); - count = 0; - for (i = 0; i < comm_size; i++) - count += coll_info.u.ireduce_scatter.recvcounts[i]; - total_bytes *= count; - break; - - case MPIR_CSEL_COLL_TYPE__IREDUCE_SCATTER_BLOCK: - MPIR_Datatype_get_size_macro(coll_info.u.ireduce_scatter_block.datatype, total_bytes); - total_bytes = coll_info.u.ireduce_scatter_block.recvcount * comm_size; - break; - - case MPIR_CSEL_COLL_TYPE__IGATHER: - if (coll_info.u.igather.root == MPI_ROOT) { - MPIR_Datatype_get_size_macro(coll_info.u.igather.recvtype, total_bytes); - /* use remote size for intercomm root */ - if (coll_info.comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) - total_bytes = coll_info.u.igather.recvcount * (coll_info.comm_ptr->remote_size); - else - total_bytes = coll_info.u.igather.recvcount * comm_size; - } else { - MPIR_Datatype_get_size_macro(coll_info.u.igather.sendtype, total_bytes); - total_bytes = coll_info.u.igather.sendcount * comm_size; - } - break; - - case MPIR_CSEL_COLL_TYPE__ISCATTER: - if (coll_info.u.iscatter.root == MPI_ROOT) { - MPIR_Datatype_get_size_macro(coll_info.u.iscatter.sendtype, total_bytes); - /* use remote size for intercomm root */ - if (coll_info.comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) - total_bytes = - coll_info.u.iscatter.sendcount * (coll_info.comm_ptr->remote_size); - else - total_bytes = coll_info.u.iscatter.sendcount * comm_size; - } else { - MPIR_Datatype_get_size_macro(coll_info.u.iscatter.recvtype, total_bytes); - total_bytes = coll_info.u.iscatter.recvcount * comm_size; - } - break; - - default: - MPIR_Assert(0); - break; - } - - return total_bytes; + struct csel_subtree *p = (void *) utarray_eltptr(csel_subtrees, idx); + MPIR_Assert(p); + p->json_obj = obj; + free_tree(p->node); + p->node = NULL; } -void *MPIR_Csel_search(void *csel_, MPIR_Csel_coll_sig_s coll_info) +static int parse_named_trees(void) { - csel_s *csel = (csel_s *) csel_; - csel_node_s *node = NULL; - MPIR_Comm *comm_ptr = coll_info.comm_ptr; - - MPIR_Assert(csel_); - - csel_node_s *root; - if (csel->type == CSEL_TYPE__ROOT) - root = csel->u.root.tree; - else - root = csel->u.pruned.coll_trees[coll_info.coll_type]; - - for (node = root; node;) { - switch (node->type) { - case CSEL_NODE_TYPE__OPERATOR__IS_MULTI_THREADED: - if (MPIR_IS_THREADED == node->u.is_multi_threaded.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTRA: - if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_TYPE_INTER: - if (comm_ptr->comm_kind == MPIR_COMM_KIND__INTERCOMM) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LE: - if (comm_ptr->local_size <= node->u.comm_size_le.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_LT: - if (comm_ptr->local_size <= node->u.comm_size_lt.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_NODE_COMM_SIZE: - if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && comm_ptr->num_external == 1) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_SIZE_POW2: - if (!(comm_ptr->local_size & (comm_ptr->local_size - 1))) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COLLECTIVE: - if (node->u.collective.coll_type == coll_info.coll_type) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LE: - if (get_avg_msgsize(coll_info) <= node->u.avg_msg_size_le.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__AVG_MSG_SIZE_LT: - if (get_avg_msgsize(coll_info) < node->u.avg_msg_size_lt.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LE: - if (get_total_msgsize(coll_info) <= node->u.total_msg_size_le.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__TOTAL_MSG_SIZE_LT: - if (get_total_msgsize(coll_info) < node->u.total_msg_size_lt.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COUNT_LE: - if (get_count(coll_info) <= node->u.count_le.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COUNT_LT_POW2: - if (get_count(coll_info) < MPL_pof2(coll_info.comm_ptr->local_size)) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__IS_COMMUTATIVE: - if (is_commutative(coll_info) == node->u.is_commutative.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__IS_SBUF_INPLACE: - if (is_sendbuf_inplace(coll_info) == node->u.is_sbuf_inplace.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__IS_OP_BUILT_IN: - if (is_op_built_in(coll_info) == node->u.is_op_built_in.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__IS_BLOCK_REGULAR: - if (is_block_regular(coll_info) == node->u.is_block_regular.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__IS_NODE_CONSECUTIVE: - if (MPII_Comm_is_node_consecutive(coll_info.comm_ptr) == - node->u.is_node_consecutive.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LE: - if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && - comm_ptr->local_size <= node->u.comm_avg_ppn_le.val * comm_ptr->num_external) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_AVG_PPN_LT: - if ((comm_ptr->attr & MPIR_COMM_ATTR__HIERARCHY) && - comm_ptr->local_size < node->u.comm_avg_ppn_le.val * comm_ptr->num_external) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__COMM_HIERARCHY: - if (MPIR_Comm_is_parent_comm(comm_ptr) == node->u.comm_hierarchy.val) - node = node->success; - else - node = node->failure; - break; - - case CSEL_NODE_TYPE__OPERATOR__ANY: - node = node->success; - break; - - case CSEL_NODE_TYPE__CONTAINER: - goto fn_exit; + int mpi_errno = MPI_SUCCESS; - default: - MPIR_Assert(0); + struct csel_subtree *p; + for (p = (struct csel_subtree *) utarray_front(csel_subtrees); p; + p = (struct csel_subtree *) utarray_next(csel_subtrees, p)) { + if (!p->node && p->json_obj) { + p->node = parse_json_tree(p->json_obj); + MPIR_Assert(p->node); + json_object_put(p->json_obj); + p->json_obj = NULL; } } - - MPIR_Assert(node == NULL); - - fn_exit: - return node ? node->u.cnt.container : NULL; + return mpi_errno; } diff --git a/src/mpi/coll/src/csel_container.c b/src/mpi/coll/src/csel_container.c deleted file mode 100644 index 5b9c0aeb739..00000000000 --- a/src/mpi/coll/src/csel_container.c +++ /dev/null @@ -1,856 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#include "mpiimpl.h" -#include "coll_impl.h" -#include "csel_container.h" -#include "mpl.h" - -static void parse_container_params(struct json_object *obj, MPII_Csel_container_s * cnt) -{ - MPIR_Assert(obj != NULL); - char *ckey; - - switch (cnt->id) { - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_tree: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.ibcast.intra_tsp_tree.chunk_size = - atoi(ckey + strlen("chunk_size=")); - else if (!strncmp(ckey, "tree_type=", strlen("tree_type="))) - cnt->u.ibcast.intra_tsp_tree.tree_type = - get_tree_type_from_string(ckey + strlen("tree_type=")); - else if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.ibcast.intra_tsp_tree.k = atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_ring: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.ibcast.intra_tsp_ring.chunk_size = - atoi(ckey + strlen("chunk_size=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_tree: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "tree_type=", strlen("tree_type="))) - cnt->u.bcast.intra_tree.tree_type = - get_tree_type_from_string(ckey + strlen("tree_type=")); - else if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.bcast.intra_tree.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "is_non_blocking=", strlen("is_non_blocking="))) - cnt->u.bcast.intra_tree.is_non_blocking = - atoi(ckey + strlen("is_non_blocking=")); - else if (!strncmp(ckey, "topo_overhead=", strlen("topo_overhead="))) - cnt->u.bcast.intra_tree.topo_overhead = - atoi(ckey + strlen("topo_overhead=")); - else if (!strncmp(ckey, "topo_diff_groups=", strlen("topo_diff_groups="))) - cnt->u.bcast.intra_tree.topo_diff_groups = - atoi(ckey + strlen("topo_diff_groups=")); - else if (!strncmp(ckey, "topo_diff_switches=", strlen("topo_diff_switches="))) - cnt->u.bcast.intra_tree.topo_diff_switches = - atoi(ckey + strlen("topo_diff_switches=")); - else if (!strncmp(ckey, "topo_same_switches=", strlen("topo_same_switches="))) - cnt->u.bcast.intra_tree.topo_same_switches = - atoi(ckey + strlen("topo_same_switches=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_pipelined_tree: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "tree_type=", strlen("tree_type="))) - cnt->u.bcast.intra_pipelined_tree.tree_type = - get_tree_type_from_string(ckey + strlen("tree_type=")); - else if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.bcast.intra_pipelined_tree.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "is_non_blocking=", strlen("is_non_blocking="))) - cnt->u.bcast.intra_pipelined_tree.is_non_blocking = - atoi(ckey + strlen("is_non_blocking=")); - else if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.bcast.intra_pipelined_tree.chunk_size = - atoi(ckey + strlen("chunk_size=")); - else if (!strncmp(ckey, "recv_pre_posted=", strlen("recv_pre_posted="))) - cnt->u.bcast.intra_pipelined_tree.recv_pre_posted = - atoi(ckey + strlen("recv_pre_posted=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_tree: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "buffer_per_child=", strlen("buffer_per_child="))) - cnt->u.ireduce.intra_tsp_tree.buffer_per_child = - atoi(ckey + strlen("buffer_per_child=")); - else if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.ireduce.intra_tsp_tree.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "tree_type=", strlen("tree_type="))) - cnt->u.ireduce.intra_tsp_tree.tree_type = - get_tree_type_from_string(ckey + strlen("tree_type=")); - else if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.ireduce.intra_tsp_tree.chunk_size = - atoi(ckey + strlen("chunk_size=")); - else if (!strncmp(ckey, "topo_overhead=", strlen("topo_overhead="))) - cnt->u.ireduce.intra_tsp_tree.topo_overhead = - atoi(ckey + strlen("topo_overhead=")); - else if (!strncmp(ckey, "topo_diff_groups=", strlen("topo_diff_groups="))) - cnt->u.ireduce.intra_tsp_tree.topo_diff_groups = - atoi(ckey + strlen("topo_diff_groups=")); - else if (!strncmp(ckey, "topo_diff_switches=", strlen("topo_diff_switches="))) - cnt->u.ireduce.intra_tsp_tree.topo_diff_switches = - atoi(ckey + strlen("topo_diff_switches=")); - else if (!strncmp(ckey, "topo_same_switches=", strlen("topo_same_switches="))) - cnt->u.ireduce.intra_tsp_tree.topo_same_switches = - atoi(ckey + strlen("topo_same_switches=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_ring: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "buffer_per_child=", strlen("buffer_per_child="))) - cnt->u.ireduce.intra_tsp_ring.buffer_per_child = - atoi(ckey + strlen("buffer_per_child=")); - else if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.ireduce.intra_tsp_tree.chunk_size = - atoi(ckey + strlen("chunk_size=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_tree: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "buffer_per_child=", strlen("buffer_per_child="))) - cnt->u.iallreduce.intra_tsp_tree.buffer_per_child = - atoi(ckey + strlen("buffer_per_child=")); - else if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.iallreduce.intra_tsp_tree.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "tree_type=", strlen("tree_type="))) - cnt->u.iallreduce.intra_tsp_tree.tree_type = - get_tree_type_from_string(ckey + strlen("tree_type=")); - else if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.iallreduce.intra_tsp_tree.chunk_size = - atoi(ckey + strlen("chunk_size=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recursive_multiplying: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allreduce.intra_recursive_multiplying.k = atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_tree: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "buffer_per_child=", strlen("buffer_per_child="))) - cnt->u.allreduce.intra_tree.buffer_per_child = - atoi(ckey + strlen("buffer_per_child=")); - else if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allreduce.intra_tree.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "tree_type=", strlen("tree_type="))) - cnt->u.allreduce.intra_tree.tree_type = - get_tree_type_from_string(ckey + strlen("tree_type=")); - else if (!strncmp(ckey, "chunk_size=", strlen("chunk_size="))) - cnt->u.allreduce.intra_tree.chunk_size = atoi(ckey + strlen("chunk_size=")); - else if (!strncmp(ckey, "topo_overhead=", strlen("topo_overhead="))) - cnt->u.allreduce.intra_tree.topo_overhead = - atoi(ckey + strlen("topo_overhead=")); - else if (!strncmp(ckey, "topo_diff_groups=", strlen("topo_diff_groups="))) - cnt->u.allreduce.intra_tree.topo_diff_groups = - atoi(ckey + strlen("topo_diff_groups=")); - else if (!strncmp(ckey, "topo_diff_switches=", strlen("topo_diff_switches="))) - cnt->u.allreduce.intra_tree.topo_diff_switches = - atoi(ckey + strlen("topo_diff_switches=")); - else if (!strncmp(ckey, "topo_same_switches=", strlen("topo_same_switches="))) - cnt->u.allreduce.intra_tree.topo_same_switches = - atoi(ckey + strlen("topo_same_switches=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recexch: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allreduce.intra_recexch.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "single_phase_recv=", strlen("single_phase_recv="))) - cnt->u.allreduce.intra_recexch.single_phase_recv = - atoi(ckey + strlen("single_phase_recv=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_k_reduce_scatter_allgather: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allreduce.intra_k_reduce_scatter_allgather.k = - atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "single_phase_recv=", strlen("single_phase_recv="))) - cnt->u.allreduce.intra_k_reduce_scatter_allgather.single_phase_recv = - atoi(ckey + strlen("single_phase_recv=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_ccl: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "ccl=", strlen("ccl="))) - cnt->u.allreduce.intra_ccl.ccl = get_ccl_from_string(ckey + strlen("ccl=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_scatterv_recexch_allgatherv: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "scatterv_k=", strlen("scatterv_k="))) - cnt->u.ibcast.intra_tsp_scatterv_recexch_allgatherv.scatterv_k = - atoi(ckey + strlen("scatterv_k=")); - else if (!strncmp(ckey, "allgatherv_k=", strlen("allgatherv_k="))) - cnt->u.ibcast.intra_tsp_scatterv_recexch_allgatherv.allgatherv_k = - atoi(ckey + strlen("allgatherv_k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_single_buffer: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.iallreduce.intra_tsp_recexch_single_buffer.k = - atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_multiple_buffer: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.iallreduce.intra_tsp_recexch_multiple_buffer.k = - atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_reduce_scatter_recexch_allgatherv: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.iallreduce.intra_tsp_recexch_reduce_scatter_recexch_allgatherv.k = - atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_k_brucks: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allgather.intra_k_brucks.k = atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recexch_doubling: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allgather.intra_recexch_doubling.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "single_phase_recv=", strlen("single_phase_recv="))) - cnt->u.allgather.intra_recexch_doubling.single_phase_recv = - atoi(ckey + strlen("single_phase_recv=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recexch_halving: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.allgather.intra_recexch_halving.k = atoi(ckey + strlen("k=")); - else if (!strncmp(ckey, "single_phase_recv=", strlen("single_phase_recv="))) - cnt->u.allgather.intra_recexch_halving.single_phase_recv = - atoi(ckey + strlen("single_phase_recv=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_k_brucks: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.alltoall.intra_k_brucks.k = atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_k_dissemination: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.barrier.intra_k_dissemination.k = atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - case MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_recexch: - { - json_object_object_foreach(obj, key, val) { - ckey = MPL_strdup_no_spaces(key); - if (!strncmp(ckey, "k=", strlen("k="))) - cnt->u.barrier.intra_recexch.k = atoi(ckey + strlen("k=")); - MPL_free(ckey); - } - } - break; - - default: - /* Algorithm does not have parameters */ - break; - } -} - -void *MPII_Create_container(struct json_object *obj) -{ - MPII_Csel_container_s *cnt = MPL_malloc(sizeof(MPII_Csel_container_s), MPL_MEM_COLL); - - json_object_object_foreach(obj, key, val) { - char *ckey = MPL_strdup_no_spaces(key); - - if (!strcmp(ckey, "algorithm=MPIR_Allgather_intra_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_intra_k_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_k_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_intra_recursive_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_intra_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_intra_recexch_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recexch_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_intra_recexch_halving")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_intra_recexch_halving; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_inter_local_gather_remote_bcast")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_inter_local_gather_remote_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Allgather_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgather_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Allgatherv_intra_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_intra_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Allgatherv_intra_recursive_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Allgatherv_intra_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_intra_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Allgatherv_inter_remote_gather_local_bcast")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_inter_remote_gather_local_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Allgatherv_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allgatherv_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_recursive_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_recursive_multiplying")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recursive_multiplying; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_reduce_scatter_allgather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_reduce_scatter_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_recexch")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_recexch; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_k_reduce_scatter_allgather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_k_reduce_scatter_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_intra_ccl")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_intra_ccl; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_inter_reduce_exchange_bcast")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_inter_reduce_exchange_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Allreduce_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_intra_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_intra_k_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_k_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_intra_pairwise")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_pairwise; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_intra_pairwise_sendrecv_replace")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_pairwise_sendrecv_replace; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_intra_scattered")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_intra_scattered; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_inter_pairwise_exchange")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_inter_pairwise_exchange; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoall_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoall_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallv_intra_pairwise_sendrecv_replace")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_intra_pairwise_sendrecv_replace; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallv_intra_scattered")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_intra_scattered; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallv_inter_pairwise_exchange")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_inter_pairwise_exchange; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallv_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallv_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallw_intra_pairwise_sendrecv_replace")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_intra_pairwise_sendrecv_replace; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallw_intra_scattered")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_intra_scattered; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallw_inter_pairwise_exchange")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_inter_pairwise_exchange; - else if (!strcmp(ckey, "algorithm=MPIR_Alltoallw_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Alltoallw_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Barrier_intra_k_dissemination")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_k_dissemination; - else if (!strcmp(ckey, "algorithm=MPIR_Barrier_intra_recexch")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_recexch; - else if (!strcmp(ckey, "algorithm=MPIR_Barrier_intra_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_intra_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Barrier_inter_bcast")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_inter_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Barrier_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_scatter_recursive_doubling_allgather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_recursive_doubling_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_scatter_ring_allgather")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_scatter_ring_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_intra_pipelined_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_intra_pipelined_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_inter_remote_send_local_bcast")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_inter_remote_send_local_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Bcast_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Exscan_intra_recursive_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Exscan_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Exscan_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Exscan_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Gather_intra_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_intra_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Gather_inter_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_inter_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Gather_inter_local_gather_remote_send")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_inter_local_gather_remote_send; - else if (!strcmp(ckey, "algorithm=MPIR_Gather_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gather_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Gatherv_allcomm_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gatherv_allcomm_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Gatherv_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Gatherv_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_tsp_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_sched_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_sched_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_sched_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_sched_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_tsp_recexch_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_recexch_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_tsp_recexch_halving")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_recexch_halving; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_intra_tsp_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_intra_tsp_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgather_inter_sched_local_gather_remote_bcast")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgather_inter_sched_local_gather_remote_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_tsp_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_sched_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_sched_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_sched_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_sched_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_tsp_recexch_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_recexch_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_tsp_recexch_halving")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_recexch_halving; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_intra_tsp_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_intra_tsp_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Iallgatherv_inter_sched_remote_gather_local_bcast")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallgatherv_inter_sched_remote_gather_local_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_sched_naive")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_naive; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_sched_reduce_scatter_allgather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_reduce_scatter_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_tsp_recexch_single_buffer")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_single_buffer; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_tsp_recexch_multiple_buffer")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_multiple_buffer; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_tsp_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_tsp_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_ring; - else if (!strcmp - (ckey, - "algorithm=MPIR_Iallreduce_intra_tsp_recexch_reduce_scatter_recexch_allgatherv")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_tsp_recexch_reduce_scatter_recexch_allgatherv; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_intra_sched_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_intra_sched_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Iallreduce_inter_sched_remote_reduce_local_bcast")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iallreduce_inter_sched_remote_reduce_local_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_tsp_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_tsp_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_tsp_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_tsp_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_tsp_scattered")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_tsp_scattered; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_sched_brucks")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_brucks; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_sched_inplace")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_inplace; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_sched_pairwise")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_pairwise; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_intra_sched_permuted_sendrecv")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_intra_sched_permuted_sendrecv; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoall_inter_sched_pairwise_exchange")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoall_inter_sched_pairwise_exchange; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallv_intra_sched_blocked")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_sched_blocked; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallv_intra_sched_inplace")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_sched_inplace; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallv_intra_tsp_scattered")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_tsp_scattered; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallv_intra_tsp_blocked")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_tsp_blocked; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallv_intra_tsp_inplace")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_intra_tsp_inplace; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallv_inter_sched_pairwise_exchange")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallv_inter_sched_pairwise_exchange; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallw_intra_tsp_blocked")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_tsp_blocked; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallw_intra_tsp_inplace")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_tsp_inplace; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallw_intra_sched_blocked")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_sched_blocked; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallw_intra_sched_inplace")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_intra_sched_inplace; - else if (!strcmp(ckey, "algorithm=MPIR_Ialltoallw_inter_sched_pairwise_exchange")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ialltoallw_inter_sched_pairwise_exchange; - else if (!strcmp(ckey, "algorithm=MPIR_Ibarrier_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Ibarrier_intra_tsp_recexch")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_tsp_recexch; - else if (!strcmp(ckey, "algorithm=MPIR_Ibarrier_intra_tsp_k_dissem")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_intra_tsp_k_dissemination; - else if (!strcmp(ckey, "algorithm=MPIR_Ibarrier_inter_sched_bcast")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibarrier_inter_sched_bcast; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_intra_tsp_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_intra_tsp_scatterv_recexch_allgatherv")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_scatterv_recexch_allgatherv; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_intra_tsp_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_tsp_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_intra_sched_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_binomial; - else if (!strcmp - (ckey, "algorithm=MPIR_Ibcast_intra_sched_scatter_recursive_doubling_allgather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_scatter_recursive_doubling_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_intra_sched_scatter_ring_allgather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_scatter_ring_allgather; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_intra_sched_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_intra_sched_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Ibcast_inter_sched_flat")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ibcast_inter_sched_flat; - else if (!strcmp(ckey, "algorithm=MPIR_Iexscan_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iexscan_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Igather_intra_tsp_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_intra_tsp_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Igather_intra_sched_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_intra_sched_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Igather_inter_sched_long")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_inter_sched_long; - else if (!strcmp(ckey, "algorithm=MPIR_Igather_inter_sched_short")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igather_inter_sched_short; - else if (!strcmp(ckey, "algorithm=MPIR_Igatherv_allcomm_tsp_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igatherv_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Igatherv_allcomm_sched_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Igatherv_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_allgather_allcomm_tsp_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgather_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_allgather_allcomm_sched_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgather_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_allgatherv_allcomm_tsp_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgatherv_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_allgatherv_allcomm_sched_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_allgatherv_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_alltoall_allcomm_tsp_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoall_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_alltoall_allcomm_sched_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoall_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_alltoallv_allcomm_tsp_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallv_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_alltoallv_allcomm_sched_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallv_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_alltoallw_allcomm_tsp_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallw_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ineighbor_alltoallw_allcomm_sched_linear")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ineighbor_alltoallw_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_intra_tsp_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_intra_tsp_ring")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_tsp_ring; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_intra_sched_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_sched_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_intra_sched_reduce_scatter_gather")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_sched_reduce_scatter_gather; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_intra_sched_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_intra_sched_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_inter_sched_local_reduce_remote_send")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_inter_sched_local_reduce_remote_send; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_intra_sched_noncommutative")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_noncommutative; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_intra_sched_pairwise")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_pairwise; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_intra_sched_recursive_halving")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_sched_recursive_halving; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_intra_tsp_recexch")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_intra_tsp_recexch; - else if (!strcmp - (ckey, "algorithm=MPIR_Ireduce_scatter_inter_sched_remote_reduce_local_scatterv")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_inter_sched_remote_reduce_local_scatterv; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_block_intra_tsp_recexch")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_tsp_recexch; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_block_intra_sched_noncommutative")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_noncommutative; - else if (!strcmp(ckey, "algorithm=MPIR_Ireduce_scatter_block_intra_sched_pairwise")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_pairwise; - else if (!strcmp - (ckey, "algorithm=MPIR_Ireduce_scatter_block_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_recursive_doubling; - else if (!strcmp - (ckey, "algorithm=MPIR_Ireduce_scatter_block_intra_sched_recursive_halving")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_intra_sched_recursive_halving; - else if (!strcmp - (ckey, - "algorithm=MPIR_Ireduce_scatter_block_inter_sched_remote_reduce_local_scatterv")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Ireduce_scatter_block_inter_sched_remote_reduce_local_scatterv; - else if (!strcmp(ckey, "algorithm=MPIR_Iscan_intra_sched_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscan_intra_sched_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iscan_intra_sched_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscan_intra_sched_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Iscan_intra_tsp_recursive_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscan_intra_tsp_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Iscatter_intra_tsp_tree")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_intra_tsp_tree; - else if (!strcmp(ckey, "algorithm=MPIR_Iscatter_intra_sched_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_intra_sched_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Iscatter_inter_sched_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_inter_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Iscatter_inter_sched_remote_send_local_scatter")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatter_inter_sched_remote_send_local_scatter; - else if (!strcmp(ckey, "algorithm=MPIR_Iscatterv_allcomm_tsp_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatterv_allcomm_tsp_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Iscatterv_allcomm_sched_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Iscatterv_allcomm_sched_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Neighbor_allgather_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_allgather_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Neighbor_allgatherv_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_allgatherv_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Neighbor_alltoall_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_alltoall_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Neighbor_alltoallv_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_alltoallv_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Neighbor_alltoallw_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Neighbor_alltoallw_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_intra_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_intra_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_intra_reduce_scatter_gather")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_intra_reduce_scatter_gather; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_intra_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_intra_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_inter_local_reduce_remote_send")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_inter_local_reduce_remote_send; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_intra_noncommutative")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_noncommutative; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_intra_pairwise")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_pairwise; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_intra_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_intra_recursive_halving")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_intra_recursive_halving; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_inter_remote_reduce_local_scatter")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_inter_remote_reduce_local_scatter; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_block_intra_noncommutative")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_noncommutative; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_block_intra_pairwise")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_pairwise; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_block_intra_recursive_doubling")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_block_intra_recursive_halving")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_intra_recursive_halving; - else if (!strcmp - (ckey, "algorithm=MPIR_Reduce_scatter_block_inter_remote_reduce_local_scatter")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_inter_remote_reduce_local_scatter; - else if (!strcmp(ckey, "algorithm=MPIR_Reduce_scatter_block_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_scatter_block_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Scan_intra_recursive_doubling")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scan_intra_recursive_doubling; - else if (!strcmp(ckey, "algorithm=MPIR_Scan_intra_smp")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scan_intra_smp; - else if (!strcmp(ckey, "algorithm=MPIR_Scan_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scan_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Scatter_intra_binomial")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_intra_binomial; - else if (!strcmp(ckey, "algorithm=MPIR_Scatter_inter_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_inter_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Scatter_inter_remote_send_local_scatter")) - cnt->id = - MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_inter_remote_send_local_scatter; - else if (!strcmp(ckey, "algorithm=MPIR_Scatter_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatter_allcomm_nb; - else if (!strcmp(ckey, "algorithm=MPIR_Scatterv_allcomm_linear")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatterv_allcomm_linear; - else if (!strcmp(ckey, "algorithm=MPIR_Scatterv_allcomm_nb")) - cnt->id = MPII_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Scatterv_allcomm_nb; - else { - fprintf(stderr, "unrecognized key %s\n", key); - MPIR_Assert(0); - } - - MPL_free(ckey); - } - - /* process algorithm parameters */ - parse_container_params(json_object_object_get(obj, key), cnt); - - return (void *) cnt; -} diff --git a/src/mpi/comm/comm_impl.c b/src/mpi/comm/comm_impl.c index 6547317cf75..a94518b65e3 100644 --- a/src/mpi/comm/comm_impl.c +++ b/src/mpi/comm/comm_impl.c @@ -690,8 +690,8 @@ int MPIR_Intercomm_create_from_groups_impl(MPIR_Group * local_group_ptr, int loc /* synchronize mpi_errno */ int tmp_err = mpi_errno; - mpi_errno = MPIR_Bcast_impl(&tmp_err, 1, MPIR_INT_INTERNAL, local_leader, local_comm, - MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(&tmp_err, 1, MPIR_INT_INTERNAL, local_leader, local_comm, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); mpi_errno = tmp_err; MPIR_ERR_CHECK(mpi_errno); diff --git a/src/mpi/comm/comm_split.c b/src/mpi/comm/comm_split.c index 7f941e43ff5..33f3c9f229b 100644 --- a/src/mpi/comm/comm_split.c +++ b/src/mpi/comm/comm_split.c @@ -111,8 +111,9 @@ int MPIR_Comm_split_impl(MPIR_Comm * comm_ptr, int color, int key, MPIR_Comm ** local_comm_ptr = comm_ptr; } /* Gather information on the local group of processes */ - mpi_errno = MPIR_Allgather(MPI_IN_PLACE, 2, MPIR_INT_INTERNAL, table, 2, MPIR_INT_INTERNAL, - local_comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allgather_fallback(MPI_IN_PLACE, 2, MPIR_INT_INTERNAL, table, 2, MPIR_INT_INTERNAL, + local_comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* Step 2: How many processes have our same color? */ diff --git a/src/mpi/comm/comm_split_type_nbhd.c b/src/mpi/comm/comm_split_type_nbhd.c index 4a9375245d4..a8d2efdfe9a 100644 --- a/src/mpi/comm/comm_split_type_nbhd.c +++ b/src/mpi/comm/comm_split_type_nbhd.c @@ -275,8 +275,9 @@ static int network_split_by_minsize(MPIR_Comm * comm_ptr, int key, int subcomm_m } MPIR_Assert(num_processes_at_node != NULL); /* Send the count to processes */ - mpi_errno = MPIR_Allreduce(MPI_IN_PLACE, num_processes_at_node, num_nodes, - MPIR_INT_INTERNAL, MPI_SUM, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allreduce_fallback(MPI_IN_PLACE, num_processes_at_node, num_nodes, + MPIR_INT_INTERNAL, MPI_SUM, comm_ptr, + MPIR_COLL_ATTR_SYNC); if (topo_type == MPIR_NETTOPO_TYPE__FAT_TREE || topo_type == MPIR_NETTOPO_TYPE__CLOS_NETWORK) { @@ -375,8 +376,8 @@ static int network_split_by_minsize(MPIR_Comm * comm_ptr, int key, int subcomm_m tree_depth = MPIR_hwtopo_get_depth(obj_containing_cpuset); /* get min tree depth to all processes */ - MPIR_Allreduce(&tree_depth, &min_tree_depth, 1, MPIR_INT_INTERNAL, MPI_MIN, node_comm, - MPIR_COLL_ATTR_SYNC); + MPIR_Allreduce_fallback(&tree_depth, &min_tree_depth, 1, MPIR_INT_INTERNAL, MPI_MIN, + node_comm, MPIR_COLL_ATTR_SYNC); if (min_tree_depth) { int num_hwloc_objs_at_depth; diff --git a/src/mpi/comm/commutil.c b/src/mpi/comm/commutil.c index 33b579188c1..ebbf11f2c88 100644 --- a/src/mpi/comm/commutil.c +++ b/src/mpi/comm/commutil.c @@ -316,7 +316,6 @@ int MPII_Comm_init(MPIR_Comm * comm_p) MPIR_stream_comm_init(comm_p); comm_p->persistent_requests = NULL; - comm_p->csel_comm = NULL; /* mutex is only used in VCI granularity. But the overhead of * creation is low, so we always create it. */ @@ -780,8 +779,7 @@ static int init_comm_seq(MPIR_Comm * comm) /* Every rank need share the same seq from root. NOTE: it is possible for * different communicators to have the same seq. It is only used as an * opportunistic optimization */ - mpi_errno = MPIR_Bcast_allcomm_auto(&tmp, 1, MPIR_INT_INTERNAL, 0, comm, - MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(&tmp, 1, MPIR_INT_INTERNAL, 0, comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); comm->seq = tmp; diff --git a/src/mpi/comm/contextid.c b/src/mpi/comm/contextid.c index 66dd26e7678..eda23239ffa 100644 --- a/src/mpi/comm/contextid.c +++ b/src/mpi/comm/contextid.c @@ -455,9 +455,9 @@ int MPIR_Get_contextid_sparse_group(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr MPIR_INT_INTERNAL, MPI_BAND, comm_ptr, group_ptr, coll_tag, MPIR_COLL_ATTR_SYNC); } else { - mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, st.local_mask, MPIR_MAX_CONTEXT_MASK + 1, - MPIR_INT_INTERNAL, MPI_BAND, comm_ptr, - MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(MPI_IN_PLACE, st.local_mask, MPIR_MAX_CONTEXT_MASK + 1, + MPIR_INT_INTERNAL, MPI_BAND, comm_ptr, MPIR_COLL_ATTR_SYNC); } MPIR_ERR_CHECK(mpi_errno); @@ -555,8 +555,8 @@ int MPIR_Get_contextid_sparse_group(MPIR_Comm * comm_ptr, MPIR_Group * group_ptr MPI_MIN, comm_ptr, group_ptr, coll_tag, MPIR_COLL_ATTR_SYNC); } else { - mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, &minfree, 1, MPIR_INT_INTERNAL, - MPI_MIN, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allreduce_fallback(MPI_IN_PLACE, &minfree, 1, MPIR_INT_INTERNAL, + MPI_MIN, comm_ptr, MPIR_COLL_ATTR_SYNC); } if (minfree > 0) { @@ -1053,8 +1053,8 @@ int MPIR_Get_intercomm_contextid(MPIR_Comm * comm_ptr, int *context_id, int *rec /* Make sure that all of the local processes now have this * id */ - mpi_errno = MPIR_Bcast_impl(&remote_context_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, - 0, comm_ptr->local_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(&remote_context_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, + 0, comm_ptr->local_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* The recvcontext_id must be the one that was allocated out of the local * group, not the remote group. Otherwise we could end up posting two diff --git a/src/mpi/errhan/errnames.txt b/src/mpi/errhan/errnames.txt index 3783948d523..e6a068ce37d 100644 --- a/src/mpi/errhan/errnames.txt +++ b/src/mpi/errhan/errnames.txt @@ -242,7 +242,7 @@ A memory object id was expected but a non-memory object id was passed instead **partitioninvalid %d %d: Invalid partition range, values are %d %d **opencolltuningfile: Unable to open collective tuning file -**opencolltuningfile %s: Unable to open collective tuning file %s.\nCheck the file or unset MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE to use the default settings. +**opencolltuningfile %s: Unable to open collective tuning file %s.\nCheck the file or unset MPIR_CVAR_COLL_SELECTION_JSON_FILE to use the default settings. # -- FIXME: Some (but not all) of the messages below this line have been used #---- The messages below this line haven't been used yet. @@ -944,6 +944,9 @@ is too big (> MPIU_SHMW_GHND_SZ) **mpir_wingetattr:MPII_Win_get_attr failed **mpir_wingetattr %W %d %p %p:MPII_Win_get_attr(%W, win_keyval=%d, attribute_val=%p, flag=%p) failed +## Collective selection +**csel_noresult: Collective selection failed to find an algorithm + ## Gentran related error messages **nofence: invalid Gentran fence **nullvertex: Gentran vertex is NULL diff --git a/src/mpi/stream/stream_impl.c b/src/mpi/stream/stream_impl.c index f15ddf8bddb..f1910bed451 100644 --- a/src/mpi/stream/stream_impl.c +++ b/src/mpi/stream/stream_impl.c @@ -268,8 +268,9 @@ int MPIR_Stream_comm_create_impl(MPIR_Comm * comm_ptr, MPIR_Stream * stream_ptr, vci_table = MPL_malloc(comm_ptr->local_size * sizeof(int), MPL_MEM_OTHER); MPIR_ERR_CHKANDJUMP(!vci_table, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Allgather_impl(&vci, 1, MPIR_INT_INTERNAL, - vci_table, 1, MPIR_INT_INTERNAL, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(&vci, 1, MPIR_INT_INTERNAL, + vci_table, 1, MPIR_INT_INTERNAL, comm_ptr, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); (*newcomm_ptr)->stream_comm_type = MPIR_STREAM_COMM_SINGLE; @@ -312,9 +313,9 @@ int MPIR_Stream_comm_create_multiplex_impl(MPIR_Comm * comm_ptr, MPIR_ERR_CHKANDJUMP(!displs, mpi_errno, MPI_ERR_OTHER, "**nomem"); MPI_Aint num_tmp = num_streams; - mpi_errno = MPIR_Allgather_impl(&num_tmp, 1, MPIR_AINT_INTERNAL, - num_table, 1, MPIR_AINT_INTERNAL, comm_ptr, - MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(&num_tmp, 1, MPIR_AINT_INTERNAL, + num_table, 1, MPIR_AINT_INTERNAL, comm_ptr, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); MPI_Aint num_total = 0; @@ -346,9 +347,9 @@ int MPIR_Stream_comm_create_multiplex_impl(MPIR_Comm * comm_ptr, local_vcis[i] = stream_ptr ? stream_ptr->vci : 0; } - mpi_errno = MPIR_Allgatherv_impl(local_vcis, num_streams, MPI_INT, - vci_table, num_table, displs, MPI_INT, comm_ptr, - MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgatherv_fallback(local_vcis, num_streams, MPI_INT, + vci_table, num_table, displs, MPI_INT, comm_ptr, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); (*newcomm_ptr)->stream_comm_type = MPIR_STREAM_COMM_MULTIPLEX; diff --git a/src/mpi/threadcomm/threadcomm_impl.c b/src/mpi/threadcomm/threadcomm_impl.c index 025c4bb89b2..215d8c932c9 100644 --- a/src/mpi/threadcomm/threadcomm_impl.c +++ b/src/mpi/threadcomm/threadcomm_impl.c @@ -34,8 +34,8 @@ int MPIR_Threadcomm_init_impl(MPIR_Comm * comm, int num_threads, MPIR_Comm ** co threads_table = MPL_malloc(comm_size * sizeof(int), MPL_MEM_OTHER); MPIR_ERR_CHKANDJUMP(!threads_table, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Allgather_impl(&num_threads, 1, MPIR_INT_INTERNAL, threads_table, 1, - MPIR_INT_INTERNAL, comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(&num_threads, 1, MPIR_INT_INTERNAL, threads_table, 1, + MPIR_INT_INTERNAL, comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); int *rank_offset_table;; diff --git a/src/mpi_t/mpit_initthread.c b/src/mpi_t/mpit_initthread.c index 39393b21f88..5d34377b8b4 100644 --- a/src/mpi_t/mpit_initthread.c +++ b/src/mpi_t/mpit_initthread.c @@ -9,6 +9,7 @@ * Read global config files that have MPICH_ environment variables for us to * use */ +#ifdef USE_CONFIGFILE static int read_config_files(const char *fname) { int lineno = 0; @@ -49,6 +50,7 @@ static int read_config_files(const char *fname) fclose(f); return TRUE; } +#endif static inline void MPIR_T_enum_env_init(void) { diff --git a/src/mpid/ch3/Makefile.mk b/src/mpid/ch3/Makefile.mk index 80d7136fab7..1b44a2d0c0e 100644 --- a/src/mpid/ch3/Makefile.mk +++ b/src/mpid/ch3/Makefile.mk @@ -19,8 +19,7 @@ noinst_HEADERS += \ src/mpid/ch3/include/mpidpre.h \ src/mpid/ch3/include/mpid_thread.h \ src/mpid/ch3/include/mpidrma.h \ - src/mpid/ch3/include/mpid_sched.h \ - src/mpid/ch3/include/mpid_coll.h + src/mpid/ch3/include/mpid_sched.h include $(top_srcdir)/src/mpid/ch3/src/Makefile.mk include $(top_srcdir)/src/mpid/ch3/util/Makefile.mk diff --git a/src/mpid/ch3/include/mpid_coll.h b/src/mpid/ch3/include/mpid_coll.h deleted file mode 100644 index 4b922093e59..00000000000 --- a/src/mpid/ch3/include/mpid_coll.h +++ /dev/null @@ -1,577 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#ifndef MPID_COLL_H_INCLUDED -#define MPID_COLL_H_INCLUDED - -#include "mpiimpl.h" -#ifdef HAVE_HCOLL -#include "../../common/hcoll/hcoll.h" -#endif - -static inline int MPID_Barrier(MPIR_Comm * comm, int coll_attr) -{ -#ifdef HAVE_HCOLL - if (MPI_SUCCESS == hcoll_Barrier(comm, coll_attr)) - return MPI_SUCCESS; -#endif - return MPIR_Barrier_impl(comm, coll_attr); -} - -static inline int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, - MPIR_Comm * comm, int coll_attr) -{ -#ifdef HAVE_HCOLL - if (MPI_SUCCESS == hcoll_Bcast(buffer, count, datatype, root, comm, coll_attr)) - return MPI_SUCCESS; -#endif - return MPIR_Bcast_impl(buffer, count, datatype, root, comm, coll_attr); -} - -static inline int MPID_Allreduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ -#ifdef HAVE_HCOLL - if (MPI_SUCCESS == hcoll_Allreduce(sendbuf, recvbuf, count, datatype, op, comm, coll_attr)) - return MPI_SUCCESS; -#endif - return MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); -} - -static inline int MPID_Allgather(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ -#ifdef HAVE_HCOLL - if (MPI_SUCCESS == hcoll_Allgather(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr)) - return MPI_SUCCESS; -#endif - return MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr); -} - -static inline int MPID_Allgatherv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm, - coll_attr); - - return mpi_errno; -} - -static inline int MPID_Scatter(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Scatterv(const void *sendbuf, const MPI_Aint * sendcounts, const MPI_Aint * displs, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, - coll_attr); - - return mpi_errno; -} - -static inline int MPID_Gather(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Gatherv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm, - coll_attr); - - return mpi_errno; -} - -static inline int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Alltoallv(const void *sendbuf, const MPI_Aint * sendcounts, const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, MPI_Datatype recvtype, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, - comm, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Alltoallw(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, - comm_ptr, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Reduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, - comm, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Reduce_scatter(const void *sendbuf, void *recvbuf, const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, - datatype, op, comm_ptr, coll_attr); - - return mpi_errno; -} - -static inline int MPID_Reduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, - coll_attr); - - return mpi_errno; -} - -static inline int MPID_Scan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm, - coll_attr); - - return mpi_errno; -} - -static inline int MPID_Exscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm, - coll_attr); - - return mpi_errno; -} - -static inline int MPID_Neighbor_allgather(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Neighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm); - - return mpi_errno; -} - -static inline int MPID_Neighbor_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], - MPI_Datatype recvtype, MPIR_Comm * comm) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Neighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, - recvtype, comm); - - return mpi_errno; -} - -static inline int MPID_Neighbor_alltoallv(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Neighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm); - - return mpi_errno; -} - -static inline int MPID_Neighbor_alltoallw(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Neighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm); - - return mpi_errno; -} - -static inline int MPID_Neighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Neighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm); - - return mpi_errno; -} - -static inline int MPID_Ineighbor_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ineighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, request); - - return mpi_errno; -} - -static inline int MPID_Ineighbor_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ineighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, - recvtype, comm, request); - - return mpi_errno; -} - -static inline int MPID_Ineighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ineighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, request); - - return mpi_errno; -} - -static inline int MPID_Ineighbor_alltoallv(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ineighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm, - request); - - return mpi_errno; -} - -static inline int MPID_Ineighbor_alltoallw(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ineighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm, - request); - - return mpi_errno; -} - -static inline int MPID_Ibarrier(MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ibarrier_impl(comm, request); - - return mpi_errno; -} - -static inline int MPID_Ibcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ibcast_impl(buffer, count, datatype, root, comm, request); - - return mpi_errno; -} - -static inline int MPID_Iallgather(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iallgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, request); - - return mpi_errno; -} - -static inline int MPID_Iallgatherv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iallgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm, - request); - - return mpi_errno; -} - -static inline int MPID_Iallreduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iallreduce_impl(sendbuf, recvbuf, count, datatype, op, - comm, request); - - return mpi_errno; -} - -static inline int MPID_Ialltoall(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ialltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, request); - - return mpi_errno; -} - -static inline int MPID_Ialltoallv(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ialltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, - comm, request); - - return mpi_errno; -} - -static inline int MPID_Ialltoallw(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ialltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, - comm, request); - - return mpi_errno; -} - -static inline int MPID_Iexscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iexscan_impl(sendbuf, recvbuf, count, datatype, op, comm, - request); - - return mpi_errno; -} - -static inline int MPID_Igather(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Igather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); - - return mpi_errno; -} - -static inline int MPID_Igatherv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Igatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm, - request); - - return mpi_errno; -} - -static inline int MPID_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ireduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm, request); - - return mpi_errno; -} - -static inline int MPID_Ireduce_scatter(const void *sendbuf, void *recvbuf, const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ireduce_scatter_impl(sendbuf, recvbuf, recvcounts, - datatype, op, comm, request); - - return mpi_errno; -} - -static inline int MPID_Ireduce(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Ireduce_impl(sendbuf, recvbuf, count, datatype, op, root, - comm, request); - - return mpi_errno; -} - -static inline int MPID_Iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iscan_impl(sendbuf, recvbuf, count, datatype, op, comm, - request); - - return mpi_errno; -} - -static inline int MPID_Iscatter(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iscatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); - - return mpi_errno; -} - -static inline int MPID_Iscatterv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Iscatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, - request); - - return mpi_errno; -} - -#endif /* MPID_COLL_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpidpost.h b/src/mpid/ch3/include/mpidpost.h index 231bccec1bf..5feadf8bd1b 100644 --- a/src/mpid/ch3/include/mpidpost.h +++ b/src/mpid/ch3/include/mpidpost.h @@ -6,8 +6,6 @@ #ifndef MPIDPOST_H_INCLUDED #define MPIDPOST_H_INCLUDED -#include "mpid_coll.h" - /* FIXME: mpidpost.h is included by mpiimpl.h . However, mpiimpl.h should refer only to the ADI3 prototypes and should never include prototypes specific to any particular device. Factor the include files to maintain @@ -278,4 +276,8 @@ MPL_STATIC_INLINE_PREFIX int MPID_Waitsome(int incount, MPIR_Request * request_p return MPIR_Waitsome_impl(incount, request_ptrs, outcount, array_of_indices, array_of_statuses); } +MPL_STATIC_INLINE_PREFIX void MPID_Init_coll_sig(MPIR_Csel_coll_sig_s * coll_sig) +{ +} + #endif /* MPIDPOST_H_INCLUDED */ diff --git a/src/mpid/ch3/include/mpidpre.h b/src/mpid/ch3/include/mpidpre.h index 5b28b78161a..3cee5586a03 100644 --- a/src/mpid/ch3/include/mpidpre.h +++ b/src/mpid/ch3/include/mpidpre.h @@ -616,99 +616,6 @@ int MPID_Recv_init( void *buf, MPI_Aint count, MPI_Datatype datatype, int MPID_Startall(int count, MPIR_Request *requests[]); -int MPID_Bcast_init(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, - MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, MPIR_Request **request); - -int MPID_Allreduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, - MPIR_Request ** request); - -int MPID_Reduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, - MPIR_Request **request); - -int MPID_Alltoall_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, MPIR_Request** request); - -int MPID_Alltoallv_init(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Info * info_ptr, MPIR_Request ** request); - -int MPID_Alltoallw_init(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); - -int MPID_Allgather_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, MPIR_Request** request); - -int MPID_Allgatherv_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, - MPIR_Request** request); - -int MPID_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); - -int MPID_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, MPIR_Info * info, - MPIR_Request ** request); - -int MPID_Scan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - -int MPID_Gather_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); - -int MPID_Gatherv_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - -int MPID_Scatter_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); - -int MPID_Scatterv_init(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint displs[], - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, MPIR_Info * info, - MPIR_Request ** request); - -int MPID_Barrier_init(MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - -int MPID_Exscan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - -int MPID_Neighbor_allgather_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - -int MPID_Neighbor_allgatherv_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint displs[], MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); - -int MPID_Neighbor_alltoall_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - - -int MPID_Neighbor_alltoallv_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint rdispls[], - MPI_Datatype recvtype, MPIR_Comm * comm, MPIR_Info * info, - MPIR_Request ** request); - -int MPID_Neighbor_alltoallw_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); - int MPID_Probe(int source, int tag, MPIR_Comm *comm, int attr, MPI_Status * status); int MPID_Iprobe(int source, int tag, MPIR_Comm *comm, int attr, int *flag, MPI_Status *status); diff --git a/src/mpid/ch3/src/ch3u_port.c b/src/mpid/ch3/src/ch3u_port.c index 0bca16298b5..d310d109fe2 100644 --- a/src/mpid/ch3/src/ch3u_port.c +++ b/src/mpid/ch3/src/ch3u_port.c @@ -656,7 +656,7 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, /* broadcast the received info to local processes */ MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"broadcasting the received 3 ints"); - mpi_errno = MPIR_Bcast_allcomm_auto(recv_ints, 3, MPIR_INT_INTERNAL, root, comm_ptr, 0); + mpi_errno = MPIR_Bcast_fallback(recv_ints, 3, MPIR_INT_INTERNAL, root, comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); /* check if root was unable to connect to the port */ @@ -705,7 +705,7 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, /* Broadcast out the remote rank translation array */ MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"Broadcasting remote translation"); - mpi_errno = MPIR_Bcast_allcomm_auto(remote_translation, remote_comm_size * 2, MPIR_INT_INTERNAL, + mpi_errno = MPIR_Bcast_fallback(remote_translation, remote_comm_size * 2, MPIR_INT_INTERNAL, root, comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); @@ -744,7 +744,7 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, } /*printf("connect:barrier\n");fflush(stdout);*/ - mpi_errno = MPIR_Barrier_allcomm_auto(comm_ptr, 0); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); /* Free new_vc. It was explicitly allocated in MPIDI_CH3_Connect_to_root.*/ @@ -790,7 +790,7 @@ int MPIDI_Comm_connect(const char *port_name, MPIR_Info *info, int root, /* notify other processes to return an error */ MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"broadcasting 3 ints: error case"); - mpi_errno2 = MPIR_Bcast_allcomm_auto(recv_ints, 3, MPIR_INT_INTERNAL, root, comm_ptr, 0); + mpi_errno2 = MPIR_Bcast_fallback(recv_ints, 3, MPIR_INT_INTERNAL, root, comm_ptr, 0); if (mpi_errno2) MPIR_ERR_ADD(mpi_errno, mpi_errno2); goto fn_fail; } @@ -937,7 +937,7 @@ static int ReceivePGAndDistribute( MPIR_Comm *tmp_comm, MPIR_Comm *comm_ptr, /* Broadcast the size and data to the local communicator */ /*printf("accept:broadcasting 1 int\n");fflush(stdout);*/ - mpi_errno = MPIR_Bcast_allcomm_auto(&j, 1, MPIR_INT_INTERNAL, root, comm_ptr, 0); + mpi_errno = MPIR_Bcast_fallback(&j, 1, MPIR_INT_INTERNAL, root, comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); if (rank != root) { @@ -948,7 +948,7 @@ static int ReceivePGAndDistribute( MPIR_Comm *tmp_comm, MPIR_Comm *comm_ptr, } } /*printf("accept:broadcasting string of length %d\n", j);fflush(stdout);*/ - mpi_errno = MPIR_Bcast_allcomm_auto(pg_str, j, MPIR_CHAR_INTERNAL, root, comm_ptr, 0); + mpi_errno = MPIR_Bcast_fallback(pg_str, j, MPIR_CHAR_INTERNAL, root, comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); /* Then reconstruct the received process group. This step also initializes the created process group */ @@ -1179,7 +1179,7 @@ int MPIDI_Comm_accept(const char *port_name, MPIR_Info *info, int root, /* broadcast the received info to local processes */ /*printf("accept:broadcasting 2 ints - %d and %d\n", recv_ints[0], recv_ints[1]);fflush(stdout);*/ - mpi_errno = MPIR_Bcast_allcomm_auto(recv_ints, 3, MPIR_INT_INTERNAL, root, comm_ptr, 0); + mpi_errno = MPIR_Bcast_fallback(recv_ints, 3, MPIR_INT_INTERNAL, root, comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); @@ -1229,7 +1229,7 @@ int MPIDI_Comm_accept(const char *port_name, MPIR_Info *info, int root, /* Broadcast out the remote rank translation array */ MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"Broadcast remote_translation"); - mpi_errno = MPIR_Bcast_allcomm_auto(remote_translation, remote_comm_size * 2, MPIR_INT_INTERNAL, + mpi_errno = MPIR_Bcast_fallback(remote_translation, remote_comm_size * 2, MPIR_INT_INTERNAL, root, comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); #ifdef MPICH_DBG_OUTPUT @@ -1266,7 +1266,7 @@ int MPIDI_Comm_accept(const char *port_name, MPIR_Info *info, int root, } MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"Barrier"); - mpi_errno = MPIR_Barrier_allcomm_auto(comm_ptr, 0); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); /* Free new_vc once the connection is completed. It was explicitly @@ -1345,7 +1345,7 @@ static int SetupNewIntercomm( MPIR_Comm *comm_ptr, int remote_comm_size, MPIR_ERR_CHECK(mpi_errno); MPL_DBG_MSG(MPIDI_CH3_DBG_CONNECT,VERBOSE,"Barrier"); - mpi_errno = MPIR_Barrier_allcomm_auto(comm_ptr, 0); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, 0); MPIR_ERR_CHECK(mpi_errno); fn_exit: diff --git a/src/mpid/ch3/src/mpid_startall.c b/src/mpid/ch3/src/mpid_startall.c index cba93847a43..e7d9e8dcee2 100644 --- a/src/mpid/ch3/src/mpid_startall.c +++ b/src/mpid/ch3/src/mpid_startall.c @@ -315,426 +315,3 @@ int MPID_Recv_init(void * buf, MPI_Aint count, MPI_Datatype datatype, int rank, MPIR_FUNC_EXIT; return mpi_errno; } - -int MPID_Bcast_init(void *buffer, MPI_Aint count, MPI_Datatype datatype, int root, - MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Bcast_init_impl(buffer, count, datatype, root, comm_ptr, info_ptr, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Allreduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, - MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allreduce_init_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, info_ptr, - request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Reduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, - MPIR_Request **request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_init_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, - info_ptr, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Alltoall_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, MPIR_Request** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoall_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, info_ptr, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Alltoallv_init(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallv_init_impl(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm_ptr, info_ptr, - request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Alltoallw_init(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallw_init_impl(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, - recvcounts, rdispls, recvtypes, comm_ptr, info_ptr, - request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Allgather_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, MPIR_Request** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allgather_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, info_ptr, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Allgatherv_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm *comm_ptr, MPIR_Info* info_ptr, - MPIR_Request** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allgatherv_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, comm_ptr, info_ptr, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_scatter_block_init_impl(sendbuf, recvbuf, recvcount, datatype, op, comm, - info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, MPIR_Info * info, - MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_scatter_init_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, - info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Scan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scan_init_impl(sendbuf, recvbuf, count, datatype, op, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Gather_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gather_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - root, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Gatherv_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gatherv_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, - recvtype, root, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Scatter_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatter_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - root, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Scatterv_init(const void *sendbuf, const MPI_Aint sendcounts[], const MPI_Aint displs[], - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, MPIR_Info * info, - MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatterv_init_impl(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, - recvtype, root, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Barrier_init(MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Barrier_init_impl(comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Exscan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Exscan_init_impl(sendbuf, recvbuf, count, datatype, op, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Neighbor_allgather_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgather_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Neighbor_allgatherv_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], const MPI_Aint - displs[], MPI_Datatype recvtype, MPIR_Comm * comm, MPIR_Info * - info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgatherv_init_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm, info, - request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Neighbor_alltoall_init(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoall_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, info, request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Neighbor_alltoallv_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint rdispls[], - MPI_Datatype recvtype, MPIR_Comm * comm, MPIR_Info * info, - MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallv_init_impl(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm, info, - request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -int MPID_Neighbor_alltoallw_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallw_init_impl(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, - recvcounts, rdispls, recvtypes, comm, info, - request); - MPIR_ERR_CHECK(mpi_errno); - MPIDI_Request_set_type(*request, MPIDI_REQUEST_TYPE_PERSISTENT_COLL); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} diff --git a/src/mpid/ch3/src/mpid_vc.c b/src/mpid/ch3/src/mpid_vc.c index 1e62b5b3a94..9da1a4be1eb 100644 --- a/src/mpid/ch3/src/mpid_vc.c +++ b/src/mpid/ch3/src/mpid_vc.c @@ -637,7 +637,7 @@ int MPIDI_PG_ForwardPGInfo( MPIR_Comm *peer_ptr, MPIR_Comm *comm_ptr, } /* See if everyone is happy */ - mpi_errno = MPIR_Allreduce( MPI_IN_PLACE, &allfound, 1, MPIR_INT_INTERNAL, MPI_LAND, comm_ptr, 0 ); + mpi_errno = MPIR_Allreduce_fallback( MPI_IN_PLACE, &allfound, 1, MPIR_INT_INTERNAL, MPI_LAND, comm_ptr, 0 ); MPIR_ERR_CHECK(mpi_errno); if (allfound) return MPI_SUCCESS; diff --git a/src/mpid/ch4/ch4_api.txt b/src/mpid/ch4/ch4_api.txt index 631144bd573..623ef6445a0 100644 --- a/src/mpid/ch4/ch4_api.txt +++ b/src/mpid/ch4/ch4_api.txt @@ -278,138 +278,6 @@ Native API: NM*: assert, win rank_is_local : int NM*: target, comm - mpi_barrier : int - NM*: comm, coll_attr - SHM*: comm, coll_attr - mpi_bcast : int - NM*: buffer, count, datatype, root, comm, coll_attr - SHM*: buffer, count, datatype, root, comm, coll_attr - mpi_allreduce : int - NM*: sendbuf, recvbuf, count, datatype, op, comm, coll_attr - SHM*: sendbuf, recvbuf, count, datatype, op, comm, coll_attr - mpi_allgather : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, coll_attr - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, coll_attr - mpi_allgatherv : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, coll_attr - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, coll_attr - mpi_scatter : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, coll_attr - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, coll_attr - mpi_scatterv : int - NM*: sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, coll_attr - SHM*: sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, coll_attr - mpi_gather : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, coll_attr - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, coll_attr - mpi_gatherv : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, coll_attr - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, coll_attr - mpi_alltoall : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, coll_attr - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, coll_attr - mpi_alltoallv : int - NM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, coll_attr - SHM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, coll_attr - mpi_alltoallw : int - NM*: sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, coll_attr - SHM*: sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, coll_attr - mpi_reduce : int - NM*: sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr - SHM*: sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr - mpi_reduce_scatter : int - NM*: sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, coll_attr - SHM*: sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, coll_attr - mpi_reduce_scatter_block : int - NM*: sendbuf, recvbuf, recvcount, datatype, op, comm_ptr, coll_attr - SHM*: sendbuf, recvbuf, recvcount, datatype, op, comm_ptr, coll_attr - mpi_scan : int - NM*: sendbuf, recvbuf, count, datatype, op, comm, coll_attr - SHM*: sendbuf, recvbuf, count, datatype, op, comm, coll_attr - mpi_exscan : int - NM*: sendbuf, recvbuf, count, datatype, op, comm, coll_attr - SHM*: sendbuf, recvbuf, count, datatype, op, comm, coll_attr - mpi_neighbor_allgather : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm - mpi_neighbor_allgatherv : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm - mpi_neighbor_alltoallv : int - NM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm - SHM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm - mpi_neighbor_alltoallw : int - NM*: sendbuf, sendcounts, sdispls-2, sendtypes, recvbuf, recvcounts, rdispls-2, recvtypes, comm - SHM*: sendbuf, sendcounts, sdispls-2, sendtypes, recvbuf, recvcounts, rdispls-2, recvtypes, comm - mpi_neighbor_alltoall : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm - mpi_ineighbor_allgather : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - mpi_ineighbor_allgatherv : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, req_p - mpi_ineighbor_alltoall : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - mpi_ineighbor_alltoallv : int - NM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, req_p - SHM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, req_p - mpi_ineighbor_alltoallw : int - NM*: sendbuf, sendcounts, sdispls-2, sendtypes, recvbuf, recvcounts, rdispls-2, recvtypes, comm, req_p - SHM*: sendbuf, sendcounts, sdispls-2, sendtypes, recvbuf, recvcounts, rdispls-2, recvtypes, comm, req_p - mpi_ibarrier : int - NM*: comm, req_p - SHM*: comm, req_p - mpi_ibcast : int - NM*: buffer, count, datatype, root, comm, req_p - SHM*: buffer, count, datatype, root, comm, req_p - mpi_iallgather : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - mpi_iallgatherv : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm, req_p - mpi_iallreduce : int - NM*: sendbuf, recvbuf, count, datatype, op, comm, req_p - SHM*: sendbuf, recvbuf, count, datatype, op, comm, req_p - mpi_ialltoall : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, req_p - mpi_ialltoallv : int - NM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, req_p - SHM*: sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, req_p - mpi_ialltoallw : int - NM*: sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, req_p - SHM*: sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, req_p - mpi_iexscan : int - NM*: sendbuf, recvbuf, count, datatype, op, comm, req_p - SHM*: sendbuf, recvbuf, count, datatype, op, comm, req_p - mpi_igather : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, req_p - mpi_igatherv : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, req_p - mpi_ireduce_scatter_block : int - NM*: sendbuf, recvbuf, recvcount, datatype, op, comm, req_p - SHM*: sendbuf, recvbuf, recvcount, datatype, op, comm, req_p - mpi_ireduce_scatter : int - NM*: sendbuf, recvbuf, recvcounts, datatype, op, comm, req_p - SHM*: sendbuf, recvbuf, recvcounts, datatype, op, comm, req_p - mpi_ireduce : int - NM*: sendbuf, recvbuf, count, datatype, op, root, comm_ptr, req_p - SHM*: sendbuf, recvbuf, count, datatype, op, root, comm_ptr, req_p - mpi_iscan : int - NM*: sendbuf, recvbuf, count, datatype, op, comm, req_p - SHM*: sendbuf, recvbuf, count, datatype, op, comm, req_p - mpi_iscatter : int - NM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, req_p - SHM*: sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, req_p - mpi_iscatterv : int - NM*: sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, req_p - SHM*: sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm_ptr, req_p mpi_type_commit_hook : int NM : datatype_p SHM : type @@ -457,7 +325,6 @@ PARAM: disp_unit_p: int * displs: const MPI_Aint * dst_vci: int - coll_attr: int flag: int * group: MPIR_Group * handler_id: int diff --git a/src/mpid/ch4/include/mpidch4.h b/src/mpid/ch4/include/mpidch4.h index 103b3bd6900..30593ba2ae6 100644 --- a/src/mpid/ch4/include/mpidch4.h +++ b/src/mpid/ch4/include/mpidch4.h @@ -177,235 +177,6 @@ int MPID_Comm_set_hints(MPIR_Comm *, MPIR_Info *); int MPID_Comm_commit_post_hook(MPIR_Comm *); int MPID_Stream_create_hook(MPIR_Stream * stream); int MPID_Stream_free_hook(MPIR_Stream * stream); -MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *, MPI_Aint, MPI_Datatype, int, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Allgatherv(const void *, MPI_Aint, MPI_Datatype, void *, - const MPI_Aint *, const MPI_Aint *, MPI_Datatype, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Scatter(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, int, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Scatterv(const void *, const MPI_Aint *, const MPI_Aint *, - MPI_Datatype, void *, MPI_Aint, MPI_Datatype, int, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Gather(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, int, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Gatherv(const void *, MPI_Aint, MPI_Datatype, void *, - const MPI_Aint *, const MPI_Aint *, MPI_Datatype, int, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Alltoallv(const void *, const MPI_Aint *, const MPI_Aint *, - MPI_Datatype, void *, const MPI_Aint *, - const MPI_Aint *, MPI_Datatype, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Alltoallw(const void *, const MPI_Aint[], const MPI_Aint[], - const MPI_Datatype[], void *, const MPI_Aint[], - const MPI_Aint[], const MPI_Datatype[], MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, int, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Reduce_scatter(const void *, void *, const MPI_Aint[], - MPI_Datatype, MPI_Op, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Reduce_scatter_block(const void *, void *, MPI_Aint, MPI_Datatype, - MPI_Op, MPIR_Comm *, - int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Scan(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Exscan(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, - MPIR_Comm *, int) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_allgather(const void *, MPI_Aint, MPI_Datatype, void *, - MPI_Aint, MPI_Datatype, - MPIR_Comm *) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_allgatherv(const void *, MPI_Aint, MPI_Datatype, void *, - const MPI_Aint[], const MPI_Aint[], - MPI_Datatype, - MPIR_Comm *) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_alltoallv(const void *, const MPI_Aint[], - const MPI_Aint[], MPI_Datatype, void *, - const MPI_Aint[], const MPI_Aint[], - MPI_Datatype, - MPIR_Comm *) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_alltoallw(const void *, const MPI_Aint[], - const MPI_Aint[], const MPI_Datatype[], void *, - const MPI_Aint[], const MPI_Aint[], - const MPI_Datatype[], - MPIR_Comm *) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_alltoall(const void *, MPI_Aint, MPI_Datatype, void *, - MPI_Aint, MPI_Datatype, - MPIR_Comm *) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_allgather(const void *, MPI_Aint, MPI_Datatype, void *, - MPI_Aint, MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_allgatherv(const void *, MPI_Aint, MPI_Datatype, void *, - const MPI_Aint[], const MPI_Aint[], - MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_alltoall(const void *, MPI_Aint, MPI_Datatype, void *, - MPI_Aint, MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_alltoallv(const void *, const MPI_Aint[], - const MPI_Aint[], MPI_Datatype, void *, - const MPI_Aint[], const MPI_Aint[], - MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_alltoallw(const void *, const MPI_Aint[], - const MPI_Aint[], const MPI_Datatype[], - void *, const MPI_Aint[], const MPI_Aint[], - const MPI_Datatype[], MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ibarrier(MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ibcast(void *, MPI_Aint, MPI_Datatype, int, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iallgather(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iallgatherv(const void *, MPI_Aint, MPI_Datatype, void *, - const MPI_Aint *, const MPI_Aint *, MPI_Datatype, - MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iallreduce(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, - MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ialltoall(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ialltoallv(const void *, const MPI_Aint[], const MPI_Aint[], - MPI_Datatype, void *, const MPI_Aint[], - const MPI_Aint[], MPI_Datatype, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ialltoallw(const void *, const MPI_Aint[], const MPI_Aint[], - const MPI_Datatype[], void *, const MPI_Aint[], - const MPI_Aint[], const MPI_Datatype[], MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iexscan(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, - MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Igather(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, int, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Igatherv(const void *, MPI_Aint, MPI_Datatype, void *, - const MPI_Aint *, const MPI_Aint *, MPI_Datatype, int, - MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ireduce_scatter_block(const void *, void *, MPI_Aint, - MPI_Datatype, MPI_Op, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ireduce_scatter(const void *, void *, const MPI_Aint[], - MPI_Datatype, MPI_Op, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Ireduce(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, int, - MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iscan(const void *, void *, MPI_Aint, MPI_Datatype, MPI_Op, - MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iscatter(const void *, MPI_Aint, MPI_Datatype, void *, MPI_Aint, - MPI_Datatype, int, MPIR_Comm *, - MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -MPL_STATIC_INLINE_PREFIX int MPID_Iscatterv(const void *, const MPI_Aint *, const MPI_Aint *, - MPI_Datatype, void *, MPI_Aint, MPI_Datatype, int, - MPIR_Comm *, MPIR_Request **) MPL_STATIC_INLINE_SUFFIX; -int MPID_Bcast_init(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Allreduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Reduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Alltoall_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Alltoallv_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Alltoallw_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Allgather_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Allgatherv_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request); -int MPID_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Reduce_scatter_init(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Scan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); -int MPID_Gather_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Gatherv_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); -int MPID_Scatter_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Scatterv_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint displs[], MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); -int MPID_Barrier_init(MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Exscan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); -int MPID_Neighbor_allgather_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Neighbor_allgatherv_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); -int MPID_Neighbor_alltoall_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); -int MPID_Neighbor_alltoallv_init(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request); -int MPID_Neighbor_alltoallw_init(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request); int MPID_Send_enqueue(const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest, int tag, MPIR_Comm * comm_ptr); int MPID_Recv_enqueue(void *buf, MPI_Aint count, MPI_Datatype datatype, diff --git a/src/mpid/ch4/include/mpidpost.h b/src/mpid/ch4/include/mpidpost.h index 0776e979d9e..d69d19eb996 100644 --- a/src/mpid/ch4/include/mpidpost.h +++ b/src/mpid/ch4/include/mpidpost.h @@ -62,4 +62,8 @@ MPL_STATIC_INLINE_PREFIX int MPID_Finalize_async_thread(void) return MPIR_Stop_progress_thread_impl(NULL); } +MPL_STATIC_INLINE_PREFIX void MPID_Init_coll_sig(MPIR_Csel_coll_sig_s * coll_sig) +{ +} + #endif /* MPIDPOST_H_INCLUDED */ diff --git a/src/mpid/ch4/include/mpidpre.h b/src/mpid/ch4/include/mpidpre.h index d4101c158fb..9c0da32e749 100644 --- a/src/mpid/ch4/include/mpidpre.h +++ b/src/mpid/ch4/include/mpidpre.h @@ -28,7 +28,6 @@ #include "shmpre.h" #endif #include "uthash.h" -#include "ch4_csel_container.h" #define MPID_TAG_DEV_BITS 0 #define MPID_MAX_BC_SIZE 4096 @@ -596,9 +595,6 @@ typedef struct MPIDI_Devcomm_t { struct MPIDI_Multileads_comp_info_t *alltoall_comp_info, *allgather_comp_info, *allreduce_comp_info; int shm_size_per_lead; - - void *csel_comm; /* collective selection handle */ - void *csel_comm_gpu; /* collective selection handle for gpu */ } ch4; } MPIDI_Devcomm_t; diff --git a/src/mpid/ch4/netmod/include/netmod_am_fallback.h b/src/mpid/ch4/netmod/include/netmod_am_fallback.h index fd708415fa7..443ca7486c0 100644 --- a/src/mpid/ch4/netmod/include/netmod_am_fallback.h +++ b/src/mpid/ch4/netmod/include/netmod_am_fallback.h @@ -10,7 +10,6 @@ #include "netmod_am_fallback_recv.h" #include "netmod_am_fallback_probe.h" #include "netmod_am_fallback_rma.h" -#include "netmod_am_fallback_coll.h" #include "netmod_am_fallback_part.h" #endif /* NETMOD_AM_FALLBACK_H_INCLUDED */ diff --git a/src/mpid/ch4/netmod/include/netmod_am_fallback_coll.h b/src/mpid/ch4/netmod/include/netmod_am_fallback_coll.h deleted file mode 100644 index 2c004d4773f..00000000000 --- a/src/mpid/ch4/netmod/include/netmod_am_fallback_coll.h +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#ifndef NETMOD_AM_FALLBACK_COLL_H_INCLUDED -#define NETMOD_AM_FALLBACK_COLL_H_INCLUDED - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_barrier(MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Barrier_impl(comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Bcast_impl(buffer, count, datatype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scatterv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, - coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_exscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ibarrier(MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ibarrier_impl(comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ibcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ibcast_impl(buffer, count, datatype, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Iallgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Iallgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** request) -{ - return MPIR_Iallreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, request); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ialltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ialltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ialltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iexscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Iexscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Igather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Igatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ireduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ireduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ireduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Iscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** request) -{ - return MPIR_Iscatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Request ** request) -{ - return MPIR_Iscatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, request); -} - -#endif /* NETMOD_AM_FALLBACK_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/netmod/ofi/ofi_coll.h b/src/mpid/ch4/netmod/ofi/ofi_coll.h index 9fe52f2b35c..f394996e254 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_coll.h +++ b/src/mpid/ch4/netmod/ofi/ofi_coll.h @@ -30,860 +30,4 @@ === END_MPI_T_CVAR_INFO_BLOCK === */ -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_barrier(MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Barrier_impl(comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -static inline int MPIDI_OFI_bcast_json(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - enum fi_datatype fi_dt; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_BCAST_OFI_INTRA_ALGORITHM) { - case MPIR_CVAR_BCAST_OFI_INTRA_ALGORITHM_trigger_tree_tagged: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, MPIDI_OFI_ENABLE_TRIGGERED && - MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS && - MPIDI_OFI_datatype_to_ofi(datatype, &fi_dt) != -1, - mpi_errno, - "Bcast triggered_tagged cannot be applied.\n"); - mpi_errno = - MPIDI_OFI_Bcast_intra_triggered_tagged(buffer, count, datatype, root, comm, - MPIR_Bcast_tree_type, - MPIR_CVAR_BCAST_TREE_KVAL); - break; - case MPIR_CVAR_BCAST_OFI_INTRA_ALGORITHM_trigger_tree_rma: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, MPIDI_OFI_ENABLE_TRIGGERED && - MPIDI_OFI_ENABLE_DATA_AUTO_PROGRESS && - MPIDI_OFI_datatype_to_ofi(datatype, &fi_dt) != -1, - mpi_errno, "Bcast triggered_rma cannot be applied.\n"); - mpi_errno = - MPIDI_OFI_Bcast_intra_triggered_rma(buffer, count, datatype, root, comm, - MPIR_Bcast_tree_type, - MPIR_CVAR_BCAST_TREE_KVAL); - break; - case MPIR_CVAR_BCAST_OFI_INTRA_ALGORITHM_mpir: - goto fallback; - case MPIR_CVAR_BCAST_OFI_INTRA_ALGORITHM_auto: - mpi_errno = MPIDI_OFI_bcast_json(buffer, count, datatype, root, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; - -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scatterv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = - MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_exscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = - MPIR_Neighbor_allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ibarrier(MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ibarrier_impl(comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ibcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ibcast_impl(buffer, count, datatype, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iexscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iexscan_impl(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Igather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Igatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscan_impl(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - #endif /* OFI_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/netmod/ofi/ofi_comm.c b/src/mpid/ch4/netmod/ofi/ofi_comm.c index 619ee3f3c2a..b72772ee291 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_comm.c +++ b/src/mpid/ch4/netmod/ofi/ofi_comm.c @@ -105,9 +105,9 @@ static int update_nic_preferences(MPIR_Comm * comm) /* Collect the NIC IDs set for the other ranks. We always expect to receive a single * NIC id from each rank, i.e., one MPI_INT. */ - mpi_errno = MPIR_Allgather_allcomm_auto(MPI_IN_PLACE, 0, MPIR_INT_INTERNAL, - pref_nic_copy, 1, MPIR_INT_INTERNAL, comm, - MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(MPI_IN_PLACE, 0, MPIR_INT_INTERNAL, + pref_nic_copy, 1, MPIR_INT_INTERNAL, comm, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (MPIDI_OFI_COMM(comm).pref_nic == NULL) { diff --git a/src/mpid/ch4/netmod/ofi/ofi_win.c b/src/mpid/ch4/netmod/ofi/ofi_win.c index 9ac46837561..a738ce5ab2c 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_win.c +++ b/src/mpid/ch4/netmod/ofi/ofi_win.c @@ -132,8 +132,9 @@ static int win_allgather(MPIR_Win * win, void *base, int disp_unit) * Attempting to create an optimized memory region key. Gets the next MR key that's * available to the processes involved in the RMA window. Use the current maximum + 1 * to ensure that the key is available for all processes. */ - mpi_errno = MPIR_Allreduce(&MPIDI_OFI_global.global_max_optimized_mr_key, &local_key, 1, - MPI_UNSIGNED, MPI_MAX, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(&MPIDI_OFI_global.global_max_optimized_mr_key, &local_key, + 1, MPI_UNSIGNED, MPI_MAX, comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (local_key + 1 < MPIDI_OFI_NUM_OPTIMIZED_MEMORY_REGIONS) { @@ -216,7 +217,8 @@ static int win_allgather(MPIR_Win * win, void *base, int disp_unit) } /* Check if any process fails to register. If so, release local MR and force AM path. */ - MPIR_Allreduce(&rc, &allrc, 1, MPIR_INT_INTERNAL, MPI_MIN, comm_ptr, MPIR_COLL_ATTR_SYNC); + MPIR_Allreduce_fallback(&rc, &allrc, 1, MPIR_INT_INTERNAL, MPI_MIN, comm_ptr, + MPIR_COLL_ATTR_SYNC); if (allrc < 0) { if (rc >= 0 && MPIDI_OFI_WIN(win).mr) MPIDI_OFI_CALL(fi_close(&MPIDI_OFI_WIN(win).mr->fid), fi_close); @@ -963,7 +965,8 @@ int MPIDI_OFI_mpi_win_attach_hook(MPIR_Win * win, void *base, MPI_Aint size) } /* Check if any process fails to register. If so, release local MR and force AM path. */ - MPIR_Allreduce(&rc, &allrc, 1, MPIR_INT_INTERNAL, MPI_MIN, comm_ptr, MPIR_COLL_ATTR_SYNC); + MPIR_Allreduce_fallback(&rc, &allrc, 1, MPIR_INT_INTERNAL, MPI_MIN, comm_ptr, + MPIR_COLL_ATTR_SYNC); if (allrc < 0) { if (rc >= 0) MPIDI_OFI_CALL(fi_close(&mr->fid), fi_close); diff --git a/src/mpid/ch4/netmod/ucx/ucx_coll.h b/src/mpid/ch4/netmod/ucx/ucx_coll.h index 47dd5328f98..cf0f0274cff 100644 --- a/src/mpid/ch4/netmod/ucx/ucx_coll.h +++ b/src/mpid/ch4/netmod/ucx/ucx_coll.h @@ -11,872 +11,4 @@ #include "../../../common/hcoll/hcoll.h" #endif -#ifdef HAVE_UCC -#define MPIDI_UCX_UCC_CALL(_collop_fn) \ - MPIDI_COMMOM_UCC_CHECK_AND_FALLBACK(_collop_fn, (void) 0, (void) 0, \ - (void) 0); -#endif - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_barrier(MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_barrier(comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Barrier(comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_bcast(buffer, count, datatype, root, comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Bcast(buffer, count, datatype, root, comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm_ptr, coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_allreduce(sendbuf, recvbuf, count, datatype, op, comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Allreduce(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_allgather - (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Allgather(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_allgatherv - (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, - comm_ptr)); -#endif - mpi_errno = MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_gather - (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, - comm_ptr)); -#endif - mpi_errno = MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_gatherv - (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, - comm_ptr)); -#endif - mpi_errno = MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_scatter - (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, - comm_ptr)); -#endif - mpi_errno = MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scatterv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_scatterv - (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, - comm_ptr)); -#endif - mpi_errno = MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_alltoall - (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Alltoall(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_alltoallv - (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, - recvtype, comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm_ptr, - coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_reduce - (sendbuf, recvbuf, count, datatype, op, root, comm_ptr)); -#endif -#ifdef HAVE_HCOLL - mpi_errno = hcoll_Reduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr); - if (mpi_errno != MPI_SUCCESS) -#endif - { - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, - coll_attr); - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_reduce_scatter - (sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr)); -#endif - mpi_errno = MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, - datatype, op, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_reduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - -#ifdef HAVE_UCC - MPIDI_UCX_UCC_CALL(MPIDI_common_ucc_reduce_scatter_block - (sendbuf, recvbuf, recvcount, datatype, op, comm_ptr)); -#endif - mpi_errno = MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_scan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_exscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm_ptr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm_ptr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_neighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, - recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ibarrier(MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ibarrier_impl(comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ibcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ibcast_impl(buffer, count, datatype, root, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iallreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ialltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iexscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iexscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Igather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Igatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_scatter_impl(sendbuf, recvbuf, recvcounts, - datatype, op, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_ireduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_NM_mpi_iscatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - #endif /* UCX_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/netmod/ucx/ucx_vci.c b/src/mpid/ch4/netmod/ucx/ucx_vci.c index c58a5ace8c0..4c27c5d9fc4 100644 --- a/src/mpid/ch4/netmod/ucx/ucx_vci.c +++ b/src/mpid/ch4/netmod/ucx/ucx_vci.c @@ -75,9 +75,9 @@ static int all_vcis_address_exchange(MPIR_Comm * comm) MPIDI_UCX_global.ctx[i].addrname_len); } /* Allgather */ - mpi_errno = MPIR_Allgather_allcomm_auto(MPI_IN_PLACE, 0, MPIR_BYTE_INTERNAL, - all_names, my_len, MPIR_BYTE_INTERNAL, - comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(MPI_IN_PLACE, 0, MPIR_BYTE_INTERNAL, + all_names, my_len, MPIR_BYTE_INTERNAL, + comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* insert the addresses */ diff --git a/src/mpid/ch4/shm/ipc/src/ipc_fd.c b/src/mpid/ch4/shm/ipc/src/ipc_fd.c index a8713abf130..6101fce2128 100644 --- a/src/mpid/ch4/shm/ipc/src/ipc_fd.c +++ b/src/mpid/ch4/shm/ipc/src/ipc_fd.c @@ -46,8 +46,9 @@ int MPIDI_FD_comm_bootstrap(MPIR_Comm * comm) } int already_initialized; - mpi_errno = MPIR_Allreduce_impl(&ipc_fd_initialized, &already_initialized, 1, MPIR_INT_INTERNAL, - MPI_MAX, node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(&ipc_fd_initialized, &already_initialized, 1, MPIR_INT_INTERNAL, + MPI_MAX, node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (already_initialized) { @@ -141,7 +142,7 @@ static int MPIDI_IPC_mpi_socks_init(MPIR_Comm * node_comm, pid_t * all_pids, int MPIR_STRERROR_BUF_SIZE), errno); } - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* create clients for higher ranks */ @@ -224,9 +225,9 @@ static int MPIDI_IPC_mpi_fd_init(MPIR_Comm * node_comm) MPIR_CHKLMEM_MALLOC(all_pids, local_size * sizeof(pid_t)); all_pids[local_rank] = getpid(); - mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, sizeof(pid_t), MPIR_BYTE_INTERNAL, - all_pids, sizeof(pid_t), MPIR_BYTE_INTERNAL, - node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(MPI_IN_PLACE, sizeof(pid_t), MPIR_BYTE_INTERNAL, + all_pids, sizeof(pid_t), MPIR_BYTE_INTERNAL, + node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* Initialize fd_socks */ @@ -252,10 +253,10 @@ static int MPIDI_IPC_mpi_fd_init(MPIR_Comm * node_comm) if (node_comm->rank == 0) { mpi_errno = MPIDI_IPC_mpi_fd_cleanup(local_size, local_rank, all_pids, fd_socks); MPIR_ERR_CHECK(mpi_errno); - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); } else { - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); mpi_errno = MPIDI_IPC_mpi_fd_cleanup(local_size, local_rank, all_pids, fd_socks); MPIR_ERR_CHECK(mpi_errno); diff --git a/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c b/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c index 5283e9aac7d..7d29e252871 100644 --- a/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c +++ b/src/mpid/ch4/shm/posix/eager/iqueue/iqueue_init.c @@ -161,7 +161,7 @@ int MPIDI_POSIX_iqueue_set_vcis(void *slab, MPIR_Comm * comm, int max_vcis) MPIR_Comm *node_comm = MPIR_Comm_get_node_comm(comm); MPIR_Assert(node_comm); - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); fn_exit: diff --git a/src/mpid/ch4/shm/posix/posix_coll.h b/src/mpid/ch4/shm/posix/posix_coll.h index f45015d25f8..da671fa0191 100644 --- a/src/mpid/ch4/shm/posix/posix_coll.h +++ b/src/mpid/ch4/shm/posix/posix_coll.h @@ -147,1086 +147,4 @@ === END_MPI_T_CVAR_INFO_BLOCK === */ - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_barrier(MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__BARRIER, - .comm_ptr = comm, - }; - MPIDI_POSIX_csel_container_s *cnt; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_BARRIER_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_BARRIER_POSIX_INTRA_ALGORITHM_release_gather: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, !MPIR_IS_THREADED, mpi_errno, - "Barrier release_gather cannot be applied.\n"); - mpi_errno = MPIDI_POSIX_mpi_barrier_release_gather(comm, coll_attr); - break; - - case MPIR_CVAR_BARRIER_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - case MPIR_CVAR_BARRIER_POSIX_INTRA_ALGORITHM_auto: - cnt = MPIR_Csel_search(MPIDI_POSIX_COMM(comm, csel_comm), coll_sig); - if (cnt == NULL) - goto fallback; - - switch (cnt->id) { - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_barrier_release_gather: - mpi_errno = - MPIDI_POSIX_mpi_barrier_release_gather(comm, coll_attr); - break; - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Barrier_impl: - goto fallback; - default: - MPIR_Assert(0); - } - break; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Barrier_impl(comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_bcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__BCAST, - .comm_ptr = comm, - .u.bcast.buffer = buffer, - .u.bcast.count = count, - .u.bcast.datatype = datatype, - .u.bcast.root = root, - }; - MPIDI_POSIX_csel_container_s *cnt; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_BCAST_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_BCAST_POSIX_INTRA_ALGORITHM_release_gather: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, !MPIR_IS_THREADED, mpi_errno, - "Bcast release_gather cannot be applied.\n"); - mpi_errno = - MPIDI_POSIX_mpi_bcast_release_gather(buffer, count, datatype, root, comm, - coll_attr); - break; - - case MPIR_CVAR_BCAST_POSIX_INTRA_ALGORITHM_ipc_read: - mpi_errno = - MPIDI_POSIX_mpi_bcast_gpu_ipc_read(buffer, count, datatype, root, comm, coll_attr); - break; - - case MPIR_CVAR_BCAST_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - case MPIR_CVAR_BCAST_POSIX_INTRA_ALGORITHM_auto: - if (MPIR_CVAR_COLL_HYBRID_MEMORY) { - cnt = MPIR_Csel_search(MPIDI_POSIX_COMM(comm, csel_comm), coll_sig); - } else { - /* In no hybird case, local memory type can be used to select algorithm */ - MPL_pointer_attr_t pointer_attr; - MPIR_GPU_query_pointer_attr(buffer, &pointer_attr); - if (MPL_gpu_attr_is_strict_dev(&pointer_attr)) { - cnt = MPIR_Csel_search(MPIDI_POSIX_COMM(comm, csel_comm_gpu), coll_sig); - } else { - cnt = MPIR_Csel_search(MPIDI_POSIX_COMM(comm, csel_comm), coll_sig); - } - } - if (cnt == NULL) - goto fallback; - - switch (cnt->id) { - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_bcast_release_gather: - mpi_errno = - MPIDI_POSIX_mpi_bcast_release_gather(buffer, count, datatype, root, comm, - coll_attr); - break; - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_bcast_ipc_read: - mpi_errno = - MPIDI_POSIX_mpi_bcast_gpu_ipc_read(buffer, count, datatype, root, comm, - coll_attr); - break; - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Bcast_impl: - goto fallback; - default: - MPIR_Assert(0); - } - break; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLREDUCE, - .comm_ptr = comm, - .u.allreduce.sendbuf = sendbuf, - .u.allreduce.recvbuf = recvbuf, - .u.allreduce.count = count, - .u.allreduce.datatype = datatype, - .u.allreduce.op = op, - }; - MPIDI_POSIX_csel_container_s *cnt; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_ALLREDUCE_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_ALLREDUCE_POSIX_INTRA_ALGORITHM_release_gather: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, !MPIR_IS_THREADED && - MPIR_Op_is_commutative(op), mpi_errno, - "Allreduce release_gather cannot be applied.\n"); - mpi_errno = - MPIDI_POSIX_mpi_allreduce_release_gather(sendbuf, recvbuf, count, datatype, op, - comm, coll_attr); - break; - - case MPIR_CVAR_ALLREDUCE_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - case MPIR_CVAR_ALLREDUCE_POSIX_INTRA_ALGORITHM_auto: - cnt = MPIR_Csel_search(MPIDI_POSIX_COMM(comm, csel_comm), coll_sig); - if (cnt == NULL) - goto fallback; - - switch (cnt->id) { - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_allreduce_release_gather: - mpi_errno = - MPIDI_POSIX_mpi_allreduce_release_gather(sendbuf, recvbuf, count, datatype, - op, comm, coll_attr); - break; - - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Allreduce_impl: - goto fallback; - - default: - MPIR_Assert(0); - } - break; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_ALLGATHER_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_ALLGATHER_POSIX_INTRA_ALGORITHM_ipc_read: - mpi_errno = MPIDI_POSIX_mpi_allgather_gpu_ipc_read(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, coll_attr); - break; - - case MPIR_CVAR_ALLGATHER_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_ALLGATHERV_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_ALLGATHERV_POSIX_INTRA_ALGORITHM_ipc_read: - mpi_errno = MPIDI_POSIX_mpi_allgatherv_gpu_ipc_read(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, - recvtype, comm, coll_attr); - break; - - case MPIR_CVAR_ALLGATHERV_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; - -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_scatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_ALLTOALL_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_ALLTOALL_POSIX_INTRA_ALGORITHM_ipc_read: - mpi_errno = MPIDI_POSIX_mpi_alltoall_gpu_ipc_read(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, coll_attr); - break; - - case MPIR_CVAR_ALLTOALL_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_reduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__REDUCE, - .comm_ptr = comm, - .u.reduce.sendbuf = sendbuf, - .u.reduce.recvbuf = recvbuf, - .u.reduce.count = count, - .u.reduce.datatype = datatype, - .u.reduce.op = op, - .u.reduce.root = root, - }; - MPIDI_POSIX_csel_container_s *cnt; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_REDUCE_POSIX_INTRA_ALGORITHM) { - case MPIR_CVAR_REDUCE_POSIX_INTRA_ALGORITHM_release_gather: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, !MPIR_IS_THREADED && - MPIR_Op_is_commutative(op), mpi_errno, - "Reduce release_gather cannot be applied.\n"); - mpi_errno = - MPIDI_POSIX_mpi_reduce_release_gather(sendbuf, recvbuf, count, datatype, op, root, - comm, coll_attr); - break; - - case MPIR_CVAR_REDUCE_POSIX_INTRA_ALGORITHM_mpir: - goto fallback; - - case MPIR_CVAR_REDUCE_POSIX_INTRA_ALGORITHM_auto: - cnt = MPIR_Csel_search(MPIDI_POSIX_COMM(comm, csel_comm), coll_sig); - if (cnt == NULL) - goto fallback; - - switch (cnt->id) { - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_reduce_release_gather: - mpi_errno = - MPIDI_POSIX_mpi_reduce_release_gather(sendbuf, recvbuf, count, datatype, op, - root, comm, coll_attr); - break; - - case MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIR_Reduce_impl: - goto fallback; - - default: - MPIR_Assert(0); - } - break; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = - MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_reduce_scatter_block(const void *sendbuf, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_scan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_exscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_ERR_CHECK(mpi_errno); - - MPIR_FUNC_EXIT; - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_neighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = - MPIR_Neighbor_allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_neighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_neighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_neighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_neighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ineighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ineighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ineighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ineighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ibarrier(MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ibarrier_impl(comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ibcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ibcast_impl(buffer, count, datatype, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ialltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ialltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ialltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iexscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iexscan_impl(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Igather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Igatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ireduce_scatter_block(const void *sendbuf, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = - MPIR_Ireduce_scatter_block_impl(sendbuf, recvbuf, recvcount, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_ireduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Ireduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iallreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iallreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscan_impl(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_iscatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** request) -{ - int mpi_errno; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Iscatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - #endif /* POSIX_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/shm/posix/posix_coll_gpu_ipc.h b/src/mpid/ch4/shm/posix/posix_coll_gpu_ipc.h index 296588d346e..80bfb35607f 100644 --- a/src/mpid/ch4/shm/posix/posix_coll_gpu_ipc.h +++ b/src/mpid/ch4/shm/posix/posix_coll_gpu_ipc.h @@ -99,10 +99,11 @@ static int allgather_ipc_handles(const void *buf, MPI_Aint count, MPI_Datatype d } /* allgather is needed to exchange all the IPC handles */ + /* FIXME: call MPIR_Coll_auto */ mpi_errno = - MPIR_Allgather_impl(&my_ipc_handle, sizeof(MPIDI_IPCI_ipc_handle_t), MPIR_BYTE_INTERNAL, - ipc_handles, sizeof(MPIDI_IPCI_ipc_handle_t), MPIR_BYTE_INTERNAL, comm, - MPIR_COLL_ATTR_SYNC); + MPIR_Allgather_fallback(&my_ipc_handle, sizeof(MPIDI_IPCI_ipc_handle_t), MPIR_BYTE_INTERNAL, + ipc_handles, sizeof(MPIDI_IPCI_ipc_handle_t), MPIR_BYTE_INTERNAL, + comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* check the ipc_handles to make sure all the buffers are on GPU */ @@ -186,9 +187,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_bcast_gpu_ipc_read(void *buffer, fn_fail: goto fn_exit; fallback: - /* Fall back to other algorithms as gpu ipc bcast cannot be used */ - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } @@ -277,10 +277,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_alltoall_gpu_ipc_read(const void *s fn_fail: goto fn_exit; fallback: - /* Fall back to other algorithms as gpu ipc alltoall cannot be used */ - mpi_errno = MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } @@ -369,10 +367,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allgather_gpu_ipc_read(const void * fn_fail: goto fn_exit; fallback: - /* Fall back to other algorithms as gpu ipc allgather cannot be used */ - mpi_errno = MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } @@ -466,62 +462,10 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allgatherv_gpu_ipc_read(const void fn_fail: goto fn_exit; fallback: - /* Fall back to other algorithms as gpu ipc allgatherv cannot be used */ - mpi_errno = MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, - recvtype, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ goto fn_exit; } -#else /* MPIDI_CH4_SHM_ENABLE_GPU */ -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_bcast_gpu_ipc_read(void *buffer, - MPI_Aint count, - MPI_Datatype datatype, - int root, MPIR_Comm * comm_ptr, - int coll_attr) -{ - return MPIR_Bcast_impl(buffer, count, datatype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_alltoall_gpu_ipc_read(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - int coll_attr) -{ - return MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allgather_gpu_ipc_read(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - int coll_attr) -{ - return MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allgatherv_gpu_ipc_read(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - int coll_attr) -{ - return MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, - comm_ptr, coll_attr); -} #endif /* !MPIDI_CH4_SHM_ENABLE_GPU */ #endif /* POSIX_COLL_GPU_IPC_H_INCLUDED */ diff --git a/src/mpid/ch4/shm/posix/posix_coll_release_gather.h b/src/mpid/ch4/shm/posix/posix_coll_release_gather.h index e26f8ffdbe0..40a0612b256 100644 --- a/src/mpid/ch4/shm/posix/posix_coll_release_gather.h +++ b/src/mpid/ch4/shm/posix/posix_coll_release_gather.h @@ -151,9 +151,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_bcast_release_gather(void *buffer, fn_fail: goto fn_exit; fallback: - /* Fall back to other algo as release_gather based bcast cannot be used */ - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } @@ -250,9 +249,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_reduce_release_gather(const void *s fn_fail: goto fn_exit; fallback: - /* Fall back to other algo as release_gather algo cannot be used */ - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } @@ -346,8 +344,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_allreduce_release_gather(const void goto fn_exit; fallback: - mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } @@ -395,8 +393,8 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_POSIX_mpi_barrier_release_gather(MPIR_Comm * goto fn_exit; fallback: - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); + /* FIXME: proper error */ + mpi_errno = MPI_ERR_OTHER; goto fn_exit; } diff --git a/src/mpid/ch4/shm/posix/posix_comm.c b/src/mpid/ch4/shm/posix/posix_comm.c index 1e40b32f8ff..f6d9e1e4841 100644 --- a/src/mpid/ch4/shm/posix/posix_comm.c +++ b/src/mpid/ch4/shm/posix/posix_comm.c @@ -83,24 +83,6 @@ int MPIDI_POSIX_mpi_comm_commit_post_hook(MPIR_Comm * comm) } } - /* prune selection tree */ - if (MPIDI_global.shm.posix.csel_root) { - mpi_errno = MPIR_Csel_prune(MPIDI_global.shm.posix.csel_root, comm, - &MPIDI_POSIX_COMM(comm, csel_comm)); - MPIR_ERR_CHECK(mpi_errno); - } else { - MPIDI_POSIX_COMM(comm, csel_comm) = NULL; - } - - /* prune selection tree for gpu */ - if (MPIDI_global.shm.posix.csel_root_gpu) { - mpi_errno = MPIR_Csel_prune(MPIDI_global.shm.posix.csel_root_gpu, comm, - &MPIDI_POSIX_COMM(comm, csel_comm_gpu)); - MPIR_ERR_CHECK(mpi_errno); - } else { - MPIDI_POSIX_COMM(comm, csel_comm_gpu) = NULL; - } - fn_exit: MPIR_CHKLMEM_FREEALL(); MPIR_FUNC_EXIT; @@ -114,16 +96,6 @@ int MPIDI_POSIX_mpi_comm_free_hook(MPIR_Comm * comm) int mpi_errno = MPI_SUCCESS; MPIR_FUNC_ENTER; - if (MPIDI_POSIX_COMM(comm, csel_comm)) { - mpi_errno = MPIR_Csel_free(MPIDI_POSIX_COMM(comm, csel_comm)); - MPIR_ERR_CHECK(mpi_errno); - } - - if (MPIDI_POSIX_COMM(comm, csel_comm_gpu)) { - mpi_errno = MPIR_Csel_free(MPIDI_POSIX_COMM(comm, csel_comm_gpu)); - MPIR_ERR_CHECK(mpi_errno); - } - /* Release_gather primitives based collective algorithm works for Intra-comms only */ if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) { mpi_errno = MPIDI_POSIX_mpi_release_gather_comm_free(comm); diff --git a/src/mpid/ch4/shm/posix/posix_init.c b/src/mpid/ch4/shm/posix/posix_init.c index 16877a8cb94..7fce875de77 100644 --- a/src/mpid/ch4/shm/posix/posix_init.c +++ b/src/mpid/ch4/shm/posix/posix_init.c @@ -121,39 +121,6 @@ static int choose_posix_eager(void) goto fn_exit; } -static void *create_container(struct json_object *obj) -{ - MPIDI_POSIX_csel_container_s *cnt = - MPL_malloc(sizeof(MPIDI_POSIX_csel_container_s), MPL_MEM_COLL); - - json_object_object_foreach(obj, key, val) { - char *ckey = MPL_strdup_no_spaces(key); - - if (!strcmp(ckey, "algorithm=MPIDI_POSIX_mpi_bcast_release_gather")) - cnt->id = - MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_bcast_release_gather; - else if (!strcmp(ckey, "algorithm=MPIDI_POSIX_mpi_bcast_ipc_read")) - cnt->id = MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_bcast_ipc_read; - else if (!strcmp(ckey, "algorithm=MPIDI_POSIX_mpi_barrier_release_gather")) - cnt->id = - MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_barrier_release_gather; - else if (!strcmp(ckey, "algorithm=MPIDI_POSIX_mpi_allreduce_release_gather")) - cnt->id = - MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_allreduce_release_gather; - else if (!strcmp(ckey, "algorithm=MPIDI_POSIX_mpi_reduce_release_gather")) - cnt->id = - MPIDI_POSIX_CSEL_CONTAINER_TYPE__ALGORITHM__MPIDI_POSIX_mpi_reduce_release_gather; - else { - fprintf(stderr, "unrecognized key %s\n", key); - MPIR_Assert(0); - } - - MPL_free(ckey); - } - - return cnt; -} - int MPIDI_POSIX_init_vci(int vci) { int mpi_errno = MPI_SUCCESS; @@ -414,38 +381,8 @@ static int posix_coll_init(void) int mpi_errno = MPI_SUCCESS; MPIR_FUNC_ENTER; - /* Initialize collective selection */ - if (!strcmp(MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE, "")) { - mpi_errno = MPIR_Csel_create_from_buf(MPIDI_POSIX_coll_generic_json, - create_container, &MPIDI_global.shm.posix.csel_root); - MPIDI_global.shm.posix.csel_source = "MPIDI_POSIX_coll_generic_json"; - } else { - mpi_errno = MPIR_Csel_create_from_file(MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE, - create_container, &MPIDI_global.shm.posix.csel_root); - MPIDI_global.shm.posix.csel_source = MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE; - } - MPIR_ERR_CHECK(mpi_errno); - - /* Initialize collective selection for gpu */ - if (!strcmp(MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE_GPU, "")) { - mpi_errno = MPIR_Csel_create_from_buf(MPIDI_POSIX_coll_generic_json, - create_container, - &MPIDI_global.shm.posix.csel_root_gpu); - MPIDI_global.shm.posix.csel_source_gpu = "MPIDI_POSIX_coll_generic_json"; - } else { - mpi_errno = - MPIR_Csel_create_from_file(MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE_GPU, - create_container, &MPIDI_global.shm.posix.csel_root_gpu); - MPIDI_global.shm.posix.csel_source_gpu = - MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE_GPU; - } - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: MPIR_FUNC_EXIT; return mpi_errno; - fn_fail: - goto fn_exit; } static int posix_coll_finalize(void) @@ -460,21 +397,7 @@ static int posix_coll_finalize(void) * avoid segmentation fault */ MPIDI_POSIX_shm_limit_counter = &MPIDI_POSIX_dummy_shm_limit_counter; - if (MPIDI_global.shm.posix.csel_root) { - mpi_errno = MPIR_Csel_free(MPIDI_global.shm.posix.csel_root); - MPIR_ERR_CHECK(mpi_errno); - } - - if (MPIDI_global.shm.posix.csel_root_gpu) { - mpi_errno = MPIR_Csel_free(MPIDI_global.shm.posix.csel_root_gpu); - MPIR_ERR_CHECK(mpi_errno); - } - - fn_exit: - MPIR_FUNC_EXIT; return mpi_errno; - fn_fail: - goto fn_exit; } void *MPIDI_POSIX_mpi_alloc_mem(MPI_Aint size, MPIR_Info * info_ptr) diff --git a/src/mpid/ch4/shm/posix/posix_pre.h b/src/mpid/ch4/shm/posix/posix_pre.h index 293bf2d5cca..0bbda3a783a 100644 --- a/src/mpid/ch4/shm/posix/posix_pre.h +++ b/src/mpid/ch4/shm/posix/posix_pre.h @@ -28,19 +28,12 @@ typedef enum { struct MPIR_Request; typedef struct { - void *csel_root; - const char *csel_source; - void *csel_root_gpu; - const char *csel_source_gpu; + /* empty */ } MPIDI_POSIX_Global_t; -extern char MPIDI_POSIX_coll_generic_json[]; - /* These structs are populated with dummy variables because empty structs are not supported in all * compilers: https://stackoverflow.com/a/755339/491687 */ typedef struct { - void *csel_comm; - void *csel_comm_gpu; MPIDI_POSIX_release_gather_comm_t release_gather, nb_release_gather; int nb_bcast_seq_no; /* Seq number of the release-gather based nonblocking bcast call */ int nb_reduce_seq_no; /* Seq number of the release-gather based nonblocking reduce call */ diff --git a/src/mpid/ch4/shm/posix/posix_vci.c b/src/mpid/ch4/shm/posix/posix_vci.c index 37ee06cfeee..d329342ec4f 100644 --- a/src/mpid/ch4/shm/posix/posix_vci.c +++ b/src/mpid/ch4/shm/posix/posix_vci.c @@ -17,8 +17,8 @@ int MPIDI_POSIX_comm_set_vcis(MPIR_Comm * comm, int num_vcis) MPIR_Assert(node_comm); int max_vcis; - mpi_errno = MPIR_Allreduce_impl(&num_vcis, &max_vcis, 1, MPIR_INT_INTERNAL, MPI_MAX, - node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allreduce_fallback(&num_vcis, &max_vcis, 1, MPIR_INT_INTERNAL, MPI_MAX, + node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (max_vcis > 1) { @@ -38,7 +38,7 @@ int MPIDI_POSIX_comm_set_vcis(MPIR_Comm * comm, int num_vcis) slab = MPL_initshm_open(MPIDI_POSIX_global.shm_vci_name, slab_size, NULL); MPIR_ERR_CHKANDJUMP(!slab, mpi_errno, MPI_ERR_OTHER, "**nomem"); - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); #else mpi_errno = MPIDU_Init_shm_comm_alloc(comm, slab_size, (void *) &slab); @@ -49,7 +49,7 @@ int MPIDI_POSIX_comm_set_vcis(MPIR_Comm * comm, int num_vcis) mpi_errno = MPIDI_POSIX_eager_set_vcis(MPIDI_POSIX_global.shm_vci_slab, comm, max_vcis); MPIR_ERR_CHECK(mpi_errno); - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); } diff --git a/src/mpid/ch4/shm/posix/release_gather/nb_release_gather.c b/src/mpid/ch4/shm/posix/release_gather/nb_release_gather.c index b26f9e4ce2a..62c4e4d8d81 100644 --- a/src/mpid/ch4/shm/posix/release_gather/nb_release_gather.c +++ b/src/mpid/ch4/shm/posix/release_gather/nb_release_gather.c @@ -121,18 +121,19 @@ int MPIDI_POSIX_nb_release_gather_comm_init(MPIR_Comm * comm_ptr, to other algorithms.\n"); } fallback = 1; - MPIR_Bcast_impl(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); + MPIR_Bcast_fallback(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_NO_MEM, "**nomem"); } else { /* More shm can be created, update the shared counter */ MPL_atomic_fetch_add_uint64(MPIDI_POSIX_shm_limit_counter, memory_to_be_allocated); fallback = 0; mpi_errno = - MPIR_Bcast_impl(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); + MPIR_Bcast_fallback(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } } else { - mpi_errno = MPIR_Bcast_impl(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); + mpi_errno = + MPIR_Bcast_fallback(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); if (fallback) { MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_NO_MEM, "**nomem"); @@ -168,8 +169,8 @@ int MPIDI_POSIX_nb_release_gather_comm_init(MPIR_Comm * comm_ptr, topotree_fail[0] = -1; topotree_fail[1] = -1; } - mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, topotree_fail, 2, MPIR_INT_INTERNAL, - MPI_MAX, comm_ptr, coll_attr); + mpi_errno = MPIR_Allreduce_fallback(MPI_IN_PLACE, topotree_fail, 2, MPIR_INT_INTERNAL, + MPI_MAX, comm_ptr, coll_attr); } else { topotree_fail[0] = -1; topotree_fail[1] = -1; @@ -267,7 +268,7 @@ int MPIDI_POSIX_nb_release_gather_comm_init(MPIR_Comm * comm_ptr, if (initialize_ibcast_buf || initialize_ireduce_buf) { /* Make sure all the flags are set before ranks start reading each other's flags from shm */ - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } diff --git a/src/mpid/ch4/shm/posix/release_gather/release_gather.c b/src/mpid/ch4/shm/posix/release_gather/release_gather.c index ce79beac81b..15c12b80237 100644 --- a/src/mpid/ch4/shm/posix/release_gather/release_gather.c +++ b/src/mpid/ch4/shm/posix/release_gather/release_gather.c @@ -303,18 +303,19 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr, } fallback = 1; - MPIR_Bcast_impl(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); + MPIR_Bcast_fallback(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_NO_MEM, "**nomem"); } else { /* More shm can be created, update the shared counter */ MPL_atomic_fetch_add_uint64(MPIDI_POSIX_shm_limit_counter, memory_to_be_allocated); fallback = 0; mpi_errno = - MPIR_Bcast_impl(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); + MPIR_Bcast_fallback(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } } else { - mpi_errno = MPIR_Bcast_impl(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); + mpi_errno = + MPIR_Bcast_fallback(&fallback, 1, MPIR_INT_INTERNAL, 0, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); if (fallback) { MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_NO_MEM, "**nomem"); @@ -359,8 +360,8 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr, topotree_fail[0] = -1; topotree_fail[1] = -1; } - mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, topotree_fail, 2, MPIR_INT_INTERNAL, - MPI_MAX, comm_ptr, coll_attr); + mpi_errno = MPIR_Allreduce_fallback(MPI_IN_PLACE, topotree_fail, 2, MPIR_INT_INTERNAL, + MPI_MAX, comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } else { topotree_fail[0] = -1; @@ -424,7 +425,7 @@ int MPIDI_POSIX_mpi_release_gather_comm_init(MPIR_Comm * comm_ptr, release_gather_info_ptr->release_state); /* Make sure all the flags are set before ranks start reading each other's flags from shm */ - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); } diff --git a/src/mpid/ch4/shm/src/Makefile.mk b/src/mpid/ch4/shm/src/Makefile.mk index 77995e3d9ae..0445e225c75 100644 --- a/src/mpid/ch4/shm/src/Makefile.mk +++ b/src/mpid/ch4/shm/src/Makefile.mk @@ -7,14 +7,12 @@ AM_CPPFLAGS += -I$(top_srcdir)/src/mpid/ch4/shm/src noinst_HEADERS += src/mpid/ch4/shm/src/shm_impl.h \ src/mpid/ch4/shm/src/shm_am_fallback.h \ - src/mpid/ch4/shm/src/shm_am_fallback_coll.h \ src/mpid/ch4/shm/src/shm_am_fallback_probe.h \ src/mpid/ch4/shm/src/shm_am_fallback_recv.h \ src/mpid/ch4/shm/src/shm_am_fallback_rma.h \ src/mpid/ch4/shm/src/shm_am_fallback_send.h \ src/mpid/ch4/shm/src/shm_am_fallback_part.h \ src/mpid/ch4/shm/src/shm_am.h \ - src/mpid/ch4/shm/src/shm_coll.h \ src/mpid/ch4/shm/src/shm_hooks.h \ src/mpid/ch4/shm/src/shm_progress.h \ src/mpid/ch4/shm/src/shm_p2p.h \ diff --git a/src/mpid/ch4/shm/src/shm_am_fallback.h b/src/mpid/ch4/shm/src/shm_am_fallback.h index 83ceb6bd8fb..31a9d4e17cc 100644 --- a/src/mpid/ch4/shm/src/shm_am_fallback.h +++ b/src/mpid/ch4/shm/src/shm_am_fallback.h @@ -10,7 +10,6 @@ #include "shm_am_fallback_recv.h" #include "shm_am_fallback_probe.h" #include "shm_am_fallback_rma.h" -#include "shm_am_fallback_coll.h" #include "shm_am_fallback_part.h" #endif /* SHM_AM_FALLBACK_H_INCLUDED */ diff --git a/src/mpid/ch4/shm/src/shm_am_fallback_coll.h b/src/mpid/ch4/shm/src/shm_am_fallback_coll.h deleted file mode 100644 index e6d0f5c6b86..00000000000 --- a/src/mpid/ch4/shm/src/shm_am_fallback_coll.h +++ /dev/null @@ -1,450 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#ifndef SHM_AM_FALLBACK_COLL_H_INCLUDED -#define SHM_AM_FALLBACK_COLL_H_INCLUDED - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_barrier(MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Barrier_impl(comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_bcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Bcast_impl(buffer, count, datatype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_allreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - int coll_attr) -{ - return MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_scatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - int coll_attr) -{ - return MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_reduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, - coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_reduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_scan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_exscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - return MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr) -{ - return MPIR_Neighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_allgather_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_allgatherv_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ineighbor_alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ibarrier(MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ibarrier_impl(comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ibcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ibcast_impl(buffer, count, datatype, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Iallgather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Iallgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iallreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** request) -{ - return MPIR_Iallreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, request); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ialltoall_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ialltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ialltoallv_impl(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ialltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Ialltoallw_impl(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iexscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Iexscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Igather_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Igatherv_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ireduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ireduce_scatter_block_impl(sendbuf, recvbuf, recvcount, - datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ireduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ireduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - return MPIR_Ireduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - return MPIR_Iscan_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, req); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - MPIR_Request ** request) -{ - return MPIR_Iscatter_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, request); -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iscatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Request ** request) -{ - return MPIR_Iscatterv_impl(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, request); -} - -#endif /* SHM_AM_FALLBACK_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/shm/src/shm_coll.h b/src/mpid/ch4/shm/src/shm_coll.h deleted file mode 100644 index e1ba0315124..00000000000 --- a/src/mpid/ch4/shm/src/shm_coll.h +++ /dev/null @@ -1,755 +0,0 @@ -/* - * Copyright (C) by Argonne National Laboratory - * See COPYRIGHT in top-level directory - */ - -#ifndef SHM_COLL_H_INCLUDED -#define SHM_COLL_H_INCLUDED - -#include -#include "../posix/shm_inline.h" - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_barrier(MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_barrier(comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_bcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, MPIR_Comm * comm, - int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_bcast(buffer, count, datatype, root, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_allreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_allgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_scatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_scatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_gatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, root, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm, - int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_alltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, - recvcounts, rdispls, recvtypes, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_reduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint * recvcounts, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_reduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, - comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_reduce_scatter_block(const void *sendbuf, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_reduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, - op, comm_ptr, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_scan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_scan(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_exscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, int coll_attr) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_exscan(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_neighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_neighbor_allgatherv(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_neighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_alltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype * sendtypes, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype * recvtypes, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_neighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_neighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_neighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_allgather(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_allgatherv(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ineighbor_allgatherv(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_alltoall(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ineighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ineighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype * sendtypes, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype * recvtypes, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ineighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ibarrier(MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ibarrier(comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ibcast(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ibcast(buffer, count, datatype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iallgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iallreduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iallreduce(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ialltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ialltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ialltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ialltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, - recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iexscan(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iexscan(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ireduce_scatter_block(const void *sendbuf, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, - op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint * recvcounts, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_ireduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_ireduce(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iscan(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_SHM_mpi_iscatterv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, - MPIR_Comm * comm_ptr, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_POSIX_mpi_iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, - recvcount, recvtype, root, comm_ptr, req); - - MPIR_FUNC_EXIT; - return ret; -} - -#endif /* SHM_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/shm/src/shm_impl.h b/src/mpid/ch4/shm/src/shm_impl.h index dcb424154a7..df3adf9de55 100644 --- a/src/mpid/ch4/shm/src/shm_impl.h +++ b/src/mpid/ch4/shm/src/shm_impl.h @@ -12,17 +12,17 @@ #ifndef SHM_IMPL_H_INCLUDED #define SHM_IMPL_H_INCLUDED +#include "shm_am.h" + #ifdef MPIDI_ENABLE_AM_ONLY #include "shm_am_fallback.h" #else -#include "shm_coll.h" #include "shm_p2p.h" #include "shm_rma.h" #include "shm_part.h" #include "shm_hooks.h" #endif -#include "shm_am.h" #include "shm_progress.h" #include "shm_hooks_internal.h" diff --git a/src/mpid/ch4/shm/src/topotree.c b/src/mpid/ch4/shm/src/topotree.c index 2569617c042..debdc65f6fb 100644 --- a/src/mpid/ch4/shm/src/topotree.c +++ b/src/mpid/ch4/shm/src/topotree.c @@ -499,7 +499,7 @@ int MPIDI_SHM_topology_tree_init(MPIR_Comm * comm_ptr, int root, int bcast_k, in shared_region_ptr[rank][depth++] = MPIR_hwtopo_get_lid(gid); gid = MPIR_hwtopo_get_ancestor(gid, topo_depth - depth - 1); } - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* STEP 3. Root has all the bind_map information, now build tree */ @@ -557,7 +557,7 @@ int MPIDI_SHM_topology_tree_init(MPIR_Comm * comm_ptr, int root, int bcast_k, in 0 /*left_skewed */ , bcast_tree_type); MPIR_ERR_CHECK(mpi_errno); } - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* Every rank copies their tree out from shared memory */ @@ -566,7 +566,7 @@ int MPIDI_SHM_topology_tree_init(MPIR_Comm * comm_ptr, int root, int bcast_k, in MPIDI_SHM_print_topotree_file("BCAST", comm_ptr->context_id, rank, bcast_tree); /* Wait until shared memory is available */ - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* Generate the reduce tree */ /* For Reduce, package leaders are added after the package local ranks, and the per_package @@ -580,7 +580,7 @@ int MPIDI_SHM_topology_tree_init(MPIR_Comm * comm_ptr, int root, int bcast_k, in MPIR_ERR_CHECK(mpi_errno); } - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* each rank copy the reduce tree out */ @@ -589,7 +589,7 @@ int MPIDI_SHM_topology_tree_init(MPIR_Comm * comm_ptr, int root, int bcast_k, in if (MPIDI_SHM_TOPOTREE_DEBUG) MPIDI_SHM_print_topotree_file("REDUCE", comm_ptr->context_id, rank, reduce_tree); /* Wait for all ranks to copy out the tree */ - mpi_errno = MPIR_Barrier_impl(comm_ptr, coll_attr); + mpi_errno = MPIR_Barrier_fallback(comm_ptr, coll_attr); MPIR_ERR_CHECK(mpi_errno); /* Cleanup */ if (rank == root) { diff --git a/src/mpid/ch4/src/ch4_coll.h b/src/mpid/ch4/src/ch4_coll.h index c16fceaa9bc..d497fab8f26 100644 --- a/src/mpid/ch4/src/ch4_coll.h +++ b/src/mpid/ch4/src/ch4_coll.h @@ -99,211 +99,6 @@ === END_MPI_T_CVAR_INFO_BLOCK === */ -MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_allcomm_composition_json(MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__BARRIER, - .comm_ptr = comm, - }; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Barrier_impl(comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Barrier_intra_composition_alpha: - mpi_errno = MPIDI_Barrier_intra_composition_alpha(comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Barrier_intra_composition_beta: - mpi_errno = MPIDI_Barrier_intra_composition_beta(comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - - switch (MPIR_CVAR_BARRIER_COMPOSITION) { - case 1: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && - MPIR_Comm_is_parent_comm(comm), mpi_errno, - "Barrier composition alpha cannot be applied.\n"); - mpi_errno = MPIDI_Barrier_intra_composition_alpha(comm, coll_attr); - break; - case 2: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Barrier composition beta cannot be applied.\n"); - mpi_errno = MPIDI_Barrier_intra_composition_beta(comm, coll_attr); - break; - default: - mpi_errno = MPIDI_Barrier_allcomm_composition_json(comm, coll_attr); - break; - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = MPIR_Barrier_impl(comm, coll_attr); - else - mpi_errno = MPIDI_Barrier_intra_composition_beta(comm, coll_attr); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_allcomm_composition_json(void *buffer, MPI_Aint count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__BCAST, - .comm_ptr = comm, - .u.bcast.buffer = buffer, - .u.bcast.count = count, - .u.bcast.datatype = datatype, - .u.bcast.root = root, - }; - - const MPIDI_Csel_container_s *cnt = NULL; - - if (MPIR_CVAR_COLL_HYBRID_MEMORY) { - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - } else { - /* In no hybird case, local memory type can be used to select algorithm */ - MPL_pointer_attr_t pointer_attr; - MPIR_GPU_query_pointer_attr(buffer, &pointer_attr); - if (MPL_gpu_attr_is_strict_dev(&pointer_attr)) { - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm_gpu), coll_sig); - } else { - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - } - } - if (cnt == NULL) { - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_alpha: - mpi_errno = - MPIDI_Bcast_intra_composition_alpha(buffer, count, datatype, root, comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_beta: - mpi_errno = - MPIDI_Bcast_intra_composition_beta(buffer, count, datatype, root, comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_gamma: - mpi_errno = - MPIDI_Bcast_intra_composition_gamma(buffer, count, datatype, root, comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_delta: - mpi_errno = - MPIDI_Bcast_intra_composition_delta(buffer, count, datatype, root, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_BCAST_COMPOSITION) { - case 1: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && - MPIR_Comm_is_parent_comm(comm), mpi_errno, - "Bcast composition alpha cannot be applied.\n"); - mpi_errno = - MPIDI_Bcast_intra_composition_alpha(buffer, count, datatype, root, comm, coll_attr); - break; - case 2: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && - MPIR_Comm_is_parent_comm(comm), mpi_errno, - "Bcast composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Bcast_intra_composition_beta(buffer, count, datatype, root, comm, coll_attr); - break; - case 3: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Bcast composition gamma cannot be applied.\n"); - mpi_errno = - MPIDI_Bcast_intra_composition_gamma(buffer, count, datatype, root, comm, coll_attr); - break; - case 4: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && - MPIR_Comm_is_parent_comm(comm), mpi_errno, - "Bcast composition delta cannot be applied.\n"); - mpi_errno = - MPIDI_Bcast_intra_composition_delta(buffer, count, datatype, root, comm, coll_attr); - break; - default: - mpi_errno = - MPIDI_Bcast_allcomm_composition_json(buffer, count, datatype, root, comm, - coll_attr); - break; - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = MPIR_Bcast_impl(buffer, count, datatype, root, comm, coll_attr); - else - mpi_errno = - MPIDI_Bcast_intra_composition_gamma(buffer, count, datatype, root, comm, coll_attr); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - MPL_STATIC_INLINE_PREFIX void MPIDI_Allreduce_fill_multi_leads_info(MPIR_Comm * comm) { if (MPIDI_COMM(comm, allreduce_comp_info) == NULL) { @@ -326,178 +121,6 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_Allreduce_fill_multi_leads_info(MPIR_Comm * } -MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_allcomm_composition_json(const void *sendbuf, - void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - int num_leads = 0, node_comm_size = 0; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLREDUCE, - .comm_ptr = comm, - - .u.allreduce.sendbuf = sendbuf, - .u.allreduce.recvbuf = recvbuf, - .u.allreduce.count = count, - .u.allreduce.datatype = datatype, - .u.allreduce.op = op, - }; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_alpha: - mpi_errno = - MPIDI_Allreduce_intra_composition_alpha(sendbuf, recvbuf, count, datatype, op, - comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_beta: - mpi_errno = - MPIDI_Allreduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, - comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_gamma: - mpi_errno = - MPIDI_Allreduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, - comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_delta: - if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) { - MPIDI_Allreduce_fill_multi_leads_info(comm); - if (comm->node_comm) - node_comm_size = MPIR_Comm_size(comm->node_comm); - /* Reset number of leaders, so that (node_comm_size % num_leads) is zero. The new number of - * leaders must lie within a range +/- from the leaders specified, or every rank is made - * as a leader. Currently we use range as 15. */ - num_leads = - MPL_round_closest_multiple(node_comm_size, MPIR_CVAR_NUM_MULTI_LEADS, 15); - } - /* make sure that the algo can be run */ - if (MPIDI_COMM_ALLREDUCE(comm, use_multi_leads) == 1 && - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM && - count >= num_leads && MPIR_Op_is_commutative(op)) { - mpi_errno = - MPIDI_Allreduce_intra_composition_delta(sendbuf, recvbuf, count, datatype, op, - num_leads, comm, coll_attr); - } else - mpi_errno = - MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - int is_commutative = -1; - int num_leads = 0, node_comm_size = 0; - - MPIR_FUNC_ENTER; - - is_commutative = MPIR_Op_is_commutative(op); - - switch (MPIR_CVAR_ALLREDUCE_COMPOSITION) { - case 1: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && - MPIR_Comm_is_parent_comm(comm) && - is_commutative, mpi_errno, - "Allreduce composition alpha cannot be applied.\n"); - mpi_errno = - MPIDI_Allreduce_intra_composition_alpha(sendbuf, recvbuf, count, datatype, op, comm, - coll_attr); - break; - case 2: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Allreduce composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Allreduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, comm, - coll_attr); - break; - case 3: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) && - (comm->node_comm != NULL) && - (MPIR_Comm_size(comm) == - MPIR_Comm_size(comm->node_comm)), mpi_errno, - "Allreduce composition gamma cannot be applied.\n"); - mpi_errno = - MPIDI_Allreduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, comm, - coll_attr); - break; - case 4: - if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) { - MPIDI_Allreduce_fill_multi_leads_info(comm); - if (comm->node_comm) - node_comm_size = MPIR_Comm_size(comm->node_comm); - /* Reset number of leaders, so that (node_comm_size % num_leads) is zero. The new number of - * leaders must lie within a range +/- from the leaders specified, or every rank is made - * as a leader. Currently we use range as 15. */ - num_leads = - MPL_round_closest_multiple(node_comm_size, MPIR_CVAR_NUM_MULTI_LEADS, 15); - } - - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM - && MPIDI_COMM_ALLREDUCE(comm, use_multi_leads) == 1 && - count >= num_leads && is_commutative, mpi_errno, - "Allreduce composition delta cannot be applied.\n"); - /* Multi-leaders based composition can only be used if the comm is spanned over more than - * 1 node, has equal number of ranks on each node, count is more than number of leaders and - * the operation is commutative. This composition is beneficial for large messages. - */ - - mpi_errno = - MPIDI_Allreduce_intra_composition_delta(sendbuf, recvbuf, count, datatype, op, - num_leads, comm, coll_attr); - break; - - default: - mpi_errno = - MPIDI_Allreduce_allcomm_composition_json(sendbuf, recvbuf, count, datatype, op, - comm, coll_attr); - break; - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = MPIR_Allreduce_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - else - mpi_errno = - MPIDI_Allreduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, comm, - coll_attr); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - MPL_STATIC_INLINE_PREFIX void MPIDI_Allgather_fill_multi_leads_info(MPIR_Comm * comm) { if (MPIDI_COMM(comm, allgather_comp_info) == NULL) { @@ -516,430 +139,6 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_Allgather_fill_multi_leads_info(MPIR_Comm * } } -MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_allcomm_composition_json(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size, data_size; - const MPIDI_Csel_container_s *cnt = NULL; - - if (sendbuf != MPI_IN_PLACE) { - MPIR_Datatype_get_size_macro(sendtype, type_size); - data_size = sendcount * type_size; - } else { - MPIR_Datatype_get_size_macro(recvtype, type_size); - data_size = recvcount * type_size; - } - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLGATHER, - .comm_ptr = comm, - - .u.allgather.sendbuf = sendbuf, - .u.allgather.sendcount = sendcount, - .u.allgather.sendtype = sendtype, - .u.allgather.recvbuf = recvbuf, - .u.allgather.recvcount = recvcount, - .u.allgather.recvtype = recvtype, - }; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = - MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, - coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_alpha: - /* make sure that the algo can be run */ - if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) - MPIDI_Allgather_fill_multi_leads_info(comm); - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - MPIDI_COMM_ALLGATHER(comm, use_multi_leads) == 1 && - data_size <= MPIR_CVAR_ALLGATHER_SHM_PER_RANK, mpi_errno, - "Allgather composition alpha cannot be applied.\n"); - mpi_errno = - MPIDI_Allgather_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_beta: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Allgather composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, - coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = - MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, - coll_attr); - else - mpi_errno = - MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, coll_attr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size, data_size; - - MPIR_FUNC_ENTER; - - if (sendbuf != MPI_IN_PLACE) { - MPIR_Datatype_get_size_macro(sendtype, type_size); - data_size = sendcount * type_size; - } else { - MPIR_Datatype_get_size_macro(recvtype, type_size); - data_size = recvcount * type_size; - } - - switch (MPIR_CVAR_ALLGATHER_COMPOSITION) { - case 1: - if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) - MPIDI_Allgather_fill_multi_leads_info(comm); - - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - MPIDI_COMM_ALLGATHER(comm, use_multi_leads) == 1 && - data_size <= MPIR_CVAR_ALLGATHER_SHM_PER_RANK, mpi_errno, - "Allgather composition alpha cannot be applied.\n"); - /* Multi-leaders based composition can only be used if the comm is spanned over more than - * 1 node, has equal number of ranks on each node, ranks on a node are consecutive and - * the combined msg from all the ranks on a node fits the allocated shared memory buffer - */ - - mpi_errno = - MPIDI_Allgather_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, - comm, coll_attr); - break; - case 2: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Allgather composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr); - break; - default: - mpi_errno = MPIDI_Allgather_allcomm_composition_json(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, - coll_attr); - break; - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = - MPIR_Allgather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, - coll_attr); - else - mpi_errno = - MPIDI_Allgather_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, coll_attr); - - fn_exit: - MPIR_FUNC_ENTER; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLGATHERV, - .comm_ptr = comm, - - .u.allgatherv.sendbuf = sendbuf, - .u.allgatherv.sendcount = sendcount, - .u.allgatherv.sendtype = sendtype, - .u.allgatherv.recvbuf = recvbuf, - .u.allgatherv.recvcounts = recvcounts, - .u.allgatherv.displs = displs, - .u.allgatherv.recvtype = recvtype, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = - MPIR_Allgatherv_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, - recvtype, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgatherv_intra_composition_alpha: - mpi_errno = - MPIDI_Allgatherv_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, - recvtype, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Scatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__SCATTER, - .comm_ptr = comm, - - .u.scatter.sendbuf = sendbuf, - .u.scatter.sendcount = sendcount, - .u.scatter.sendtype = sendtype, - .u.scatter.recvcount = recvcount, - .u.scatter.recvbuf = recvbuf, - .u.scatter.recvtype = recvtype, - .u.scatter.root = root, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - MPIR_Scatter_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, - coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scatter_intra_composition_alpha: - mpi_errno = - MPIDI_Scatter_intra_composition_alpha(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Scatterv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, MPI_Datatype recvtype, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__SCATTERV, - .comm_ptr = comm, - - .u.scatterv.sendbuf = sendbuf, - .u.scatterv.sendcounts = sendcounts, - .u.scatterv.displs = displs, - .u.scatterv.sendtype = sendtype, - .u.scatterv.recvcount = recvcount, - .u.scatterv.recvbuf = recvbuf, - .u.scatterv.recvtype = recvtype, - .u.scatterv.root = root, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - MPIR_Scatterv_impl(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, - root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scatterv_intra_composition_alpha: - mpi_errno = - MPIDI_Scatterv_intra_composition_alpha(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, - comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Gather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__GATHER, - .comm_ptr = comm, - - .u.gather.sendbuf = sendbuf, - .u.gather.sendcount = sendcount, - .u.gather.sendtype = sendtype, - .u.gather.recvcount = recvcount, - .u.gather.recvbuf = recvbuf, - .u.gather.recvtype = recvtype, - .u.gather.root = root, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = - MPIR_Gather_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, - coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gather_intra_composition_alpha: - mpi_errno = - MPIDI_Gather_intra_composition_alpha(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Gatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__GATHERV, - .comm_ptr = comm, - - .u.gatherv.sendbuf = sendbuf, - .u.gatherv.sendcount = sendcount, - .u.gatherv.sendtype = sendtype, - .u.gatherv.recvbuf = recvbuf, - .u.gatherv.recvcounts = recvcounts, - .u.gatherv.displs = displs, - .u.gatherv.recvtype = recvtype, - .u.gatherv.root = root, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Gatherv_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gatherv_intra_composition_alpha: - mpi_errno = - MPIDI_Gatherv_intra_composition_alpha(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, - comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - MPL_STATIC_INLINE_PREFIX void MPIDI_Alltoall_fill_multi_leads_info(MPIR_Comm * comm) { if (MPIDI_COMM(comm, alltoall_comp_info) == NULL) { @@ -958,1028 +157,4 @@ MPL_STATIC_INLINE_PREFIX void MPIDI_Alltoall_fill_multi_leads_info(MPIR_Comm * c } } -MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_allcomm_composition_json(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size, data_size; - - if (sendbuf != MPI_IN_PLACE) { - MPIR_Datatype_get_size_macro(sendtype, type_size); - data_size = sendcount * type_size; - } else { - MPIR_Datatype_get_size_macro(recvtype, type_size); - data_size = recvcount * type_size; - } - - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLTOALL, - .comm_ptr = comm, - - .u.alltoall.sendbuf = sendbuf, - .u.alltoall.sendcount = sendcount, - .u.alltoall.sendtype = sendtype, - .u.alltoall.recvcount = recvcount, - .u.alltoall.recvbuf = recvbuf, - .u.alltoall.recvtype = recvtype, - }; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_alpha: - if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) - MPIDI_Alltoall_fill_multi_leads_info(comm); - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM - && MPIDI_COMM_ALLTOALL(comm, use_multi_leads) == 1 && - data_size <= MPIR_CVAR_ALLTOALL_SHM_PER_RANK, mpi_errno, - "Alltoall composition alpha cannot be applied.\n"); - mpi_errno = - MPIDI_Alltoall_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, - coll_attr); - break; - - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_beta: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Alltoall composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, - coll_attr); - break; - - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = - MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, - coll_attr); - else - mpi_errno = MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - MPI_Aint type_size, data_size; - - MPIR_FUNC_ENTER; - - if (sendbuf != MPI_IN_PLACE) { - MPIR_Datatype_get_size_macro(sendtype, type_size); - data_size = sendcount * type_size; - } else { - MPIR_Datatype_get_size_macro(recvtype, type_size); - data_size = recvcount * type_size; - } - - switch (MPIR_CVAR_ALLTOALL_COMPOSITION) { - case 1: - if (comm->comm_kind == MPIR_COMM_KIND__INTRACOMM) - MPIDI_Alltoall_fill_multi_leads_info(comm); - - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM - && MPIDI_COMM_ALLTOALL(comm, use_multi_leads) == 1 && - data_size <= MPIR_CVAR_ALLTOALL_SHM_PER_RANK, mpi_errno, - "Alltoall composition alpha cannot be applied.\n"); - /* Multi-leaders based composition can only be used if the comm is spanned over more than - * 1 node, has equal number of ranks on each node, ranks on a node are consecutive and - * the combined msg from all the ranks on a node fits the allocated shared memory buffer - */ - - mpi_errno = - MPIDI_Alltoall_intra_composition_alpha(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, - coll_attr); - break; - case 2: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Alltoall composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr); - break; - default: - mpi_errno = MPIDI_Alltoall_allcomm_composition_json(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, - coll_attr); - break; - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = - MPIR_Alltoall_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, - coll_attr); - else - mpi_errno = MPIDI_Alltoall_intra_composition_beta(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, coll_attr); - - fn_exit: - MPIR_FUNC_ENTER; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Alltoallv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, MPI_Datatype recvtype, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLTOALLV, - .comm_ptr = comm, - - .u.alltoallv.sendbuf = sendbuf, - .u.alltoallv.sendcounts = sendcounts, - .u.alltoallv.sdispls = sdispls, - .u.alltoallv.sendtype = sendtype, - .u.alltoallv.recvbuf = recvbuf, - .u.alltoallv.recvcounts = recvcounts, - .u.alltoallv.rdispls = rdispls, - .u.alltoallv.recvtype = recvtype, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Alltoallv_impl(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallv_intra_composition_alpha: - mpi_errno = - MPIDI_Alltoallv_intra_composition_alpha(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, - rdispls, recvtype, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Alltoallw(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__ALLTOALLW, - .comm_ptr = comm, - - .u.alltoallw.sendbuf = sendbuf, - .u.alltoallw.sendcounts = sendcounts, - .u.alltoallw.sdispls = sdispls, - .u.alltoallw.sendtypes = sendtypes, - .u.alltoallw.recvbuf = recvbuf, - .u.alltoallw.recvcounts = recvcounts, - .u.alltoallw.rdispls = rdispls, - .u.alltoallw.recvtypes = recvtypes, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Alltoallw_impl(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallw_intra_composition_alpha: - mpi_errno = - MPIDI_Alltoallw_intra_composition_alpha(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_allcomm_composition_json(const void *sendbuf, - void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__REDUCE, - .comm_ptr = comm, - - .u.reduce.sendbuf = sendbuf, - .u.reduce.recvbuf = recvbuf, - .u.reduce.count = count, - .u.reduce.datatype = datatype, - .u.reduce.op = op, - .u.reduce.root = root, - }; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_alpha: - mpi_errno = - MPIDI_Reduce_intra_composition_alpha(sendbuf, recvbuf, count, datatype, op, - root, comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_beta: - mpi_errno = - MPIDI_Reduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, - root, comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_gamma: - mpi_errno = - MPIDI_Reduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, - root, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - switch (MPIR_CVAR_REDUCE_COMPOSITION) { - case 1: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM - && MPIR_Comm_is_parent_comm(comm) && - MPIR_Op_is_commutative(op), mpi_errno, - "Reduce composition alpha cannot be applied.\n"); - mpi_errno = - MPIDI_Reduce_intra_composition_alpha(sendbuf, recvbuf, count, datatype, op, root, - comm, coll_attr); - break; - case 2: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, comm->comm_kind == MPIR_COMM_KIND__INTRACOMM - && MPIR_Comm_is_parent_comm(comm) && - MPIR_Op_is_commutative(op), mpi_errno, - "Reduce composition beta cannot be applied.\n"); - mpi_errno = - MPIDI_Reduce_intra_composition_beta(sendbuf, recvbuf, count, datatype, op, root, - comm, coll_attr); - break; - case 3: - MPII_COLLECTIVE_FALLBACK_CHECK(comm->rank, - comm->comm_kind == MPIR_COMM_KIND__INTRACOMM, mpi_errno, - "Reduce composition gamma cannot be applied.\n"); - mpi_errno = - MPIDI_Reduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, root, - comm, coll_attr); - break; - default: - mpi_errno = - MPIDI_Reduce_allcomm_composition_json(sendbuf, recvbuf, count, datatype, op, - root, comm, coll_attr); - break; - } - - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - - fallback: - if (comm->comm_kind == MPIR_COMM_KIND__INTERCOMM) - mpi_errno = MPIR_Reduce_impl(sendbuf, recvbuf, count, datatype, op, root, comm, coll_attr); - else - mpi_errno = - MPIDI_Reduce_intra_composition_gamma(sendbuf, recvbuf, count, datatype, op, root, comm, - coll_attr); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Reduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER, - .comm_ptr = comm, - - .u.reduce_scatter.sendbuf = sendbuf, - .u.reduce_scatter.recvbuf = recvbuf, - .u.reduce_scatter.recvcounts = recvcounts, - .u.reduce_scatter.datatype = datatype, - .u.reduce_scatter.op = op, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = - MPIR_Reduce_scatter_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_scatter_intra_composition_alpha: - mpi_errno = - MPIDI_Reduce_scatter_intra_composition_alpha(sendbuf, recvbuf, recvcounts, - datatype, op, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Reduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__REDUCE_SCATTER_BLOCK, - .comm_ptr = comm, - - .u.reduce_scatter_block.sendbuf = sendbuf, - .u.reduce_scatter_block.recvbuf = recvbuf, - .u.reduce_scatter_block.recvcount = recvcount, - .u.reduce_scatter_block.datatype = datatype, - .u.reduce_scatter_block.op = op, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = - MPIR_Reduce_scatter_block_impl(sendbuf, recvbuf, recvcount, datatype, op, comm, - coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_scatter_block_intra_composition_alpha: - mpi_errno = - MPIDI_Reduce_scatter_block_intra_composition_alpha(sendbuf, recvbuf, recvcount, - datatype, op, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Scan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__SCAN, - .comm_ptr = comm, - - .u.scan.sendbuf = sendbuf, - .u.scan.recvbuf = recvbuf, - .u.scan.count = count, - .u.scan.datatype = datatype, - .u.scan.op = op, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Scan_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scan_intra_composition_alpha: - mpi_errno = - MPIDI_Scan_intra_composition_alpha(sendbuf, recvbuf, count, - datatype, op, comm, coll_attr); - break; - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scan_intra_composition_beta: - mpi_errno = - MPIDI_Scan_intra_composition_beta(sendbuf, recvbuf, count, - datatype, op, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Exscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - const MPIDI_Csel_container_s *cnt = NULL; - - MPIR_Csel_coll_sig_s coll_sig = { - .coll_type = MPIR_CSEL_COLL_TYPE__EXSCAN, - .comm_ptr = comm, - - .u.exscan.sendbuf = sendbuf, - .u.exscan.recvbuf = recvbuf, - .u.exscan.count = count, - .u.exscan.datatype = datatype, - .u.exscan.op = op, - }; - - MPIR_FUNC_ENTER; - - cnt = MPIR_Csel_search(MPIDI_COMM(comm, csel_comm), coll_sig); - - if (cnt == NULL) { - mpi_errno = MPIR_Exscan_impl(sendbuf, recvbuf, count, datatype, op, comm, coll_attr);; - MPIR_ERR_CHECK(mpi_errno); - goto fn_exit; - } - - switch (cnt->id) { - case MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Exscan_intra_composition_alpha: - mpi_errno = - MPIDI_Exscan_intra_composition_alpha(sendbuf, recvbuf, count, - datatype, op, comm, coll_attr); - break; - default: - MPIR_Assert(0); - } - - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - MPIR_FUNC_EXIT; - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_neighbor_allgather(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_neighbor_allgatherv(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_neighbor_alltoallv(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_alltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype * sendtypes, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype * recvtypes, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_neighbor_alltoallw(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Neighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_neighbor_alltoall(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_allgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ineighbor_allgather(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_allgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ineighbor_allgatherv(sendbuf, sendcount, sendtype, - recvbuf, recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_alltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ineighbor_alltoall(sendbuf, sendcount, sendtype, - recvbuf, recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_alltoallv(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ineighbor_alltoallv(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, - req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ineighbor_alltoallw(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype * sendtypes, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - const MPI_Datatype * recvtypes, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ineighbor_alltoallw(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, rdispls, recvtypes, - comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ibarrier(MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ibarrier(comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ibcast(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ibcast(buffer, count, datatype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iallgather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iallgather(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iallgatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iallgatherv(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iallreduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iallreduce(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ialltoall(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ialltoall(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ialltoallv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ialltoallv(sendbuf, sendcounts, sdispls, sendtype, - recvbuf, recvcounts, rdispls, recvtype, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ialltoallw(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - const MPI_Datatype * sendtypes, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * rdispls, - const MPI_Datatype * recvtypes, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ialltoallw(sendbuf, sendcounts, sdispls, sendtypes, - recvbuf, recvcounts, rdispls, recvtypes, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iexscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iexscan(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Igather(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_igather(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Igatherv(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, const MPI_Aint * displs, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_igatherv(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ireduce_scatter(const void *sendbuf, void *recvbuf, - const MPI_Aint * recvcounts, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Ireduce(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_ireduce(sendbuf, recvbuf, count, datatype, op, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iscan(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iscan(sendbuf, recvbuf, count, datatype, op, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iscatter(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iscatter(sendbuf, sendcount, sendtype, recvbuf, - recvcount, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - -MPL_STATIC_INLINE_PREFIX int MPID_Iscatterv(const void *sendbuf, const MPI_Aint * sendcounts, - const MPI_Aint * displs, MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Request ** req) -{ - int ret; - - MPIR_FUNC_ENTER; - - ret = MPIDI_NM_mpi_iscatterv(sendbuf, sendcounts, displs, sendtype, - recvbuf, recvcount, recvtype, root, comm, req); - - MPIR_FUNC_EXIT; - return ret; -} - #endif /* CH4_COLL_H_INCLUDED */ diff --git a/src/mpid/ch4/src/ch4_coll_impl.h b/src/mpid/ch4/src/ch4_coll_impl.h index cae0128c8b2..4eac499fc77 100644 --- a/src/mpid/ch4/src/ch4_coll_impl.h +++ b/src/mpid/ch4/src/ch4_coll_impl.h @@ -95,11 +95,14 @@ static void MPIDI_Coll_host_buffer_genq_free(void *sendbuf, void *recvbuf, MPI_A } } -static void MPIDI_Coll_host_buffer_genq_alloc(const void *sendbuf, const void *recvbuf, - MPI_Aint count, MPI_Datatype datatype, - void **host_sendbuf, void **host_recvbuf, - MPL_pointer_attr_t send_attr, - MPL_pointer_attr_t recv_attr, MPI_Aint shift) +MPL_STATIC_INLINE_PREFIX void MPIDI_Coll_host_buffer_genq_alloc(const void *sendbuf, + const void *recvbuf, MPI_Aint count, + MPI_Datatype datatype, + void **host_sendbuf, + void **host_recvbuf, + MPL_pointer_attr_t send_attr, + MPL_pointer_attr_t recv_attr, + MPI_Aint shift) { void *tmp_send = NULL, *tmp_recv = NULL; if (sendbuf != MPI_IN_PLACE) { @@ -145,8 +148,8 @@ static void MPIDI_Coll_host_buffer_genq_alloc(const void *sendbuf, const void *r goto fn_exit; } -static void MPIDI_Coll_calculate_size_shift(MPI_Aint count, MPI_Datatype datatype, MPI_Aint * size, - MPI_Aint * shift) +MPL_STATIC_INLINE_PREFIX void MPIDI_Coll_calculate_size_shift(MPI_Aint count, MPI_Datatype datatype, + MPI_Aint * size, MPI_Aint * shift) { MPI_Aint true_lb, true_extent, extent; @@ -166,6 +169,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_intra_composition_alpha(MPIR_Comm * c { int mpi_errno = MPI_SUCCESS; +#if 1 + /* FIXME: call MPIR_Coll_auto or MPIR_Barrier_impl on sub-comms */ + MPIR_Assert(0); + return mpi_errno; +#else /* do the intranode barrier on all nodes */ if (comm->node_comm != NULL) { #ifndef MPIDI_CH4_DIRECT_NETMOD @@ -201,19 +209,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_intra_composition_alpha(MPIR_Comm * c return mpi_errno; fn_fail: goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_intra_composition_beta(MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIDI_NM_mpi_barrier(comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_alpha(void *buffer, MPI_Aint count, @@ -222,6 +218,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_alpha(void *buffer, M int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *host_buffer = NULL; void *saved_buffer = buffer; MPL_pointer_attr_t attr; @@ -308,6 +310,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_alpha(void *buffer, M return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_beta(void *buffer, MPI_Aint count, @@ -316,6 +319,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_beta(void *buffer, MP int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *host_buffer = NULL; void *saved_buffer = buffer; MPL_pointer_attr_t attr; @@ -377,6 +386,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_beta(void *buffer, MP return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_gamma(void *buffer, MPI_Aint count, @@ -385,6 +395,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_gamma(void *buffer, M int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *host_buffer = NULL; void *saved_buffer = buffer; MPL_pointer_attr_t attr; @@ -420,6 +436,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_gamma(void *buffer, M return mpi_errno; fn_fail: goto fn_exit; +#endif } /* @@ -436,6 +453,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_delta(void *buffer, M int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *host_buffer = NULL; void *saved_buffer = buffer; MPL_pointer_attr_t attr; @@ -527,6 +550,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intra_composition_delta(void *buffer, M return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_alpha(const void *sendbuf, @@ -537,6 +561,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_alpha(const void int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *in_recvbuf = recvbuf; void *host_sendbuf = NULL; void *host_recvbuf = NULL; @@ -622,6 +652,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_alpha(const void return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_beta(const void *sendbuf, @@ -631,6 +662,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_beta(const void * MPIR_Comm * comm, int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *in_recvbuf = recvbuf; void *host_sendbuf = NULL; void *host_recvbuf = NULL; @@ -669,6 +706,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_beta(const void * return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_gamma(const void *sendbuf, @@ -679,6 +717,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_gamma(const void int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else void *in_recvbuf = recvbuf; void *host_sendbuf = NULL; void *host_recvbuf = NULL; @@ -720,6 +764,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_gamma(const void return mpi_errno; fn_fail: goto fn_exit; +#endif } /* Multi-leaders based composition. Have num_leaders per node, which reduce the data within @@ -738,6 +783,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_delta(const void int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else char *shm_addr; int my_leader_rank = -1, iter; MPI_Aint num_chunks, chunk_size_floor, chunk_size_ceil; @@ -931,6 +982,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intra_composition_delta(const void return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_alpha(const void *sendbuf, @@ -940,6 +992,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_alpha(const void *se MPIR_Comm * comm, int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else MPI_Aint true_lb = 0; MPI_Aint true_extent = 0; MPI_Aint extent = 0; @@ -1014,6 +1072,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_alpha(const void *se return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_beta(const void *sendbuf, @@ -1023,6 +1082,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_beta(const void *sen MPIR_Comm * comm, int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else MPI_Aint true_lb = 0; MPI_Aint true_extent = 0; MPI_Aint extent = 0; @@ -1117,24 +1182,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_beta(const void *sen fn_fail: goto fn_exit; -} - - -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intra_composition_gamma(const void *sendbuf, - void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, - MPI_Op op, int root, - MPIR_Comm * comm, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; +#endif } /* Node-aware multi-leaders based inter-node and intra-node composition. Each rank on a node places @@ -1150,6 +1198,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_alpha(const void * int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else int num_nodes; int num_ranks = MPIR_Comm_size(comm_ptr); int node_comm_size = MPIR_Comm_size(comm_ptr->node_comm); @@ -1245,6 +1299,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_alpha(const void * return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_beta(const void *sendbuf, @@ -1257,6 +1312,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_beta(const void *s int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else int node_comm_size = 0; if (comm_ptr->node_comm != NULL) { node_comm_size = MPIR_Comm_size(comm_ptr->node_comm); @@ -1285,58 +1346,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoall_intra_composition_beta(const void *s return mpi_errno; fn_fail: goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoallv_intra_composition_alpha(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * sdispls, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * rdispls, - MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_alltoallv(sendbuf, sendcounts, sdispls, - sendtype, recvbuf, recvcounts, rdispls, recvtype, comm_ptr, - coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Alltoallw_intra_composition_alpha(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype - sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype - recvtypes[], - MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_alltoallw(sendbuf, sendcounts, sdispls, - sendtypes, recvbuf, recvcounts, - rdispls, recvtypes, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_alpha(const void *sendbuf, @@ -1349,6 +1359,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_alpha(const void int coll_attr) { int mpi_errno = MPI_SUCCESS; +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else int node_comm_size = MPIR_Comm_size(comm_ptr->node_comm); int my_node_comm_rank = MPIR_Comm_rank(comm_ptr->node_comm); MPI_Aint type_size, extent, true_extent, lb; @@ -1436,6 +1451,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_alpha(const void return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_beta(const void *sendbuf, @@ -1448,6 +1464,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_beta(const void * int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else int node_comm_size = 0; if (comm_ptr->node_comm != NULL) { node_comm_size = MPIR_Comm_size(comm_ptr->node_comm); @@ -1476,6 +1498,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allgather_intra_composition_beta(const void * return mpi_errno; fn_fail: goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Allgatherv_intra_composition_alpha(const void *sendbuf, @@ -1489,6 +1512,12 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allgatherv_intra_composition_alpha(const void int coll_attr) { int mpi_errno = MPI_SUCCESS; + +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else int node_comm_size = 0; if (comm_ptr->node_comm != NULL) { node_comm_size = MPIR_Comm_size(comm_ptr->node_comm); @@ -1517,141 +1546,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Allgatherv_intra_composition_alpha(const void return mpi_errno; fn_fail: goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Gather_intra_composition_alpha(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_gather(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Gatherv_intra_composition_alpha(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, - MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_gatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Scatter_intra_composition_alpha(const void *sendbuf, - MPI_Aint sendcount, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_scatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Scatterv_intra_composition_alpha(const void *sendbuf, - const MPI_Aint * sendcounts, - const MPI_Aint * displs, - MPI_Datatype sendtype, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype recvtype, - int root, MPIR_Comm * comm, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_scatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, - recvcount, recvtype, root, comm, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_scatter_intra_composition_alpha(const void *sendbuf, - void *recvbuf, - const MPI_Aint - recvcounts[], - MPI_Datatype - datatype, MPI_Op op, - MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_reduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm_ptr, - coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_scatter_block_intra_composition_alpha(const void - *sendbuf, - void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype - datatype, - MPI_Op op, - MPIR_Comm * - comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = - MPIDI_NM_mpi_reduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, - op, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; +#endif } MPL_STATIC_INLINE_PREFIX int MPIDI_Scan_intra_composition_alpha(const void *sendbuf, @@ -1662,6 +1557,11 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Scan_intra_composition_alpha(const void *send MPIR_Comm * comm_ptr, int coll_attr) { int mpi_errno = MPI_SUCCESS; +#if 1 + /* FIXME */ + MPIR_Assert(0); + return mpi_errno; +#else int rank = comm_ptr->rank; MPI_Status status; void *tempbuf = NULL; @@ -1794,43 +1694,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_Scan_intra_composition_alpha(const void *send return mpi_errno; fn_fail: goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Scan_intra_composition_beta(const void *sendbuf, - void *recvbuf, - MPI_Aint count, - MPI_Datatype datatype, - MPI_Op op, - MPIR_Comm * comm_ptr, int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIDI_NM_mpi_scan(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; -} - -MPL_STATIC_INLINE_PREFIX int MPIDI_Exscan_intra_composition_alpha(const void *sendbuf, - void *recvbuf, - MPI_Aint count, - MPI_Datatype datatype, - MPI_Op op, - MPIR_Comm * comm_ptr, - int coll_attr) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIDI_NM_mpi_exscan(sendbuf, recvbuf, count, datatype, op, comm_ptr, coll_attr); - MPIR_ERR_CHECK(mpi_errno); - - fn_exit: - return mpi_errno; - fn_fail: - goto fn_exit; +#endif } #endif /* CH4_COLL_IMPL_H_INCLUDED */ diff --git a/src/mpid/ch4/src/ch4_comm.c b/src/mpid/ch4/src/ch4_comm.c index da7d7a9d3f9..279558304d6 100644 --- a/src/mpid/ch4/src/ch4_comm.c +++ b/src/mpid/ch4/src/ch4_comm.c @@ -197,23 +197,6 @@ int MPID_Comm_commit_post_hook(MPIR_Comm * comm) MPIR_ERR_CHECK(mpi_errno); } - /* prune selection tree */ - if (MPIDI_global.csel_root) { - mpi_errno = MPIR_Csel_prune(MPIDI_global.csel_root, comm, &MPIDI_COMM(comm, csel_comm)); - MPIR_ERR_CHECK(mpi_errno); - } else { - MPIDI_COMM(comm, csel_comm) = NULL; - } - - /* prune selection tree for gpu */ - if (MPIDI_global.csel_root_gpu) { - mpi_errno = MPIR_Csel_prune(MPIDI_global.csel_root_gpu, comm, - &MPIDI_COMM(comm, csel_comm_gpu)); - MPIR_ERR_CHECK(mpi_errno); - } else { - MPIDI_COMM(comm, csel_comm_gpu) = NULL; - } - fn_exit: MPIR_FUNC_EXIT; return mpi_errno; @@ -281,16 +264,6 @@ int MPID_Comm_free_hook(MPIR_Comm * comm) MPIR_ERR_CHECK(mpi_errno); #endif - if (MPIDI_COMM(comm, csel_comm)) { - mpi_errno = MPIR_Csel_free(MPIDI_COMM(comm, csel_comm)); - MPIR_ERR_CHECK(mpi_errno); - } - - if (MPIDI_COMM(comm, csel_comm_gpu)) { - mpi_errno = MPIR_Csel_free(MPIDI_COMM(comm, csel_comm_gpu)); - MPIR_ERR_CHECK(mpi_errno); - } - mpi_errno = MPIDIG_destroy_comm(comm); MPIR_ERR_CHECK(mpi_errno); @@ -365,8 +338,8 @@ int MPID_Intercomm_exchange(MPIR_Comm * local_comm, int local_leader, bcast_data.mpi_errno = mpi_errno; bcast_data.remote_data_size = remote_data_size; } - mpi_errno = MPIR_Bcast_impl(&bcast_data, 2, MPIR_INT_INTERNAL, - local_leader, local_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(&bcast_data, 2, MPIR_INT_INTERNAL, + local_leader, local_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* error checking of previous leader exchange */ @@ -384,8 +357,8 @@ int MPID_Intercomm_exchange(MPIR_Comm * local_comm, int local_leader, MPIR_ERR_CHKANDJUMP(!remote_data, mpi_errno, MPI_ERR_OTHER, "**nomem"); } - mpi_errno = MPIR_Bcast_impl(remote_data, remote_data_size, MPIR_BYTE_INTERNAL, - local_leader, local_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(remote_data, remote_data_size, MPIR_BYTE_INTERNAL, + local_leader, local_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* Stage 3: Each process extract data (if necessary: add worlds, convert lpids) */ diff --git a/src/mpid/ch4/src/ch4_init.c b/src/mpid/ch4/src/ch4_init.c index c94c573cb1d..8bbaa736052 100644 --- a/src/mpid/ch4/src/ch4_init.c +++ b/src/mpid/ch4/src/ch4_init.c @@ -171,106 +171,6 @@ === END_MPI_T_CVAR_INFO_BLOCK === */ -static const char *devcollstr(void) -{ - if (MPIR_CVAR_DEVICE_COLLECTIVES == MPIR_CVAR_DEVICE_COLLECTIVES_all) { - return "all"; - } else if (MPIR_CVAR_DEVICE_COLLECTIVES == MPIR_CVAR_DEVICE_COLLECTIVES_none) { - return "none"; - } else if (MPIR_CVAR_DEVICE_COLLECTIVES == MPIR_CVAR_DEVICE_COLLECTIVES_percoll) { - return "percoll"; - } else { - MPIR_Assert(0); - } - return NULL; -} - -static void *create_container(struct json_object *obj) -{ - MPIDI_Csel_container_s *cnt = MPL_malloc(sizeof(MPIDI_Csel_container_s), MPL_MEM_COLL); - - json_object_object_foreach(obj, key, val) { - char *ckey = MPL_strdup_no_spaces(key); - - if (!strcmp(ckey, "composition=MPIDI_Barrier_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Barrier_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Barrier_intra_composition_beta")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Barrier_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_beta")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_gamma")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_gamma; - else if (!strcmp(ckey, "composition=MPIDI_Bcast_intra_composition_delta")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Bcast_intra_composition_delta; - else if (!strcmp(ckey, "composition=MPIDI_Allreduce_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Allreduce_intra_composition_beta")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Allreduce_intra_composition_gamma")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allreduce_intra_composition_gamma; - else if (!strcmp(ckey, "composition=MPIDI_Reduce_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Reduce_intra_composition_beta")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Reduce_intra_composition_gamma")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_intra_composition_gamma; - else if (!strcmp(ckey, "composition=MPIDI_Alltoall_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Alltoall_intra_composition_beta")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoall_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Alltoallv_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallv_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Alltoallw_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Alltoallw_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Allgather_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Allgather_intra_composition_beta")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgather_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Allgatherv_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Allgatherv_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Gather_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gather_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Gatherv_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Gatherv_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Scatter_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scatter_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Scatterv_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scatterv_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Reduce_scatter_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_scatter_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Reduce_scatter_block_intra_composition_alpha")) - cnt->id = - MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Reduce_scatter_block_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Scan_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scan_intra_composition_alpha; - else if (!strcmp(ckey, "composition=MPIDI_Scan_intra_composition_beta")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Scan_intra_composition_beta; - else if (!strcmp(ckey, "composition=MPIDI_Exscan_intra_composition_alpha")) - cnt->id = MPIDI_CSEL_CONTAINER_TYPE__COMPOSITION__MPIDI_Exscan_intra_composition_alpha; - else { - fprintf(stderr, "unrecognized key %s\n", ckey); - MPIR_Assert(0); - } - - MPL_free(ckey); - } - - return (void *) cnt; -} - static int choose_netmod(void); static int choose_netmod(void) @@ -520,8 +420,6 @@ int MPID_Init(int requested, int *provided) MPIDIU_avt_init(); MPIDIU_map_create((void **) &(MPIDI_global.win_map), MPL_MEM_RMA); - MPIDI_global.csel_root = NULL; - MPIDI_global.csel_root_gpu = NULL; mpi_errno = MPIDI_vci_init(); MPIR_ERR_CHECK(mpi_errno); @@ -558,30 +456,6 @@ int MPID_Init(int requested, int *provided) MPIDIG_am_check_init(); - /* Initialize collective selection */ - if (!strcmp(MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE, "")) { - mpi_errno = MPIR_Csel_create_from_buf(MPIDI_coll_generic_json, - create_container, &MPIDI_global.csel_root); - MPIDI_global.csel_source = "MPIDI_coll_generic_json"; - } else { - mpi_errno = MPIR_Csel_create_from_file(MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE, - create_container, &MPIDI_global.csel_root); - MPIDI_global.csel_source = MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE; - } - MPIR_ERR_CHECK(mpi_errno); - - /* Initialize collective selection for gpu */ - if (!strcmp(MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE_GPU, "")) { - mpi_errno = MPIR_Csel_create_from_buf(MPIDI_coll_generic_json, - create_container, &MPIDI_global.csel_root_gpu); - MPIDI_global.csel_source_gpu = "MPIDI_coll_generic_json"; - } else { - mpi_errno = MPIR_Csel_create_from_file(MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE_GPU, - create_container, &MPIDI_global.csel_root_gpu); - MPIDI_global.csel_source_gpu = MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE_GPU; - } - MPIR_ERR_CHECK(mpi_errno); - /* Override split_type */ MPIDI_global.MPIR_Comm_fns_store.split_type = MPIDI_Comm_split_type; MPIR_Comm_fns = &MPIDI_global.MPIR_Comm_fns_store; @@ -609,24 +483,6 @@ int MPID_Init(int requested, int *provided) #endif fprintf(stdout, "==== Various sizes and limits ====\n"); fprintf(stdout, "sizeof(MPIDI_per_vci_t): %d\n", (int) sizeof(MPIDI_per_vci_t)); - printf("==== collective selection ====\n"); - printf("MPIR_CVAR_DEVICE_COLLECTIVES: %s\n", devcollstr()); - MPIR_Assert(MPIR_Csel_source); - printf("MPIR: %s\n", MPIR_Csel_source); - MPIR_Assert(MPIDI_global.csel_source); - printf("MPID: %s\n", MPIDI_global.csel_source); -#ifndef MPIDI_CH4_DIRECT_NETMOD - MPIR_Assert(MPIDI_global.shm.posix.csel_source); - printf("MPID/shm: %s\n", MPIDI_global.shm.posix.csel_source); -#endif - if (MPIR_CVAR_ENABLE_GPU) { - MPIR_Assert(MPIDI_global.csel_source_gpu); - printf("MPID (GPU): %s\n", MPIDI_global.csel_source_gpu); -#ifndef MPIDI_CH4_DIRECT_NETMOD - MPIR_Assert(MPIDI_global.shm.posix.csel_source_gpu); - printf("MPID/shm (GPU): %s\n", MPIDI_global.shm.posix.csel_source_gpu); -#endif - } } fn_exit: @@ -714,16 +570,6 @@ int MPID_Finalize(void) MPIR_ERR_CHECK(mpi_errno); #endif - if (MPIDI_global.csel_root) { - mpi_errno = MPIR_Csel_free(MPIDI_global.csel_root); - MPIR_ERR_CHECK(mpi_errno); - } - - if (MPIDI_global.csel_root_gpu) { - mpi_errno = MPIR_Csel_free(MPIDI_global.csel_root_gpu); - MPIR_ERR_CHECK(mpi_errno); - } - MPIDIG_am_finalize(); MPIDU_genq_private_pool_destroy(MPIDI_global.gpu_coll_pool); diff --git a/src/mpid/ch4/src/ch4_persist.c b/src/mpid/ch4/src/ch4_persist.c index 8b82e260d52..a10878a680d 100644 --- a/src/mpid/ch4/src/ch4_persist.c +++ b/src/mpid/ch4/src/ch4_persist.c @@ -166,345 +166,3 @@ int MPID_Recv_init(void *buf, fn_fail: goto fn_exit; } - -int MPID_Bcast_init(void *buffer, MPI_Aint count, MPI_Datatype datatype, - int root, MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Bcast_init_impl(buffer, count, datatype, root, comm_ptr, info_ptr, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Allreduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allreduce_init_impl(sendbuf, recvbuf, count, datatype, op, comm_ptr, info_ptr, - request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Reduce_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_init_impl(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, - info_ptr, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Alltoall_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoall_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, info_ptr, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Alltoallv_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], MPI_Datatype sendtype, - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallv_init_impl(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm_ptr, info_ptr, - request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Alltoallw_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Alltoallw_init_impl(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, - recvcounts, rdispls, recvtypes, comm_ptr, info_ptr, - request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Allgather_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allgather_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - comm_ptr, info_ptr, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Allgatherv_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint * recvcounts, - const MPI_Aint * displs, MPI_Datatype recvtype, - MPIR_Comm * comm_ptr, MPIR_Info * info_ptr, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Allgatherv_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, - displs, recvtype, comm_ptr, info_ptr, request); - - return mpi_errno; -} - -int MPID_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, - MPI_Aint recvcount, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_scatter_block_init_impl(sendbuf, recvbuf, recvcount, datatype, op, comm, - info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Reduce_scatter_init(const void *sendbuf, void *recvbuf, - const MPI_Aint recvcounts[], - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Reduce_scatter_init_impl(sendbuf, recvbuf, recvcounts, datatype, op, comm, - info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Scan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scan_init_impl(sendbuf, recvbuf, count, datatype, op, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Gather_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gather_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - root, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Gatherv_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], const MPI_Aint displs[], - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Gatherv_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, - recvtype, root, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Scatter_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, int root, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatter_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, - root, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Scatterv_init(const void *sendbuf, const MPI_Aint sendcounts[], - const MPI_Aint displs[], MPI_Datatype sendtype, - void *recvbuf, MPI_Aint recvcount, - MPI_Datatype recvtype, int root, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Scatterv_init_impl(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, - recvtype, root, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Barrier_init(MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Barrier_init_impl(comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Exscan_init(const void *sendbuf, void *recvbuf, MPI_Aint count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Exscan_init_impl(sendbuf, recvbuf, count, datatype, op, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Neighbor_allgather_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgather_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Neighbor_allgatherv_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint displs[], - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_allgatherv_init_impl(sendbuf, sendcount, sendtype, recvbuf, - recvcounts, displs, recvtype, comm, info, - request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Neighbor_alltoall_init(const void *sendbuf, MPI_Aint sendcount, - MPI_Datatype sendtype, void *recvbuf, - MPI_Aint recvcount, MPI_Datatype recvtype, - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoall_init_impl(sendbuf, sendcount, sendtype, recvbuf, recvcount, - recvtype, comm, info, request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Neighbor_alltoallv_init(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - MPI_Datatype sendtype, void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - MPI_Datatype recvtype, MPIR_Comm * comm, - MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallv_init_impl(sendbuf, sendcounts, sdispls, sendtype, recvbuf, - recvcounts, rdispls, recvtype, comm, info, - request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} - -int MPID_Neighbor_alltoallw_init(const void *sendbuf, - const MPI_Aint sendcounts[], - const MPI_Aint sdispls[], - const MPI_Datatype sendtypes[], - void *recvbuf, - const MPI_Aint recvcounts[], - const MPI_Aint rdispls[], - const MPI_Datatype recvtypes[], - MPIR_Comm * comm, MPIR_Info * info, MPIR_Request ** request) -{ - int mpi_errno = MPI_SUCCESS; - MPIR_FUNC_ENTER; - - mpi_errno = MPIR_Neighbor_alltoallw_init_impl(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, - recvcounts, rdispls, recvtypes, comm, info, - request); - - MPIR_FUNC_EXIT; - return mpi_errno; -} diff --git a/src/mpid/ch4/src/ch4_spawn.c b/src/mpid/ch4/src/ch4_spawn.c index 489815720b8..06a5dee05dd 100644 --- a/src/mpid/ch4/src/ch4_spawn.c +++ b/src/mpid/ch4/src/ch4_spawn.c @@ -378,10 +378,10 @@ static int dynamic_intercomm_create(const char *port_name, MPIR_Info * info, int fn_fail: /* In case root fails, we bcast mpi_errno so other ranks will abort too */ - MPIR_Bcast_impl(&mpi_errno, 1, MPIR_INT_INTERNAL, root, comm_ptr, MPIR_COLL_ATTR_SYNC); + MPIR_Bcast_fallback(&mpi_errno, 1, MPIR_INT_INTERNAL, root, comm_ptr, MPIR_COLL_ATTR_SYNC); } else { int root_errno; - MPIR_Bcast_impl(&root_errno, 1, MPIR_INT_INTERNAL, root, comm_ptr, MPIR_COLL_ATTR_SYNC); + MPIR_Bcast_fallback(&root_errno, 1, MPIR_INT_INTERNAL, root, comm_ptr, MPIR_COLL_ATTR_SYNC); if (root_errno) { MPIR_ERR_SET(mpi_errno, MPI_ERR_PORT, "**comm_connect_fail"); } diff --git a/src/mpid/ch4/src/ch4_types.h b/src/mpid/ch4/src/ch4_types.h index c84a93637d4..b4f3f5931f2 100644 --- a/src/mpid/ch4/src/ch4_types.h +++ b/src/mpid/ch4/src/ch4_types.h @@ -294,10 +294,6 @@ typedef struct MPIDI_CH4_Global_t { MPIDI_per_vci_t per_vci[MPIDI_CH4_MAX_VCIS]; MPIDI_CH4_configurations_t settings; - void *csel_root; - const char *csel_source; - void *csel_root_gpu; - const char *csel_source_gpu; #ifndef MPIDI_CH4_DIRECT_NETMOD MPIDI_SHM_Global_t shm; @@ -305,7 +301,6 @@ typedef struct MPIDI_CH4_Global_t { MPIDI_NM_Global_t nm; } MPIDI_CH4_Global_t; extern MPIDI_CH4_Global_t MPIDI_global; -extern char MPIDI_coll_generic_json[]; #ifdef MPL_USE_DBG_LOGGING extern MPL_dbg_class MPIDI_CH4_DBG_GENERAL; extern MPL_dbg_class MPIDI_CH4_DBG_MAP; diff --git a/src/mpid/ch4/src/mpidig_win.c b/src/mpid/ch4/src/mpidig_win.c index 923602e46c1..aa63a49ceb6 100644 --- a/src/mpid/ch4/src/mpidig_win.c +++ b/src/mpid/ch4/src/mpidig_win.c @@ -394,8 +394,8 @@ static int win_init(MPI_Aint length, int disp_unit, MPIR_Win ** win_ptr, MPIR_In if (!comm_ptr->node_comm) no_local = true; - mpi_errno = MPIR_Allreduce(&no_local, &all_no_local, 1, MPIR_C_BOOL_INTERNAL, - MPI_LAND, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allreduce_fallback(&no_local, &all_no_local, 1, MPIR_C_BOOL_INTERNAL, + MPI_LAND, comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (all_no_local) MPIDI_WIN(win, winattr) |= MPIDI_WINATTR_ACCU_NO_SHM; @@ -561,8 +561,9 @@ static int win_shm_alloc_impl(MPI_Aint size, int disp_unit, MPIR_Comm * comm_ptr * thus all process can be assigned to a page aligned start address. * - user sets alloc_shared_noncontig=true, thus we can internally make * the size aligned on each process. */ - mpi_errno = MPIR_Allreduce(&symheap_flag, &global_symheap_flag, 1, MPIR_C_BOOL_INTERNAL, - MPI_LAND, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(&symheap_flag, &global_symheap_flag, 1, MPIR_C_BOOL_INTERNAL, + MPI_LAND, comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); } else global_symheap_flag = false; diff --git a/src/mpid/ch4/src/mpidig_win.h b/src/mpid/ch4/src/mpidig_win.h index a172bceceec..b4a6e91dafa 100644 --- a/src/mpid/ch4/src/mpidig_win.h +++ b/src/mpid/ch4/src/mpidig_win.h @@ -522,7 +522,7 @@ MPL_STATIC_INLINE_PREFIX int MPIDIG_mpi_win_fence(int massert, MPIR_Win * win) * the VCI lock internally. */ MPID_THREAD_CS_EXIT(VCI, MPIDI_VCI_LOCK(vci)); need_unlock = 0; - mpi_errno = MPIR_Barrier(win->comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(win->comm_ptr, MPIR_COLL_ATTR_SYNC); fn_exit: if (need_unlock) { diff --git a/src/mpid/common/shm/mpidu_init_shm_alloc.c b/src/mpid/common/shm/mpidu_init_shm_alloc.c index 55291a90b5d..37371505093 100644 --- a/src/mpid/common/shm/mpidu_init_shm_alloc.c +++ b/src/mpid/common/shm/mpidu_init_shm_alloc.c @@ -199,13 +199,13 @@ int MPIDU_Init_shm_comm_alloc(MPIR_Comm * comm, size_t len, void **ptr) } MPIR_Assert(serialized_hnd_size <= MPIDU_INIT_SHM_BLOCK_SIZE); if (node_comm) { - mpi_errno = MPIR_Bcast_impl(serialized_hnd, MPIDU_INIT_SHM_BLOCK_SIZE, - MPIR_BYTE_INTERNAL, 0, node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(serialized_hnd, MPIDU_INIT_SHM_BLOCK_SIZE, + MPIR_BYTE_INTERNAL, 0, node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); } } else { - mpi_errno = MPIR_Bcast_impl(serialized_hnd_buffer, MPIDU_INIT_SHM_BLOCK_SIZE, - MPIR_BYTE_INTERNAL, 0, node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(serialized_hnd_buffer, MPIDU_INIT_SHM_BLOCK_SIZE, + MPIR_BYTE_INTERNAL, 0, node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); serialized_hnd = serialized_hnd_buffer; serialized_hnd_size = strlen(serialized_hnd) + 1; /* add 1 for null char */ @@ -222,7 +222,7 @@ int MPIDU_Init_shm_comm_alloc(MPIR_Comm * comm, size_t len, void **ptr) } if (node_comm) { - mpi_errno = MPIR_Barrier_impl(node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Barrier_fallback(node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); } if (need_remove) { diff --git a/src/mpid/common/shm/mpidu_shm_alloc.c b/src/mpid/common/shm/mpidu_shm_alloc.c index d3e0faf5f90..2450399f698 100644 --- a/src/mpid/common/shm/mpidu_shm_alloc.c +++ b/src/mpid/common/shm/mpidu_shm_alloc.c @@ -235,8 +235,8 @@ static int allreduce_maxloc(size_t mysz, int myloc, MPIR_Comm * comm, size_t * m maxloc.loc = myloc; mpi_errno = - MPIR_Allreduce(&maxloc, &maxloc_result, 1, maxloc_type, maxloc_op->handle, comm, - MPIR_COLL_ATTR_SYNC); + MPIR_Allreduce_fallback(&maxloc, &maxloc_result, 1, maxloc_type, maxloc_op->handle, comm, + MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); *maxsz_loc = maxloc_result.loc; @@ -341,8 +341,9 @@ static int map_symm_shm(MPIR_Comm * shm_comm_ptr, MPIDU_shm_seg_t * shm_seg, int /* check results of all processes. If any failure happens (max result > 0), * return SYMSHM_OTHER_FAIL if anyone reports it (max result == 2). * Otherwise return SYMSHM_MAP_FAIL (max result == 1). */ - mpi_errno = MPIR_Allreduce(map_result_ptr, &all_map_result, 1, MPIR_INT_INTERNAL, - MPI_MAX, shm_comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(map_result_ptr, &all_map_result, 1, MPIR_INT_INTERNAL, + MPI_MAX, shm_comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (all_map_result != SYMSHM_SUCCESS) @@ -452,8 +453,8 @@ static int shm_alloc_symm_all(MPIR_Comm * comm_ptr, size_t offset, MPIDU_shm_seg MPIR_ERR_CHECK(mpi_errno); /* check if any mapping failure occurs */ - mpi_errno = MPIR_Allreduce(&map_result, &all_map_result, 1, MPIR_INT_INTERNAL, - MPI_MAX, comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allreduce_fallback(&map_result, &all_map_result, 1, MPIR_INT_INTERNAL, + MPI_MAX, comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* cleanup local shm segment if mapping failed on other process */ @@ -503,16 +504,17 @@ static int shm_alloc(MPIR_Comm * shm_comm_ptr, MPIDU_shm_seg_t * shm_seg) if (shm_fail_flag) serialized_hnd = &mpl_err_hnd[0]; - mpi_errno = MPIR_Bcast_impl(serialized_hnd, MPL_SHM_GHND_SZ, MPIR_BYTE_INTERNAL, 0, - shm_comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(serialized_hnd, MPL_SHM_GHND_SZ, MPIR_BYTE_INTERNAL, 0, + shm_comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (shm_fail_flag) goto map_fail; /* ensure all other processes have mapped successfully */ - mpi_errno = MPIR_Allreduce_impl(&shm_fail_flag, &any_shm_fail_flag, 1, MPIR_C_BOOL_INTERNAL, - MPI_LOR, shm_comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(&shm_fail_flag, &any_shm_fail_flag, 1, MPIR_C_BOOL_INTERNAL, + MPI_LOR, shm_comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* unlink shared memory region so it gets deleted when all processes exit */ @@ -526,8 +528,8 @@ static int shm_alloc(MPIR_Comm * shm_comm_ptr, MPIDU_shm_seg_t * shm_seg) char serialized_hnd[MPL_SHM_GHND_SZ] = { 0 }; /* get serialized handle from rank 0 and deserialize it */ - mpi_errno = MPIR_Bcast_impl(serialized_hnd, MPL_SHM_GHND_SZ, MPIR_CHAR_INTERNAL, 0, - shm_comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(serialized_hnd, MPL_SHM_GHND_SZ, MPIR_CHAR_INTERNAL, 0, + shm_comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); /* empty handler means root fails */ @@ -549,8 +551,9 @@ static int shm_alloc(MPIR_Comm * shm_comm_ptr, MPIDU_shm_seg_t * shm_seg) mapped_flag = true; result_sync: - mpi_errno = MPIR_Allreduce_impl(&shm_fail_flag, &any_shm_fail_flag, 1, MPIR_C_BOOL_INTERNAL, - MPI_LOR, shm_comm_ptr, MPIR_COLL_ATTR_SYNC); + mpi_errno = + MPIR_Allreduce_fallback(&shm_fail_flag, &any_shm_fail_flag, 1, MPIR_C_BOOL_INTERNAL, + MPI_LOR, shm_comm_ptr, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); if (any_shm_fail_flag) diff --git a/src/util/mpir_nodemap.c b/src/util/mpir_nodemap.c index ea10f44af0c..1a6ea95e612 100644 --- a/src/util/mpir_nodemap.c +++ b/src/util/mpir_nodemap.c @@ -454,16 +454,16 @@ int MPIR_nodeid_init(void) MPIR_Strerror(errno, strerrbuf, MPIR_STRERROR_BUF_SIZE), errno); my_hostname[MAX_HOSTNAME_LEN - 1] = '\0'; - mpi_errno = MPIR_Allgather_impl(MPI_IN_PLACE, MAX_HOSTNAME_LEN, MPIR_CHAR_INTERNAL, - allhostnames, MAX_HOSTNAME_LEN, MPIR_CHAR_INTERNAL, - node_roots_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Allgather_fallback(MPI_IN_PLACE, MAX_HOSTNAME_LEN, MPIR_CHAR_INTERNAL, + allhostnames, MAX_HOSTNAME_LEN, MPIR_CHAR_INTERNAL, + node_roots_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); } MPIR_Comm *node_comm = MPIR_Process.comm_world->node_comm; if (node_comm) { - mpi_errno = MPIR_Bcast_impl(allhostnames, MAX_HOSTNAME_LEN * MPIR_Process.num_nodes, - MPIR_CHAR_INTERNAL, 0, node_comm, MPIR_COLL_ATTR_SYNC); + mpi_errno = MPIR_Bcast_fallback(allhostnames, MAX_HOSTNAME_LEN * MPIR_Process.num_nodes, + MPIR_CHAR_INTERNAL, 0, node_comm, MPIR_COLL_ATTR_SYNC); MPIR_ERR_CHECK(mpi_errno); }