Remove inter-dependencies between OSC modules. #4527

clementFoyer · 2017-11-22T10:01:21Z

The osc monitoring component needed to include other OSC components header in order to be able to access communicator through the component specific ompi_osc_*_module_t structures. Now we keep a hashtable to keep track of the window specific communicator. This commit remove the dependency, and resolve the issue #4523 opened by @tkordenbrock .

@bosilca @jsquyres Please review.

ompiteam-bot · 2017-11-22T10:01:23Z

Can one of the admins verify this patch?

bwbarrett · 2017-11-22T18:00:40Z

ompi/mca/osc/monitoring/osc_monitoring_component.c

-OSC_MONITORING_MODULE_TEMPLATE_GENERATE(portals4, ompi_osc_portals4_module_t, comm)
-#undef GET_MODULE
-#endif /* OMPI_WITH_OSC_PORTALS4 */
+OSC_MONITORING_MODULE_TEMPLATE_GENERATE(rdma)


This still feels pretty brittle (although it is a step up). There probably not a generic solution, but a less-brittle version would be to have an autogenerated file from configure with these OSC_MONITORING_MODULE_TEMPLATE_GENERATE() statements, which was generated during configure time by looping over mca_ompi_osc_m4_config_component_list and mca_ompi_osc_no_config_component_list.

As an example, I notice that there's no template for the ucx component.

Indeed I have not came back for a few few months, and I missed this new component. You are totally right on how careful this whole component have to be handle. I had not thought of this m4 solution, but that's a very good idea.

bwbarrett · 2017-11-22T18:01:24Z

ok to test

bosilca · 2017-11-23T17:48:36Z

Another possible approach is to unify the different OSC modules structs by moving the ompi_communicator_t pointer from the specialized modules down into the base ompi_osc_base_module_t structure (in MPI every window has it's corresponding communicator).

clementFoyer · 2017-11-24T16:09:15Z

Maybe this PR should not be merged yet, as it may be followed by another change to remove the introduced hashtable.
Also, should the upcoming commits affect all osc components that saved the communicator to their own module structures? And finally, should the communicator field in the base structure be at the same nested level as the functions?
I.E. :

struct ompi_osc_base_module_3_0_0_t {
    struct ompi_communicator_t *comm;
    ompi_osc_base_module_win_shared_query_fn_t osc_win_shared_query;
    ompi_osc_base_module_win_attach_fn_t osc_win_attach;
    ompi_osc_base_module_win_detach_fn_t osc_win_detach;
    ompi_osc_base_module_free_fn_t osc_free;
    ompi_osc_base_module_put_fn_t osc_put;
    ompi_osc_base_module_get_fn_t osc_get;
    ompi_osc_base_module_accumulate_fn_t osc_accumulate;
    ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap;
    ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op;
    ompi_osc_base_module_get_accumulate_fn_t osc_get_accumulate;
    ompi_osc_base_module_rput_fn_t osc_rput;
    ompi_osc_base_module_rget_fn_t osc_rget;
    ompi_osc_base_module_raccumulate_fn_t osc_raccumulate;
    ompi_osc_base_module_rget_accumulate_fn_t osc_rget_accumulate;
    ompi_osc_base_module_fence_fn_t osc_fence;
    ompi_osc_base_module_start_fn_t osc_start;
    ompi_osc_base_module_complete_fn_t osc_complete;
    ompi_osc_base_module_post_fn_t osc_post;
    ompi_osc_base_module_wait_fn_t osc_wait;
    ompi_osc_base_module_test_fn_t osc_test;
    ompi_osc_base_module_lock_fn_t osc_lock;
    ompi_osc_base_module_unlock_fn_t osc_unlock;
    ompi_osc_base_module_lock_all_fn_t osc_lock_all;
    ompi_osc_base_module_unlock_all_fn_t osc_unlock_all;
    ompi_osc_base_module_sync_fn_t osc_sync;
    ompi_osc_base_module_flush_fn_t osc_flush;
    ompi_osc_base_module_flush_all_fn_t osc_flush_all;
    ompi_osc_base_module_flush_local_fn_t osc_flush_local;
    ompi_osc_base_module_flush_local_all_fn_t osc_flush_local_all;
};
typedef struct ompi_osc_base_module_3_0_0_t ompi_osc_base_module_3_0_0_t;
typedef ompi_osc_base_module_3_0_0_t ompi_osc_base_module_t;

or

struct ompi_osc_base_module_3_0_0_fn_t {
    ompi_osc_base_module_win_shared_query_fn_t osc_win_shared_query;
    ompi_osc_base_module_win_attach_fn_t osc_win_attach;
    ompi_osc_base_module_win_detach_fn_t osc_win_detach;
    ompi_osc_base_module_free_fn_t osc_free;
    ompi_osc_base_module_put_fn_t osc_put;
    ompi_osc_base_module_get_fn_t osc_get;
    ompi_osc_base_module_accumulate_fn_t osc_accumulate;
    ompi_osc_base_module_compare_and_swap_fn_t osc_compare_and_swap;
    ompi_osc_base_module_fetch_and_op_fn_t osc_fetch_and_op;
    ompi_osc_base_module_get_accumulate_fn_t osc_get_accumulate;
    ompi_osc_base_module_rput_fn_t osc_rput;
    ompi_osc_base_module_rget_fn_t osc_rget;
    ompi_osc_base_module_raccumulate_fn_t osc_raccumulate;
    ompi_osc_base_module_rget_accumulate_fn_t osc_rget_accumulate;
    ompi_osc_base_module_fence_fn_t osc_fence;
    ompi_osc_base_module_start_fn_t osc_start;
    ompi_osc_base_module_complete_fn_t osc_complete;
    ompi_osc_base_module_post_fn_t osc_post;
    ompi_osc_base_module_wait_fn_t osc_wait;
    ompi_osc_base_module_test_fn_t osc_test;
    ompi_osc_base_module_lock_fn_t osc_lock;
    ompi_osc_base_module_unlock_fn_t osc_unlock;
    ompi_osc_base_module_lock_all_fn_t osc_lock_all;
    ompi_osc_base_module_unlock_all_fn_t osc_unlock_all;
    ompi_osc_base_module_sync_fn_t osc_sync;
    ompi_osc_base_module_flush_fn_t osc_flush;
    ompi_osc_base_module_flush_all_fn_t osc_flush_all;
    ompi_osc_base_module_flush_local_fn_t osc_flush_local;
    ompi_osc_base_module_flush_local_all_fn_t osc_flush_local_all;
};
typedef struct ompi_osc_base_module_3_0_0_fn_t ompi_osc_base_module_3_0_0_fn_t;
struct ompi_osc_base_module_3_0_0_t {
    struct ompi_communicator_t *comm;
    ompi_osc_base_module_3_0_0_fn_t module;
};
typedef ompi_osc_base_module_3_0_0_t ompi_osc_base_module_t;

hppritcha · 2017-11-26T03:37:00Z

OS-X is not happy:

aking all in mca/osc/monitoring
  CC       osc_monitoring_component.lo
In file included from osc_monitoring_component.c:24:
./osc_monitoring_template_gen.h:40:3: error: expected ';' after expression
-n     
  ^
  ;
./osc_monitoring_template_gen.h:40:2: error: use of undeclared identifier 'n'
-n     
 ^
./osc_monitoring_template_gen.h:43:3: error: expected ';' after expression
-n     } else 
  ^
  ;
./osc_monitoring_template_gen.h:43:2: error: use of undeclared identifier 'n'
-n     } else 
 ^
./osc_monitoring_template_gen.h:46:3: error: expected ';' after expression
-n     } else 
  ^
  ;
./osc_monitoring_template_gen.h:46:2: error: use of undeclared identifier 'n'
-n     } else 
 ^
./osc_monitoring_template_gen.h:49:3: error: expected ';' after expression
-n     } else 
  ^
  ;
./osc_monitoring_template_gen.h:49:2: error: use of undeclared identifier 'n'
-n     } else 
 ^
./osc_monitoring_template_gen.h:52:3: error: expected ';' after expression
-n     } else 
  ^
  ;
./osc_monitoring_template_gen.h:52:2: error: use of undeclared identifier 'n'
-n     } else 
 ^
./osc_monitoring_template_gen.h:55:3: error: expected ';' after expression
-n     } else 
  ^
  ;
./osc_monitoring_template_gen.h:55:2: error: use of undeclared identifier 'n'
-n     } else 
 ^
12 errors generated.
make[2]: *** [osc_monitoring_component.lo] Error 1
make[1]: *** [all-recursive] Error 1
make: *** [all-recursive] Error 1
Build step 'Execute shell' marked build as failure
Finished: FAILURE
Page generated: Nov 26, 2017 3:35:18 AM UTCREST APIJenkins ver. 2.91

ibm-ompi · 2017-11-27T08:05:25Z

The IBM CI (GNU Compiler) build failed! Please review the log, linked below.

Gist: https://gist.github.com/5692e25afdb5edea04a1451fad841cf2

ibm-ompi · 2017-11-27T08:14:00Z

The IBM CI (PGI Compiler) build failed! Please review the log, linked below.

Gist: https://gist.github.com/a6c633a1019927effca8fdabc35289eb

jsquyres

Most of these commits should be squashed.

bwbarrett · 2017-11-30T19:13:01Z

ompi/mca/osc/monitoring/Makefile.am

@@ -17,7 +17,8 @@ monitoring_sources  = \
 	osc_monitoring_active_target.h \
 	osc_monitoring_dynamic.h \
 	osc_monitoring_module.h \
-	osc_monitoring_template.h
+	osc_monitoring_template.h \
+	osc_monitoring_template_gen.h


template_gen.h shouldn't be in monitoring_sources. Since it's generated at every run of configure, it shouldn't be distributed in the tarball (which monitoring_sources will be). You probably need to add a nodist_mca_osc_monitoring_la_SOURCES line (I think that's right, my Automake foo is getting rusty).

Done in commit 2011660 .
I'm not a automake guru, but it seems like adding osc_monitoring_template_gen.h to the nodist_mcacomponent_HEADERS target worked as intended.

yeah, I saw that in the next commit (I review commits independently, because changes should be bisectable). You need to merge those two commits into one before merging this PR to master.

bwbarrett · 2017-11-30T19:22:19Z

ompi/mca/osc/monitoring/configure.m4

+        done
+        echo '' >>[$1]
+        cat <<EOF >>[$1]
+        return OMPI_ERR_NOT_SUPPORTED;


I have to say that this shell script is really ugly to read. It would be a lot clearer (in my opinion) if you didn't try to get the else if on the same line and just had some slightly uglier code generated.

I have to say I totally agree with you :-D And it actually led to issue with OS-X, and replacing "echo -n" with "printf" did not worked either. Commit 2011660 fixes the issue. And at this point I start feeling like I should not have kept the wrong commits, and only keep once everything was fixed. I'll do that tomorrow.

Another option you might want to do here is not generate the inline code here in the .h file, but rather just instantiate an argv-style array of the component names. That's the only thing that needs to be generated (the array of component names), right? Then the code can be beautifully indented in a .c file. 😄

bwbarrett · 2017-11-30T19:22:36Z

ompi/mca/osc/monitoring/configure.m4

+          [$1],
+          [$2])
+
+   mca_ompi_osc_monitoring_generate_templates([$OMPI_TOP_SRCDIR/ompi/mca/osc/monitoring/osc_monitoring_template_gen.h],


I'm surprised this worked with "make distcheck"; you should be writing into the build directory, not the source directory. You actually shouldn't need the prefix ($OMPI_TOP_SRCDIR/) at all; configure is by definition run at the top of the list.

It did not. I think the cleancheck failed (but I did not know that at that point. I fought latter with targets to make it succeed. And I reached beyond despair at some point... But now it seems fine)

bwbarrett

sigh; this commit should be squashed into the previous one.

bwbarrett

This should be squashed into 08ec754

bosilca · 2017-11-30T20:04:29Z

As we can use the group to translate ranks why do we need the knowledge about the other OSC modules ?

clementFoyer · 2017-11-30T20:24:48Z

The OSC component specific module is kept in the window. As it keeps all the structured data needed by a component to be effective, it cannot be replaced in the window structure, because some component are expecting these information to be there, and are accessing directly through the window given as parameter. I would have love to "extend" these module to add my own informations, but I would have to be able to know their module sizes, and thus, I would have to include their module headers. Otherwise I could create a module big enough to contain any OSC module but that is quite fragile and it would get us back to the original issue that had me to create this PR.
In order to intercept the call, I cannot replace this module. So I change the module functions to my templated OSC component specific functions, so for each OSC function, and for each component, I can intercept the call, record data, then forward to the originally targeted function.

hppritcha · 2017-12-05T16:14:50Z

bot:ompi:retest

clementFoyer · 2017-12-11T17:06:32Z

Everything here is squashed into two commits. The first one does the first quick solution that translate rank from windows' group to MPI_COMM_WORLD rank. The second commit enables the automatic generation at config time. IMO, it is relevant to keep these two commits separated. From the differents tests I've made, all seems to work smoothly.

As the monitoring was introduced into v3.1, this PR may be also added to the target v3.1.x

bwbarrett

If you get a chance, it would be awesome to fix the one Makefile bit, but let's ship it...

bwbarrett · 2017-12-11T17:10:27Z

ompi/mca/osc/monitoring/Makefile.am


 noinst_LTLIBRARIES = $(component_noinst)
 libmca_osc_monitoring_la_SOURCES = $(monitoring_sources)
 libmca_osc_monitoring_la_LDFLAGS = -module -avoid-version
+
+# C file depends on the header file generated at configure time
+$(srcdir)/osc_monitoring_component.c: $(builddir)/osc_monitoring_template_gen.h


What's the point of this rule? You don't have any generating rules, so all it's doing is expressing a dependency, which should be picked up in dependency generation. It's not harmful, but it's odd.

Expressing the dependency was the only point. I wasn't sure whether or not it would have been expressed otherwise. I think I tried to check that if the generated .h file was edited by hand (should not be, but who knows...) osc_monitoring_component.c was compiled again, and I think it was not. If it should be here, I'll remove it.

you can look at the .deps/... file to make sure, but the compiler should generate a dependency line. In general we don't add extra dependency information unless we're adding a build rule. So it's not required, but it's also not harmful. If you can change it, that would be great. But I'm also fine if you change it in a follow-up commit.

bwbarrett · 2017-12-11T17:15:18Z

@jsquyres, I think @clementFoyer has addressed your squash concern; mind dismissing your review (or doing another review)?

clementFoyer · 2017-12-11T21:40:29Z

There was a prototype of a function that became useless and needed to be removed. I did it and removed the useless dependency explicitation from Makefile.am

jsquyres · 2017-12-12T19:05:54Z

ompi/mca/osc/monitoring/configure.m4

+#define MCA_OSC_MONITORING_GEN_TEMPLATE_H
+
+#include <string.h>
+#include <ompi_config.h>


<ompi_config.h> should be first.

jsquyres · 2017-12-12T19:06:56Z

ompi/mca/osc/monitoring/configure.m4

+                          AC_CONFIG_COMMANDS([$1],
+                                             [filename="$1"
+                                              components=`echo "$2" | sed -e 's/,/ /g' -e 's/monitoring//'`
+                                              cat <<EOF >$filename


Minor nit: this indenting is inconsistent with the indenting below (i.e., the indenting below is nice 4-space indenting, but this is much larger than that).

jsquyres · 2017-12-12T19:09:54Z

ompi/mca/osc/monitoring/configure.m4

 #
+AC_DEFUN([mca_ompi_osc_monitoring_generate_templates],[


m4 macros as usually named with upper case, so that we know they are macros.

jsquyres · 2017-12-12T19:10:25Z

ompi/mca/osc/monitoring/osc_monitoring_component.c

-/* Include template generating macros */
-#include "osc_monitoring_template.h"
+/**************************************/
+/* Include templated macros */


It's not just macros, right? There's also inline functions.

jsquyres · 2017-12-12T19:12:39Z

ompi/mca/osc/monitoring/configure.m4

+        done
+        echo '' >>[$1]
+        cat <<EOF >>[$1]
+        return OMPI_ERR_NOT_SUPPORTED;


Another option you might want to do here is not generate the inline code here in the .h file, but rather just instantiate an argv-style array of the component names. That's the only thing that needs to be generated (the array of component names), right? Then the code can be beautifully indented in a .c file. 😄

jsquyres

Is there a reason that iterating over the already-existing ompi_osc_base_framework.framework_components list isn't sufficient such that you need to build your own? I'm sorry I didn't make that clear in my initial review...

Also, you don't need to keep all the commits in this PR.

The 1st one adds some hard-coded component names that you then remove in later commits. That kind of stuff can be squashed together so that we only merge the final/desired/good code (multiple commits on a PR is fine -- it's just a little silly to have ABC code in commit X, and then remove ABC code in commit Y on the same PR. Know what I mean?).

jsquyres · 2018-01-09T17:48:59Z

ompi/mca/osc/monitoring/osc_monitoring_module.h

@@ -40,10 +40,10 @@

 /* Define and set the ompi_osc_monitoring_## template ##_template
 * variable. The functions recorded here are linked to the original
- * functions of the original {template} module that were replaced.
+ * functions of the original {template} module that weas replaced.


clementFoyer · 2018-01-09T18:26:50Z

Not sure where you ask about the ompi_osc_base_framework.framework_components. Is it the list created in the configure.m4 file or when I iterate in osc_monitoring_component.c?

jsquyres · 2018-01-09T18:36:43Z

@clementFoyer This list should already exist and be fully populated by the time your monitoring component is opened. I.e., you shouldn't have to do anything to create this list -- you should be able to just use it. It should contain the names of all the components that are available (but not necessarily selected).

bwbarrett · 2018-01-09T18:36:53Z

I wouldn't have squashed all the commits into one commit (I disagree with Jeff there). But since you did, you should make the commit message read like one commit.

I'm not sure I understand Jeff's comment about iterating over the framework list, because that's a run-time construct. Since the code needs the list at compile time (to build the templates), that list won't do what you want. Now, there's the question of whether you still need the templates, but I'm pretty sure the answer is yes, because you need to know the size of the module for the underlying transport in order to do the replace logic. So I think this is the best you can do...

clementFoyer · 2018-01-09T18:47:55Z

If needed I still have a version somewhere where everything is not squashed, if that would be required.

Two things about the list : I need the list to omit the monitoring component, and it is used at compilation time as it is used in the macros.

Saddly, I still need the templates as I cannot simply replace the modules in the windows structures. Without it, I cannot know wich was the underlying osc omponent. I originally wanted to be able to replace the module pointers by inheriting from them, but that would require whether to know the size of the module, or to include the other headers, which led to the original issue.

jsquyres · 2018-01-09T18:52:35Z

Sorry, I wasn't advocating for squashing into a single commit; multiple commits are both fine and good. But I do dislike seeing:

Commit A introduces code XYZ.
Commit B removes code XYZ.

I also agree with Brian -- I forgot to mention that one of your commit messages looks like a squash of 2 commit messages. If you're going to squash, please make the commit message look like a single commit message.

In terms of iterating over the list of components, let me go read the code again; perhaps I misunderstood what it was trying to do...

jsquyres · 2018-01-09T19:39:45Z

Ok, I understand your code now -- you did it all at compile time with macros and m4 (I think my bias would have been to do it at run time iterating over the framework component list and swapping around function pointers, which is why my brain got confused when I read your code).

jsquyres · 2018-01-09T20:26:05Z

BTW, I still left the "X" review so that the commits can be cleaned up per Brian's and my comments. Thanks!

jsquyres

I'm sorry, I missed that a new commit was put here to be reviewed. I have super minor nit-pick requests. Other than that, I think this is good.

jsquyres · 2018-02-06T16:52:34Z

ompi/mca/common/monitoring/common_monitoring.h

@@ -94,6 +94,13 @@ static inline int mca_common_monitoring_get_world_rank(int dst, struct ompi_comm
    return ret;
 }

+/* Translate the rank from the given communicator of a process to its rank in MPI_COMM_RANK. */
+static inline int mca_common_monitoring_get_world_rank(int dst, struct ompi_communicator_t*comm,
+                                                       int*world_rank)


Why rename this function, and leave a 1-line inline alias function to get to the new function? Why not just call the new function name directly?

jsquyres · 2018-02-06T16:54:53Z

ompi/mca/osc/monitoring/configure.m4

 #
-# Additional copyrights may follow
+# Append to file $1 an array of components by listing component names in $2.


Just to clarify this comment: this function does not append to the end of an existing file -- it overwrites file $1.

jsquyres · 2018-02-06T16:57:47Z

ompi/mca/osc/monitoring/Makefile.am


 noinst_LTLIBRARIES = $(component_noinst)
 libmca_osc_monitoring_la_SOURCES = $(monitoring_sources)
 libmca_osc_monitoring_la_LDFLAGS = -module -avoid-version
+
+distclean-local:
+	rm -f $(builddir)/osc_monitoring_template_gen.h


Minor nit: you can add this file to DISTCLEANFILES and not have to make a distclean-local rule.

The osc monitoring component needed to include other OSC components header in order to be able tu access communicator through the component specific ompi_osc_*_module_t structures. This commit remove the dependency, and resolve the issue open-mpi#4523. Extend the common monitoring API. * Now it's possible to translate from local rank to world rank from both the communicator and the group. * Remove useless hashtable as we directly use the w_group contained in window structure. Add automatic generation at config time. The templates are expanded at configure time. It creates a new header file that generates all the variables/functions needed. Adding this during the autogen automagicaly generates for each of the available modules the proper functions. Only keep a generated argv-style array. Following Jeff's advice, the configure.m4 file generate a simple array of module variables to be iterated over to find the proper module. Signed-off-by: Clement Foyer <clement.foyer@inria.fr>

clementFoyer · 2018-02-07T14:03:00Z

@jsquyres Every thing seems to pass. The requested changes have been made.

jsquyres · 2018-02-08T22:06:20Z

@bwbarrett A bunch of things have changed since you reviewed. Care to review again, or are you good / we should merge?

jsquyres · 2018-02-09T20:49:47Z

I'll take Brian's silence and his prior approval as a positive sign. Let's merge.

Thanks for your patience, @clementFoyer!

bwbarrett reviewed Nov 22, 2017

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch from 5d6207a to f058ac3 Compare November 23, 2017 16:05

bosilca mentioned this pull request Nov 27, 2017

configury: add portals4 include path to osc-monitoring component build #4521

Closed

jsquyres requested changes Nov 27, 2017

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch from 8726fd3 to 2011660 Compare November 28, 2017 10:33

bwbarrett requested changes Nov 30, 2017

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch 3 times, most recently from 704d6e1 to 8127b2d Compare December 4, 2017 16:58

jsquyres mentioned this pull request Dec 5, 2017

osc-monitoring component build fails when osc-portals4 is configured #4523

Closed

bwbarrett added the Target: main label Dec 8, 2017

bwbarrett approved these changes Dec 11, 2017

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch from 8127b2d to fd1b6c4 Compare December 11, 2017 21:38

jsquyres requested changes Dec 12, 2017

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch from fd1b6c4 to fe0c02c Compare December 22, 2017 15:38

clementFoyer force-pushed the osc-no-includes branch from fe0c02c to 70d6dfa Compare December 22, 2017 16:26

jsquyres requested changes Jan 9, 2018

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch from 70d6dfa to 48221f7 Compare January 9, 2018 18:26

clementFoyer force-pushed the osc-no-includes branch from 48221f7 to efecde9 Compare January 10, 2018 08:02

jsquyres requested changes Feb 6, 2018

View reviewed changes

clementFoyer force-pushed the osc-no-includes branch from efecde9 to f5b4fc0 Compare February 7, 2018 13:17

jsquyres approved these changes Feb 8, 2018

View reviewed changes

jsquyres merged commit e7f91f8 into open-mpi:master Feb 9, 2018

clementFoyer mentioned this pull request Feb 15, 2018

Remove inter-dependencies between OSC modules. #4817

Merged

		#
		AC_DEFUN([mca_ompi_osc_monitoring_generate_templates],[

Remove inter-dependencies between OSC modules. #4527

Remove inter-dependencies between OSC modules. #4527

Conversation

clementFoyer commented Nov 22, 2017

ompiteam-bot commented Nov 22, 2017

Choose a reason for hiding this comment

Choose a reason for hiding this comment

bwbarrett commented Nov 22, 2017

bosilca commented Nov 23, 2017

clementFoyer commented Nov 24, 2017 • edited by bosilca

hppritcha commented Nov 26, 2017

ibm-ompi commented Nov 27, 2017

ibm-ompi commented Nov 27, 2017

jsquyres left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

bwbarrett left a comment

Choose a reason for hiding this comment

bwbarrett left a comment

Choose a reason for hiding this comment

bosilca commented Nov 30, 2017

clementFoyer commented Nov 30, 2017

hppritcha commented Dec 5, 2017

clementFoyer commented Dec 11, 2017

bwbarrett left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

bwbarrett commented Dec 11, 2017

clementFoyer commented Dec 11, 2017

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

jsquyres left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

clementFoyer commented Jan 9, 2018

jsquyres commented Jan 9, 2018

bwbarrett commented Jan 9, 2018

clementFoyer commented Jan 9, 2018

jsquyres commented Jan 9, 2018

jsquyres commented Jan 9, 2018 • edited

jsquyres commented Jan 9, 2018

jsquyres left a comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

clementFoyer commented Feb 7, 2018

jsquyres commented Feb 8, 2018

jsquyres commented Feb 9, 2018

clementFoyer commented Nov 24, 2017 •

edited by bosilca

jsquyres commented Jan 9, 2018 •

edited