From 86eaecda8169de9d122b804301b73e8e5522865b Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Mon, 2 Nov 2020 13:34:04 -0600 Subject: [PATCH 1/4] Fix two attribute references. * `PMIX_LOCALITY` is already in the v4 deprecation list * Only declare `PMIX_SERVER_SCHEDULER` once. Consolidate the definition. * Fixes #264 Signed-off-by: Joshua Hursey --- Chap_API_Fabric.tex | 4 +--- Chap_API_Proc_Mgmt.tex | 4 ---- Chap_API_Server.tex | 3 +-- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Chap_API_Fabric.tex b/Chap_API_Fabric.tex index d32065a5..c9bdbf82 100644 --- a/Chap_API_Fabric.tex +++ b/Chap_API_Fabric.tex @@ -570,9 +570,7 @@ \section{Fabric Support Attributes} The following attribute is used by the \ac{PMIx} server library supporting the system's \ac{WLM} to indicate that it wants access to the fabric support functions: -\declareAttributeNEW{PMIX_SERVER_SCHEDULER}{"pmix.srv.sched"}{bool}{ -Server requests access to \ac{WLM}-supporting features - passed solely to the \refapi{PMIx_server_init} \ac{API} to indicate that the library is to be initialized for scheduler support. -} +\pasteAttributeItem{PMIX_SERVER_SCHEDULER} \vspace{\baselineskip} The following attributes may be returned in response to fabric-specific \acp{API} or queries (e.g., \refapi{PMIx_Get} or \refapi{PMIx_Query_info}). These attributes are not related to a specific \refterm{data realm} (as described in Section \ref{api:struct:attributes:retrieval}) - the \refapi{PMIx_Get} function shall therefore ignore the value in its \refarg{proc} process identifier argument when retrieving these values. diff --git a/Chap_API_Proc_Mgmt.tex b/Chap_API_Proc_Mgmt.tex index 6e80f040..eba58458 100644 --- a/Chap_API_Proc_Mgmt.tex +++ b/Chap_API_Proc_Mgmt.tex @@ -1280,10 +1280,6 @@ \subsubsection{Relative locality of two processes} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsubsection{Locality keys} -% -\declareAttributeNEW{PMIX_LOCALITY}{"pmix.loc"}{pmix_locality_t}{ -Bitmask describing the location of the referenced process. -} % \declareAttribute{PMIX_LOCALITY_STRING}{"pmix.locstr"}{char*}{ String describing a process's bound location - referenced using the process's diff --git a/Chap_API_Server.tex b/Chap_API_Server.tex index 366b351e..6ff5d5c9 100644 --- a/Chap_API_Server.tex +++ b/Chap_API_Server.tex @@ -223,10 +223,9 @@ \subsection{Server Initialization Attributes} } % \declareAttributeNEW{PMIX_SERVER_SCHEDULER}{"pmix.srv.sched"}{bool}{ -Server is supporting system scheduler and desires access to appropriate services. +Server is supporting system scheduler and desires access to appropriate \ac{WLM}-supporting features. Indicates that the library is to be initialized for scheduler support. } % -% \declareAttributeNEW{PMIX_EXTERNAL_PROGRESS}{"pmix.evext"}{bool}{ The host shall progress the \ac{PMIx} library via calls to \refapi{PMIx_Progress} } From b436c5ce13d4a88ab140e23afed93c64da934dda Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Mon, 2 Nov 2020 13:57:14 -0600 Subject: [PATCH 2/4] make check-decl: Add detection for deprecated+removed Signed-off-by: Joshua Hursey --- bin/check-multi-declare.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/bin/check-multi-declare.py b/bin/check-multi-declare.py index e7cbaea7..eb777920 100755 --- a/bin/check-multi-declare.py +++ b/bin/check-multi-declare.py @@ -16,6 +16,8 @@ std_structs = {} std_apis = {} std_all_refs = {} + std_deprecated = {} + std_removed = {} # # Command line parsing @@ -55,6 +57,21 @@ # Count will return to 0 when verified value = m.group(1) + + # Check for deprecated and removed identifiers + m = re.search(r"Deprecated", line) + if m is not None: + if value in std_deprecated: + std_deprecated[value] = std_deprecated[value] + 1 + else: + std_deprecated[value] = 1 + m = re.search(r"Removed", line) + if m is not None: + if value in std_removed: + std_removed[value] = std_removed[value] + 1 + else: + std_removed[value] = 1 + #print("Found \""+ref_str+"\" : "+value+" on line " + line) std_all_refs[value] = -1 if ref_str == "attr": @@ -89,7 +106,15 @@ return_count = 0 for val in std_attributes: if std_attributes[val] > 1: - print("Error: " + val + " declared " + str(std_attributes[val]) + " times") + # Look for deprecation and removal (2 references) + if std_attributes[val] == 2: + if val in std_deprecated and val in std_removed: + # Skip this since it was marked as deprecated and removed + continue + if val in std_deprecated: + print("Deprecated: " + val + " declared " + str(std_attributes[val]) + " times (Deprecated in " + str(std_deprecated[val]) + " of those declarations)") + else: + print("Error: " + val + " declared " + str(std_attributes[val]) + " times") return_count += 1 for val in std_consts: if std_consts[val] > 1: From 440e31b9f784ad26de2c92117d8b175083aa1377 Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Mon, 2 Nov 2020 19:55:55 -0600 Subject: [PATCH 3/4] Add some more document checking to the end of make Signed-off-by: Joshua Hursey --- Makefile | 16 +++++++++++----- bin/check-doc.sh | 27 ++++++++++++++++++++++++++- bin/check-multi-declare.py | 8 ++++---- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index a64f9e3c..32ff6660 100644 --- a/Makefile +++ b/Makefile @@ -72,19 +72,25 @@ pmix-standard.pdf: $(CHAPTERS) $(SOURCES) pmix.sty pmix-standard.tex figs/pmix-l FORCECHECK: -check: check-attr-ref check-openpmix check-decl +check: check-doc check-openpmix + +# Includes +# - make check-decl +# - make check-attr-ref +check-doc: pmix-standard.pdf FORCECHECK + @./bin/check-doc.sh check-attr-ref: pmix-standard.pdf FORCECHECK @echo "====> Checking for Attributes Declared, but not referenced" @./bin/check-attr-refs.py -check-openpmix: pmix-standard.pdf FORCECHECK - @echo "====> Checking cross-reference with OpenPMIx" - @./bin/check-openpmix.py - check-decl: pmix-standard.pdf FORCECHECK @echo "====> Checking for Multi-declared items" @./bin/check-multi-declare.py +check-openpmix: pmix-standard.pdf FORCECHECK + @echo "====> Checking cross-reference with OpenPMIx" + @./bin/check-openpmix.py + clean: rm -f $(INTERMEDIATE_FILES) pmix-standard-*.pdf diff --git a/bin/check-doc.sh b/bin/check-doc.sh index 036aa826..403de401 100755 --- a/bin/check-doc.sh +++ b/bin/check-doc.sh @@ -18,7 +18,7 @@ else fi # -# Check for Acronym used but not defined +# Check for missing cross references # echo "====> Checking Cross References" grep "Warning: Reference" pmix-standard.log | grep "undefined" @@ -41,6 +41,31 @@ else echo "====> Passed" fi +# +# Check for multiple declarations +# +echo "====> Checking Multiple Declarations" +./bin/check-multi-declare.py 2>/dev/null 1>/dev/null +if [ $? == 0 ] ; then + echo "====> Passed" +else + ./bin/check-multi-declare.py + echo "====> Error check references (above)" + FINAL_STATUS=$((FINAL_STATUS+1)) +fi + +# +# Check for attribute references +# +echo "====> Checking Attributes are both declared and referenced" +./bin/check-attr-refs.py 2>/dev/null 1>/dev/null +if [ $? == 0 ] ; then + echo "====> Passed" +else + ./bin/check-attr-refs.py + echo "====> Error check references (above)" + FINAL_STATUS=$((FINAL_STATUS+1)) +fi # # All done diff --git a/bin/check-multi-declare.py b/bin/check-multi-declare.py index eb777920..149c1dc0 100755 --- a/bin/check-multi-declare.py +++ b/bin/check-multi-declare.py @@ -34,9 +34,9 @@ all_ref_strs = ["attr", "const", "struct", "macro", "apifn"] for ref_str in all_ref_strs: if args.verbose is True: - print "-"*50 - print "Extracting Standard: \""+ref_str+"\"" - print "-"*50 + print("-"*50) + print("Extracting Standard: \""+ref_str+"\"") + print("-"*50) # subsection.A is Appendix A: Python Bindings p = subprocess.Popen("grep \"newlabel{"+ref_str+"\" pmix-standard.aux | grep -v subsection.A", @@ -133,8 +133,8 @@ print("Error: " + val + " declared " + str(std_apis[val]) + " times") return_count += 1 - print("-"*50) if return_count > 0: + print("-"*50) print("Found %d number of multiple declares" % (return_count)) else: print("Success. No multiple declares detected!") From f84a11de725bbdf5d34e69cf039cecc89050e0f8 Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Tue, 3 Nov 2020 09:43:09 -0600 Subject: [PATCH 4/4] Fix python3 issues Signed-off-by: Joshua Hursey --- bin/check-attr-refs.py | 40 ++++++++++++++++++++------------------ bin/check-multi-declare.py | 5 +++-- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/bin/check-attr-refs.py b/bin/check-attr-refs.py index 45cf681e..1eb68285 100755 --- a/bin/check-attr-refs.py +++ b/bin/check-attr-refs.py @@ -44,18 +44,18 @@ # grep "newlabel{attr" pmix-standard.aux # if args.verbose is True: - print "-"*50 - print "Extracting declared attributes" - print "-"*50 + print("-"*50) + print("Extracting declared attributes") + print("-"*50) p = subprocess.Popen("grep \"newlabel{attr\" pmix-standard.aux", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, close_fds=True) - p.wait() + sout = p.communicate()[0].decode("utf-8").splitlines() if p.returncode != 0: print("Error: Failed to extract declared attributes. grep error code "+str(p.returncode)+")"); sys.exit(2) - for line in p.stdout: + for line in sout: line = line.rstrip() m = re.match(r'\s*\\newlabel{attr:(\w+)', line) if m is None: @@ -65,6 +65,8 @@ # Count will return to 0 when verified attr_declared[m.group(1)] = -1 + p.wait() + if args.verbose is True: for attr in attr_declared: print("Attribute: " + attr) @@ -76,26 +78,26 @@ # If any difference then post a warning # if args.verbose is True: - print "-"*50 - print "Verifying list against the index" - print "-"*50 + print("-"*50) + print("Verifying list against the index") + print("-"*50) for fname in index_files: if os.path.exists(fname) is False: continue if args.verbose is True: - print "Processing Index File: "+fname + print("Processing Index File: "+fname) p = subprocess.Popen("grep \"\\|hyperindexformat\" "+fname, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, close_fds=True) - p.wait() + sout = p.communicate()[0].decode("utf-8").splitlines() + if p.returncode != 0: - print("Error: Failed to verify declared attribute \""+attr+"\". grep error code "+str(p.returncode)+")"); + print("Error: Failed to verify declared attribute \""+attr+"\". grep error code "+str(p.returncode)+""); sys.exit(2) - # List of Definition is larger than attribute list - for line in p.stdout: + for line in sout: line = line.rstrip() m = re.match(r'\s*\\indexentry{(\w+)', line) if m is None: @@ -113,7 +115,7 @@ if attr_to_find in attr_declared: attr_declared[attr_to_find] = attr_declared[attr_to_find] + 1 - + p.wait() # Sanity check. Should never trigger, but check just in case err_out = False @@ -124,7 +126,7 @@ err_out = True num_missing += 1 if err_out is True: - print "-"*50 + print("-"*50) print("Number of deprecated attributes: " + str(len(deprecated_attr))) print("Number of declared attributes : " + str(len(attr_declared))) print("Number of missing attributes : " + str(num_missing)) @@ -139,15 +141,15 @@ # grep "\|hyperpage" pmix-standard.idx # if args.verbose is True: - print "-"*50 - print "Count the usage of each attribute in the document" - print "-"*50 + print("-"*50) + print("Count the usage of each attribute in the document") + print("-"*50) for fname in index_files: if os.path.exists(fname) is False: continue if args.verbose is True: - print "Processing Index File: "+fname + print("Processing Index File: "+fname) # Result set was too big for Python to handle, so use an intermediate file output_file = "pmix-standard.idx-grep" diff --git a/bin/check-multi-declare.py b/bin/check-multi-declare.py index 149c1dc0..9291dd74 100755 --- a/bin/check-multi-declare.py +++ b/bin/check-multi-declare.py @@ -41,12 +41,12 @@ # subsection.A is Appendix A: Python Bindings p = subprocess.Popen("grep \"newlabel{"+ref_str+"\" pmix-standard.aux | grep -v subsection.A", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, close_fds=True) - p.wait() + sout = p.communicate()[0].decode("utf-8").splitlines() if p.returncode != 0: print("Error: Failed to extract declared \""+ref_str+"\". grep error code "+str(p.returncode)+")"); sys.exit(2) - for line in p.stdout: + for line in sout: line = line.rstrip() m = re.match(r"\s*\\newlabel{"+ re.escape(ref_str) + r":(\w+)", line) if m is None: @@ -102,6 +102,7 @@ else: print("Error: Failed to classify the attribute: "+value) sys.exit(1) + p.wait() return_count = 0 for val in std_attributes: