diff --git a/scripts/Documentation/api-objects.txt b/scripts/Documentation/api-objects.txt index 98fc8e3baca0..b7e38f996a74 100644 --- a/scripts/Documentation/api-objects.txt +++ b/scripts/Documentation/api-objects.txt @@ -2,6 +2,10 @@ build_oval_definitions_file build_oval_definitions_file-module.html build_oval_definitions_file.format_duration build_oval_definitions_file-module.html#format_duration build_oval_definitions_file.message build_oval_definitions_file-module.html#message build_oval_definitions_file.main build_oval_definitions_file-module.html#main +find_affected find_affected-module.html +find_affected.get_changed_ids_from_file find_affected-module.html#get_changed_ids_from_file +find_affected.get_changed_ids_from_git find_affected-module.html#get_changed_ids_from_git +find_affected.main find_affected-module.html#main get_related_elements get_related_elements-module.html get_related_elements.format_duration get_related_elements-module.html#format_duration get_related_elements.message get_related_elements-module.html#message @@ -40,13 +44,16 @@ lib_repo.oval_id_to_path lib_repo-module.html#oval_id_to_path lib_repo.get_schema_short_name lib_repo-module.html#get_schema_short_name lib_repo.get_element_paths_iterator lib_repo-module.html#get_element_paths_iterator lib_repo.get_oval_def_schema lib_repo-module.html#get_oval_def_schema +lib_repo.get_schema_versions lib_repo-module.html#get_schema_versions lib_search lib_search-module.html lib_xml lib_xml-module.html +lib_xml.get_element_change_status lib_xml-module.html#get_element_change_status lib_xml.schema_validate lib_xml-module.html#schema_validate lib_xml.get_element_metadata lib_xml-module.html#get_element_metadata lib_xml.__package__ lib_xml-module.html#__package__ lib_xml.apply_xslt lib_xml-module.html#apply_xslt lib_xml.get_definition_metadata lib_xml-module.html#get_definition_metadata +lib_xml.load_standalone_element lib_xml-module.html#load_standalone_element lib_xml.get_schematron_xsl_from_schema lib_xml-module.html#get_schematron_xsl_from_schema lib_xml.schematron_validate lib_xml-module.html#schematron_validate oval_decomposition oval_decomposition-module.html diff --git a/scripts/Documentation/build_oval_definitions_file-module.html b/scripts/Documentation/build_oval_definitions_file-module.html index 9f39843fe381..0ae4c3d27a62 100644 --- a/scripts/Documentation/build_oval_definitions_file-module.html +++ b/scripts/Documentation/build_oval_definitions_file-module.html @@ -170,7 +170,7 @@

Module build_oval_definitions_file

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for 137 138 139 if __name__ == '__main__': -140 140 Source Code for - Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:04 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Class Hierarchy
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 + + + + find_affected + + + + + + + + + + + + + + + + + + + + + +
   Trees      Indices      Help    + + +
OVALRepo APIs
+ + + + + +
+ + Module find_affected + + + + + + +
[hide private]
[frames] | no frames]
+
+ +

Module find_affected

source code

+
+Identify all the elements that will be affected by a submission, whether that submission
+is via a file or a pull request
+
+Authors: Gunnar Engelbach <Gunnar.Engelbach@ThreatGuard.com>
+
+For usage information, please see the command line help:
+    python3 find_affected.py -h
+
+TODO:
+- TBD
+
+
+ + + + + + + + + + + + + + + + +
+ + + + + +
Functions[hide private]
+
+   + + + + + + +
main() + source code + +
+ +
+   + + + + + + +
get_changed_ids_from_git() + source code + +
+ +
+   + + + + + + +
get_changed_ids_from_file(source) + source code + +
+ +
+ + + + + + + + + + + + + + + + +
   Trees      Indices      Help    + + +
OVALRepo APIs
+ + + + + +
+ Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 + + http://epydoc.sourceforge.net +
+ + + + diff --git a/scripts/Documentation/find_affected-pysrc.html b/scripts/Documentation/find_affected-pysrc.html new file mode 100644 index 000000000000..191c69ebabf1 --- /dev/null +++ b/scripts/Documentation/find_affected-pysrc.html @@ -0,0 +1,213 @@ + + + + + find_affected + + + + + + + + + + + + + + + + + + + + + +
   Trees      Indices      Help    + + +
OVALRepo APIs
+ + + + + +
+ + Module find_affected + + + + + + +
[hide private]
[frames] | no frames]
+
+

Source Code for Module find_affected

+
+ 1  #!/usr/bin/env/ python3 
+ 2  """Identify all the elements that will be affected by a submission, whether that submission 
+ 3  is via a file or a pull request 
+ 4   
+ 5  Authors: Gunnar Engelbach <Gunnar.Engelbach@ThreatGuard.com> 
+ 6   
+ 7  For usage information, please see the command line help: 
+ 8      python3 find_affected.py -h 
+ 9   
+10  TODO: 
+11  - TBD 
+12  """ 
+13   
+14   
+15  import argparse 
+16  import lib_search 
+17  import lib_git 
+18  import lib_oval 
+19  import lib_repo 
+20  import os 
+21   
+22   
+
23 -def main(): 
+
24      verbose = False; 
+25       
+26      parser = argparse.ArgumentParser(description='Uses a file or a diff of the local uncommitted changes to identify all items that could be affected.') 
+27      options = parser.add_argument_group('Options') 
+28      options.add_argument('-f', '--file', required=False, help='The name of the source file.  If not used the local git repository will be used as the source') 
+29      options.add_argument('-v', '--verbose', required=False, action="store_true", help='Enable verbose messages') 
+30   
+31      args = vars(parser.parse_args()) 
+32   
+33      if args['verbose']: 
+34          verbose = True 
+35       
+36      source = args['file'] 
+37      if source: 
+38          if not os.path.isfile(source): 
+39              print("\n## Error:  source file '{0}' not found.  No actions taken.\n".format(source)) 
+40              return 
+41          if verbose: 
+42              print("\n Looking for changed items in source file '{0}'".format(source)) 
+43          idlist = get_changed_ids_from_file(source) 
+44      else: 
+45          if verbose: 
+46              print("\n Checking the local repository for uncommitted changes...") 
+47          idlist = get_changed_ids_from_git() 
+48           
+49      if not idlist or idlist is None or len(idlist) < 1: 
+50          print("---  Action complete:  no changes found.\n") 
+51          return 
+
52           
+53           
+54      #Find all downstream children -- that is, a search depth of one 
+55      #Find all upstream users, all the way up to the definition 
+56       
+57      #Sort the list:  definitions, then tests, objects, states, and variables 
+58      #Show the list 
+59      #Offer to build an OVAL file that contains all the changes 
+60           
+61           
+62         
+63         
+64         
+
66      try: 
+67          changed_files = lib_git.get_uncommitted_oval() 
+68          if not changed_files or changed_files is None: 
+69              return None 
+70           
+71          change_list = [] 
+72          for file in changed_files: 
+73              change_list.add(lib_repo.path_to_oval_id(file)) 
+74               
+75          return change_list 
+76       
+77      except Exception: 
+78          if main.verbose: 
+79              print("## Error while querying git for changes: ", format(Exception)) 
+80          return None 
+
81       
+82       
+
83 -def get_changed_ids_from_file(source): 
+
84      if not source or source is None: 
+85          return None 
+
86   
+87   
+88  if __name__ == '__main__': 
+89      main() 
+90   
+
+
+ + + + + + + + + + + + + + + + +
   Trees      Indices      Help    + + +
OVALRepo APIs
+ + + + + +
+ Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 + + http://epydoc.sourceforge.net +
+ + + + diff --git a/scripts/Documentation/get_related_elements-module.html b/scripts/Documentation/get_related_elements-module.html index b8d29282378f..a4234e6392b2 100644 --- a/scripts/Documentation/get_related_elements-module.html +++ b/scripts/Documentation/get_related_elements-module.html @@ -168,7 +168,7 @@

Module get_related_elements

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for Mo 91 92 93 if __name__ == '__main__': -94 94 Source Code for Mo @@ -780,7 +801,7 @@

Identifier Index

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:04 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 grouping_fields @@ -406,32 +421,38 @@

Identifier Index

(in build_oval_definitions_file) - + - + - + - + - + - + - + + + + + +
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for Mo 133 134 135 if __name__ == '__main__': -136 136 Source Code for Mo @@ -394,7 +408,8 @@

Identifier Index

- +
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:04 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 The Navigation Bar @@ -190,177 +190,191 @@

Identifier Index

- + - - + - - + + - - + - + - - + - - - + + - - + + - - + - - + - - + + - - - - - - + - - - - + + - - + - - + + - - + - - + + - - + + - - + + + + + + + + + + + +
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Identifier Index + + + + + - - - - -
find_affectedfind_upstream_ids()
+(in ElementsIndex)		format_duration()
+(in get_repository_stats)
find_downstream_ids()
(in ElementsIndex)		 format_duration()
@@ -175,13 +182,6 @@

Identifier Index

fromStandaloneFile()
(in OvalElement)
find_upstream_ids()
-(in ElementsIndex)		format_duration()
-(in get_repository_stats)		 

G
generator_version
(in OvalGenerator)		get_root_path()
-(in lib_repo)		get_repository_stats getNamespace()
(in OvalElement)
get_current_commit_hash()
-(in lib_git)		get_schema_short_name()
+		get_changed_ids_from_file()
+(in find_affected)		get_root_path()
(in lib_repo)		 getObjects()
(in OvalDocument)
get_definition_metadata()
-(in lib_xml)		get_schematron_xsl_from_schema()
-(in lib_xml)		get_changed_ids_from_git()
+(in find_affected)		get_schema_short_name()
+(in lib_repo)		 getOvalRepositoryInformation()
(in OvalMetadata)
get_definition_paths_iterator()
-(in lib_repo)		get_scripts_path()
+		get_current_commit_hash()
+(in lib_git)		get_schema_versions()
(in lib_repo)		 getOvalTimestamp()
(in OvalDocument)
get_element_metadata()
+		get_definition_metadata()
+(in lib_xml)		get_schematron_xsl_from_schema()
(in lib_xml)		get_uncommitted_oval()
-(in lib_git)		 getPredicate()
(in OvalElement)
get_element_paths_iterator()
+		get_definition_paths_iterator()
+(in lib_repo)		get_scripts_path()
(in lib_repo)		getAffected()
-(in OvalMetadata)		 getProduct()
(in OvalGenerator)
get_element_repository_path()
-(in lib_repo)		getClass()
-(in OvalDefinition)		get_element_change_status()
+(in lib_xml)		get_uncommitted_oval()
+(in lib_git)		 getReferencingIDs()
(in OvalDefinition)
get_element_type_from_oval_id()
-(in lib_repo)		getDefinitions()
-(in OvalDocument)		get_element_metadata()
+(in lib_xml)		getAffected()
+(in OvalMetadata)		 getSchemaShortName()
(in OvalElement)
get_element_type_from_path()
+		get_element_paths_iterator()
(in lib_repo)		getDescription()
-(in OvalMetadata)		getClass()
+(in OvalDefinition)		 getSchemaVersion()
(in OvalGenerator)
get_fieldnames()
-(in SearchIndex)		getDocumentRoot()
+		get_element_repository_path()
+(in lib_repo)		getDefinitions()
(in OvalDocument)		 getStates()
(in OvalDocument)
get_index()
-(in SearchIndex)		getElement()
-(in OvalElement)		get_element_type_from_oval_id()
+(in lib_repo)		getDescription()
+(in OvalMetadata)		 getStatus()
(in OvalRepositoryInformation)
get_index_bucket()
+		get_element_type_from_path()
(in lib_repo)		getElementByID()
+		getDocumentRoot()
(in OvalDocument)		 getTests()
(in OvalDocument)
get_indexed_commit_hash()
+		get_fieldnames()
(in SearchIndex)		getElementTypeFromOvalID()
+		getElement()
(in OvalElement)		 getTimestamp()
(in OvalGenerator)
get_indexed_uncommitted_files()
+		get_index()
(in SearchIndex)		getExtra()
-(in OvalGenerator)		getElementByID()
+(in OvalDocument)		 getTitle()
(in OvalMetadata)
get_local_name()
+		get_index_bucket()
(in lib_repo)		getFileName()
+		getElementTypeFromOvalID()
(in OvalElement)		 getType()
(in OvalDefinition)
get_oval_def_schema()
-(in lib_repo)		getGenerator()
-(in OvalDocument)		get_indexed_commit_hash()
+(in SearchIndex)		getExtra()
+(in OvalGenerator)		 getType()
(in OvalObject)
get_oval_def_schematron()
-(in lib_repo)		getId()
+		get_indexed_uncommitted_files()
+(in SearchIndex)		getFileName()
(in OvalElement)		 getType()
(in OvalState)
get_paths_from_ids()
-(in ElementsIndex)		getIndexSequence()
-(in OvalElement)		get_local_name()
+(in lib_repo)		getGenerator()
+(in OvalDocument)		 getType()
(in OvalTest)
get_related_elementsgetLocalName()
+		get_oval_def_schema()
+(in lib_repo)		getId()
(in OvalElement)		 getType()
(in OvalVariable)
get_repo()
-(in lib_git)		getMetadata()
-(in OvalDefinition)		get_oval_def_schematron()
+(in lib_repo)		getIndexSequence()
+(in OvalElement)		 getVariables()
(in OvalDocument)
get_repository_root_path()
-(in lib_repo)		getMinimumSchemaVersion()
-(in OvalRepositoryInformation)		get_paths_from_ids()
+(in ElementsIndex)		getLocalName()
+(in OvalElement)		 getVersion()
(in OvalElement)
get_repository_statsgetName()
-(in OvalElement)		get_related_elementsgetMetadata()
+(in OvalDefinition)		 grouping_fields
(in get_repository_stats)
get_repo()
+(in lib_git)		getMinimumSchemaVersion()
+(in OvalRepositoryInformation)		 
get_repository_root_path()
+(in lib_repo)		getName()
+(in OvalElement)		 

I
lib_oval lib_search load_standalone_element()
+(in lib_xml)
main()
(in validate_oval_definitons_files)		message()
-(in lib_repo)		message()
+(in SearchIndex)
main()
-(in get_related_elements)		main()
+(in find_affected)		 message()
(in build_oval_definitions_file)		message()
-(in SearchIndex)		message()
+(in OvalGenerator)
main()
-(in get_repository_stats)		main()
+(in get_related_elements)		 message()
(in get_related_elements)		message()
-(in OvalGenerator)		message()
+(in validate_oval_definitons_files)
main()
-(in oval_decomposition)		main()
+(in get_repository_stats)		 message()
(in get_repository_stats)		message()
-(in validate_oval_definitons_files)		 
main()
+(in oval_decomposition)		message()
+(in lib_repo)		 
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Module lib_git

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for Module lib_git<
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:04 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Module lib_oval

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for Module lib_ova
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 NS_OVAL @@ -552,6 +570,26 @@

get_index_bucket

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 fromStandaloneFile
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 setExtra(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 __init__(
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Module lib_repo

Module lib_repo

+   +

+ + + + + +
get_schema_versions()
+ Gets sorted OVAL schema versions contained in the repo.		 + source code + +
+
+ +
+ +
+ + +
+

get_schema_versions() +

+ 		source code  +
+ +

Gets sorted OVAL schema versions contained in the repo. The order must + be newest to oldest

+
+
+
+

@@ -654,7 +692,7 @@

supported_element_types

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for Module lib_rep 19 - get_scripts_path: get full/real path for scripts folder 20 - get_oval_def_schema: gets OVAL definitions schema file for specified schema version 21 - get_oval_def_schematron: gets OVAL definitions schema file for specified schematron version - 22 - message: outputs a message to the CLI - 23 - 24 TODO: - 25 - TBD - 26 """ - 27 + 22 - get_schema_versions: get sorted list of OVAL schema version contained in the repo. + 23 - message: outputs a message to the CLI + 24 + 25 TODO: + 26 - TBD + 27 """ 28 - 29 import inspect - 30 import os.path - 31 import sys - 32 + 29 + 30 import inspect + 31 import os.path + 32 import sys 33 - 34 supported_definition_classes = ('compliance', 'inventory', 'patch', 'vulnerability', 'miscellaneous') - 35 supported_definition_statuses = ('ACCEPTED', 'DEPRECATED', 'DRAFT', 'INTERIM') - 36 supported_element_types = ('definition', 'test', 'object', 'state', 'variable') - 37 + 34 + 35 supported_definition_classes = ('compliance', 'inventory', 'patch', 'vulnerability', 'miscellaneous') + 36 supported_definition_statuses = ('ACCEPTED', 'DEPRECATED', 'DRAFT', 'INTERIM') + 37 supported_element_types = ('definition', 'test', 'object', 'state', 'variable') 38 -
39 -def path_to_oval_id(path): -
40 """ Transform a path to an oval_id. """ - 41 file_name = os.path.basename(path) - 42 oval_id = file_name.replace('_', ':') - 43 return oval_id -
44 - 45 -
46 -def oval_id_to_path(ovalid): -
47 if not ovalid or ovalid is None: - 48 return None - 49 return ovalid.replace(":", "_") + ".xml" -
50 - 51 -
52 -def get_element_type_from_oval_id(oval_id): -
53 """ Gets element type from OVAL id. """ - 54 segments = oval_id.split(':') - 55 if len(segments) != 4: - 56 raise ValueError('Invalid OVAL id: {0}.'.format(oval_id)) - 57 - 58 code = segments[2] - 59 if code == 'def': - 60 element_type = 'definition' - 61 elif code == 'tst': - 62 element_type = 'test' - 63 elif code == 'obj': - 64 element_type = 'object' - 65 elif code == 'ste': - 66 element_type = 'state' - 67 elif code == 'var': - 68 element_type = 'variable' - 69 else: - 70 raise ValueError("Unknown OVAL object type '{0}' in {1}.".format(code, oval_id)) - 71 - 72 return element_type -
73 - 74 -
75 -def get_element_repository_path(element): -
76 """Uses what is know about this element to construct the path that - 77 this element would be written to as a standalone file in the repository - 78 @type element: Element - 79 @param element: The XML element to be written - 80 - 81 @rtype: string - 82 @return: The full path to the repository file for this element or None if the path could not be created - 83 """ - 84 if not element or element is None: - 85 return None - 86 - 87 # The path is: repo_base / element type / schema short name / local node name / index bucket / converted file name - 88 # For definitions: repo_base / oval class / - 89 try: - 90 root_path = get_repository_root_path() - 91 if not root_path or root_path is None: - 92 return None - 93 - 94 oval_id = element.get("id") - 95 if not oval_id or oval_id is None: - 96 return None - 97 - 98 element_type = get_element_type_from_oval_id(oval_id) - 99 -100 if element_type == 'definition': -101 defclass = element.get("class") -102 if not defclass or defclass is None: -103 return None -104 return root_path + "/definitions/" + defclass + "/" + oval_id_to_path(oval_id) -105 -106 else: -107 platform = get_schema_short_name(element) -108 if not platform or platform is None: -109 return None -110 -111 predicate = get_local_name(element) -112 if not predicate or predicate is None: -113 return None -114 -115 bucket = get_index_bucket(oval_id) -116 if bucket < 1000: -117 return None -118 return root_path + "/" + element_type + "s/" + platform + "/" + predicate + "/" + str(bucket) + "/" + oval_id_to_path(oval_id) -119 -120 except Exception: -121 return None -
122 -123 -
124 -def get_schema_short_name(element): -
125 """The schema short name reveals is a stand-in for the platform that this element is for and can be derived from the namespace URI -126 @type element: Element -127 @param element: the XML Element -128 -129 @rtype: string -130 @return: the "short name" of the platform schema or None if it could not be determined -131 """ -132 if not element or element is None: -133 return None -134 -135 tag = element.tag -136 -137 if not tag or tag is None: -138 return None -139 -140 # If the oval ID does not contain a namespace, then we can't determine the schema shortname -141 if not '}' in tag: -142 return 'unknown' -143 -144 try: -145 schema = tag.rsplit('}', 1)[0] -146 if not '#' in schema: -147 return None -148 return schema.rsplit('#', 1)[1].strip() -149 except Exception: -150 return None -
151 -152 -
153 -def get_local_name(element): -
154 """ -155 Just the element name with the schema URI (if any) removed -156 @type element: Element -157 @param element: The XML element -158 -159 @rtype: string -160 @return: the base name of the element or None if it could not be determined -161 """ -162 -163 if not element or element is None: -164 return None -165 -166 #Check if this node name is prefixed by a URI, in which case return every after the URI -167 if '}' in element.tag: -168 return str(element.tag).rsplit('}',1)[1] -169 -170 #If no namespace prefix, just return the node name -171 return element.tag -
172 -173 -174 -
175 -def get_index_bucket(oval_id): -
176 """The current repository schema protects against having too many files in a single directory by -177 creating subdirectories based on the index portion of the OVALID that limits the number of -178 files to 1000. This function determines the proper bucket based on the OVALID -179 @type oval_id: string -180 @param oval_id: The OVALID -181 -182 @rtype: int -183 @return: an integer representing the proper subdirectory for this OVALID, or 0 if it could not -184 be computed. -185 """ -186 if not oval_id or oval_id is None: -187 return 0 -188 -189 # Get the numeric index from the end of the OVAL ID -190 position = oval_id.rfind(':') -191 if position < 0: -192 return 0 -193 -194 try: -195 position = position + 1 -196 index = oval_id[position:] -197 -198 # Apply the modulus function to determine which bucket it belongs to -199 return int(int(index)/1000 + 1) * 1000 -200 # Or another way to do it: -201 # sequence = int(index) -202 # mod = sequence % 1000 -203 # return sequence - mod + 1000 -204 except Exception: -205 return 0 -
206 -207 -208 -
210 """ Gets element type from path. """ -211 return get_element_type_from_oval_id(path_to_oval_id(path)) + 39 +
40 -def path_to_oval_id(path): +
41 """ Transform a path to an oval_id. """ + 42 file_name = os.path.basename(path) + 43 oval_id = file_name.replace('_', ':') + 44 return oval_id +
45 + 46 +
47 -def oval_id_to_path(ovalid): +
48 if not ovalid or ovalid is None: + 49 return None + 50 return ovalid.replace(":", "_") + ".xml" +
51 + 52 +
53 -def get_element_type_from_oval_id(oval_id): +
54 """ Gets element type from OVAL id. """ + 55 + 56 if not oval_id or oval_id is None: + 57 raise ValueError("Empty OVAL ID") + 58 + 59 segments = oval_id.split(':') + 60 if len(segments) != 4: + 61 raise ValueError('Invalid OVAL id: {0}.'.format(oval_id)) + 62 + 63 code = segments[2] + 64 if code == 'def': + 65 element_type = 'definition' + 66 elif code == 'tst': + 67 element_type = 'test' + 68 elif code == 'obj': + 69 element_type = 'object' + 70 elif code == 'ste': + 71 element_type = 'state' + 72 elif code == 'var': + 73 element_type = 'variable' + 74 else: + 75 raise ValueError("Unknown OVAL object type '{0}' in {1}.".format(code, oval_id)) + 76 + 77 return element_type +
78 + 79 +
80 -def get_element_repository_path(element): +
81 """Uses what is know about this element to construct the path that + 82 this element would be written to as a standalone file in the repository + 83 @type element: Element + 84 @param element: The XML element to be written + 85 + 86 @rtype: string + 87 @return: The full path to the repository file for this element or None if the path could not be created + 88 """ + 89 if element is None: + 90 return None + 91 + 92 # The path is: repo_base / element type / schema short name / local node name / index bucket / converted file name + 93 # For definitions: repo_base / oval class / + 94 try: + 95 root_path = get_repository_root_path() + 96 if not root_path or root_path is None: + 97 return None + 98 + 99 oval_id = element.get("id") +100 if not oval_id or oval_id is None: +101 return None +102 +103 element_type = get_element_type_from_oval_id(oval_id) +104 +105 if element_type == 'definition': +106 defclass = element.get("class") +107 if not defclass or defclass is None: +108 return None +109 return root_path + "/definitions/" + defclass + "/" + oval_id_to_path(oval_id) +110 +111 else: +112 platform = get_schema_short_name(element) +113 if not platform or platform is None: +114 return None +115 +116 predicate = get_local_name(element) +117 if not predicate or predicate is None: +118 return None +119 +120 bucket = get_index_bucket(oval_id) +121 if bucket < 1000: +122 return None +123 return root_path + "/" + element_type + "s/" + platform + "/" + predicate + "/" + str(bucket) + "/" + oval_id_to_path(oval_id) +124 +125 except Exception: +126 return None +
127 +128 +129 +
130 -def get_schema_short_name(element): +
131 """The schema short name reveals is a stand-in for the platform that this element is for and can be derived from the namespace URI +132 @type element: Element +133 @param element: the XML Element +134 +135 @rtype: string +136 @return: the "short name" of the platform schema or None if it could not be determined +137 """ +138 if element is None: +139 return None +140 +141 tag = element.tag +142 +143 if not tag or tag is None: +144 return None +145 +146 # If the oval ID does not contain a namespace, then we can't determine the schema shortname +147 if not '}' in tag: +148 return 'unknown' +149 +150 try: +151 schema = tag.rsplit('}', 1)[0] +152 if not '#' in schema: +153 return None +154 return schema.rsplit('#', 1)[1].strip() +155 except Exception: +156 return None +
157 +158 +
159 -def get_local_name(element): +
160 """ +161 Just the element name with the schema URI (if any) removed +162 @type element: Element +163 @param element: The XML element +164 +165 @rtype: string +166 @return: the base name of the element or None if it could not be determined +167 """ +168 +169 if element is None: +170 return None +171 +172 #Check if this node name is prefixed by a URI, in which case return every after the URI +173 if '}' in element.tag: +174 return str(element.tag).rsplit('}',1)[1] +175 +176 #If no namespace prefix, just return the node name +177 return element.tag +
178 +179 +180 +
181 -def get_index_bucket(oval_id): +
182 """The current repository schema protects against having too many files in a single directory by +183 creating subdirectories based on the index portion of the OVALID that limits the number of +184 files to 1000. This function determines the proper bucket based on the OVALID +185 @type oval_id: string +186 @param oval_id: The OVALID +187 +188 @rtype: int +189 @return: an integer representing the proper subdirectory for this OVALID, or 0 if it could not +190 be computed. +191 """ +192 if not oval_id or oval_id is None: +193 return 0 +194 +195 # Get the numeric index from the end of the OVAL ID +196 position = oval_id.rfind(':') +197 if position < 0: +198 return 0 +199 +200 try: +201 position = position + 1 +202 index = oval_id[position:] +203 +204 # Apply the modulus function to determine which bucket it belongs to +205 return int(int(index)/1000 + 1) * 1000 +206 # Or another way to do it: +207 # sequence = int(index) +208 # mod = sequence % 1000 +209 # return sequence - mod + 1000 +210 except Exception: +211 return 0
212 213 -
215 """ Returns an iterator of all definition paths. """ -216 definitions_root = os.path.join(get_repository_root_path(), 'definitions') -217 for dirpath, dirnames, filenames in os.walk(definitions_root): -218 for filename in filenames: -219 if filename.endswith('.xml'): -220 yield os.path.join(dirpath, filename) -
221 -222 -
224 """ Returns an iterator of all OVAL element paths. """ -225 elements_root = os.path.join(get_repository_root_path()) -226 for dirpath, dirnames, filenames in os.walk(elements_root): -227 for filename in filenames: -228 if filename.endswith('.xml'): -229 yield os.path.join(dirpath, filename) -
230 -231 -
232 -def get_root_path(): -
233 """ Gets root path. """ -234 global root_path -235 if 'root_path' not in globals(): -236 scripts_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) -237 root_path = os.path.realpath(scripts_path + '/../') -238 return root_path -
239 -240 -
242 """ Gets repository root path. """ -243 return os.path.realpath(get_root_path() + '/repository' ) -
244 -245 -
246 -def get_scripts_path(): -
247 """ Gets scripts folder root path. """ -248 return os.path.realpath(get_root_path() + '/scripts' ) +214 +
216 """ Gets element type from path. """ +217 return get_element_type_from_oval_id(path_to_oval_id(path)) +
218 +219 +
221 """ Returns an iterator of all definition paths. """ +222 definitions_root = os.path.join(get_repository_root_path(), 'definitions') +223 for dirpath, dirnames, filenames in os.walk(definitions_root): +224 for filename in filenames: +225 if filename.endswith('.xml'): +226 yield os.path.join(dirpath, filename) +
227 +228 +
230 """ Returns an iterator of all OVAL element paths. """ +231 elements_root = os.path.join(get_repository_root_path()) +232 for dirpath, dirnames, filenames in os.walk(elements_root): +233 for filename in filenames: +234 if filename.endswith('.xml'): +235 yield os.path.join(dirpath, filename) +
236 +237 +
238 -def get_root_path(): +
239 """ Gets root path. """ +240 global root_path +241 if 'root_path' not in globals(): +242 scripts_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +243 root_path = os.path.realpath(scripts_path + '/../') +244 return root_path +
245 +
247 """ Gets sorted OVAL schema versions contained in the repo. The order must be newest to oldest """ +248 return ["5.11.1","5.11","5.10.1","5.10","5.9","5.8","5.7","5.6","5.5","5.4","5.3","5.2","5.1","5.0"]
249 -250 -
251 -def get_oval_def_schema(schema_version='5.11.1'): -
252 """ Gets OVAL definitions schema file for specified schema version. """ -253 return os.path.realpath(get_root_path() + '/oval_schemas/' + schema_version + '/all-oval-definitions.xsd' ) -
254 -255 -
256 -def get_oval_def_schematron(schema_version='5.11.1'): -
257 """ Gets OVAL definitions schema file for specified schema version. """ -258 return os.path.realpath(get_root_path() + '/oval_schemas/' + schema_version + '/all-oval-definitions-schematron.xsl' ) -
259 -260 -
261 -def message(type, message): -
262 """ Print a message """ -263 sys.stdout.write('\r{0}: {1}\n'.format(type.upper(),
251 """ Gets repository root path. """ +252 return os.path.realpath(get_root_path() + '/repository' ) +
253 +254 +
255 -def get_scripts_path(): +
256 """ Gets scripts folder root path. """ +257 return os.path.realpath(get_root_path() + '/scripts' ) +
258 +259 +
260 -def get_oval_def_schema(schema_version='5.11.1'): +
261 """ Gets OVAL definitions schema file for specified schema version. """ +262 return os.path.realpath(get_root_path() + '/oval_schemas/' + schema_version + '/all-oval-definitions.xsd' ) +
263 +264 +
265 -def get_oval_def_schematron(schema_version='5.11.1'): +
266 """ Gets OVAL definitions schema file for specified schema version. """ +267 return os.path.realpath(get_root_path() + '/oval_schemas/' + schema_version + '/all-oval-definitions-schematron.xsl' ) +
268 +269 +
270 -def message(type, message): +
271 """ Print a message """ +272 sys.stdout.write('\r{0}: {1}\n'.format(type.upper(), message)) -
264 + + diff --git a/scripts/Documentation/toc-lib_repo-module.html b/scripts/Documentation/toc-lib_repo-module.html index 11b80738754d..07b9dfa2456b 100644 --- a/scripts/Documentation/toc-lib_repo-module.html +++ b/scripts/Documentation/toc-lib_repo-module.html @@ -25,7 +25,8 @@

Functions

>get_oval_def_schematron
get_repository_root_path
get_root_path
get_schema_short_name
get_schema_short_name
get_schema_versions
get_scripts_path
message
oval_id_to_path
Classes >SchematronValidationError

Functions

apply_xslt
get_definition_metadata
get_definition_metadata
get_element_change_status
get_element_metadata
get_schematron_xsl_from_schema
get_schematron_xsl_from_schema
load_standalone_element
schema_validate
schematron_validate

Variables

Table of Contents

Modules

build_oval_definitions_file
build_oval_definitions_file
find_affected
get_related_elements
get_repository_stats
lib_git
Module validate_oval_definitons_files

- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:03 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 Source Code for 72 73 74 if __name__ == '__main__': -75 75 Source Code for
- Generated by Epydoc 3.0.1 on Thu Jul 16 16:37:04 2015 + Generated by Epydoc 3.0.1 on Fri Jul 17 15:04:21 2015 + + + + Internet Explorer memory corruption vulnerability - CVE-2015-0056 (MS15-018) + + Microsoft Windows 7 + Microsoft Windows Server 2008 R2 + Microsoft Windows 8.1 + Microsoft Windows Server 2012 R2 + Microsoft Internet Explorer 11 + + + Microsoft Internet Explorer 11 allows remote attackers to execute arbitrary code or cause a denial of service (memory corruption) via a crafted web site, aka "Internet Explorer Memory Corruption Vulnerability," a different vulnerability than CVE-2015-1623 and CVE-2015-1626. + + + + SecPod Team + + DRAFT + INTERIM + ACCEPTED + + ACCEPTED + + + + + + + + + + + + + + + + + 3.3.0 + + + 3.3.0 + + + 3.3.0 + + + 3.3.5 + + \ No newline at end of file diff --git a/scripts/Test/OvalDefinition.py b/scripts/Test/oval_definition.py similarity index 100% rename from scripts/Test/OvalDefinition.py rename to scripts/Test/oval_definition.py diff --git a/scripts/Test/test_for_changes.py b/scripts/Test/test_for_changes.py new file mode 100644 index 000000000000..250102b86712 --- /dev/null +++ b/scripts/Test/test_for_changes.py @@ -0,0 +1,41 @@ +""" +""" + + +import sys +from lxml import etree + + +sys.path.insert(0, "../") +import lib_xml +import lib_repo + + +def main(): + + print("\n--- Starting tests...\n") + tree = etree.parse('change_tests.xml') + if tree is None: + print("ERROR", "Unable to parse test definition file into XML") + return + + root = tree.getroot() + if root is None: + print("ERROR", "Could not locate document root") + return + + test_cases = list(root) + + for element in test_cases: + # Get the corresponding element from the repository + ovalid = element.get('id') + path = lib_repo.get_element_repository_path(element) + existing = lib_xml.load_standalone_element(path) + status = lib_xml.get_element_change_status(element, existing) + print(" ++ Status for element {0} = {1}".format(ovalid, status)) + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/lib_repo.py b/scripts/lib_repo.py index d26f85b339b9..ca507695c4df 100644 --- a/scripts/lib_repo.py +++ b/scripts/lib_repo.py @@ -52,6 +52,10 @@ def oval_id_to_path(ovalid): def get_element_type_from_oval_id(oval_id): """ Gets element type from OVAL id. """ + + if not oval_id or oval_id is None: + raise ValueError("Empty OVAL ID") + segments = oval_id.split(':') if len(segments) != 4: raise ValueError('Invalid OVAL id: {0}.'.format(oval_id)) @@ -82,7 +86,7 @@ def get_element_repository_path(element): @rtype: string @return: The full path to the repository file for this element or None if the path could not be created """ - if not element or element is None: + if element is None: return None # The path is: repo_base / element type / schema short name / local node name / index bucket / converted file name @@ -120,7 +124,8 @@ def get_element_repository_path(element): except Exception: return None - + + def get_schema_short_name(element): """The schema short name reveals is a stand-in for the platform that this element is for and can be derived from the namespace URI @@ -130,7 +135,7 @@ def get_schema_short_name(element): @rtype: string @return: the "short name" of the platform schema or None if it could not be determined """ - if not element or element is None: + if element is None: return None tag = element.tag @@ -161,7 +166,7 @@ def get_local_name(element): @return: the base name of the element or None if it could not be determined """ - if not element or element is None: + if element is None: return None #Check if this node name is prefixed by a URI, in which case return every after the URI diff --git a/scripts/lib_xml.py b/scripts/lib_xml.py index bdad2e245366..092cb6359104 100644 --- a/scripts/lib_xml.py +++ b/scripts/lib_xml.py @@ -2,6 +2,8 @@ """Library for parsing, creating and validating XML files. Authors: David Ries + Gunnar Engelbach + This library provides methods to parse OVAL XML, create OVAL definitions, validate OVAL definitions and other XML-related functions TBD. @@ -26,11 +28,145 @@ - Schematron validation (xsl transform) is extremely slow; improve or replace (?saxon?) """ -import sys, os, os.path, inspect, datetime, random, re -from xml.sax.saxutils import escape +import os.path +import inspect +import datetime +import random +import re +import lib_repo +#from xml.sax.saxutils import escape from lxml import etree + +def get_element_change_status(left, right): + """Compare two OVAL elements recursively and determine if any substantive changes have been made. + + @type left: Element + @param left: the new element + @type right: Element + @param right: the existing element in the repository + + @rtype: int + @return: And integer value indicating the level of change: + -2 = an error has occurred + -1 = these are not the same elements + 0 = no changes or the elements are not the same item + 1 = has changes, but they are inconsequential (have no affect on functionality) + 2 = changes of consequence + """ + + import traceback + + ignore_attributes = ['version'] + unimportant_attributes = ['comment'] + + + if left is None: + if right is None: + return -1 + else: + return False + elif right is None: + return -1 + + try: + # Compare tags + if left.tag != right.tag: + return -1 + + # If this is the top-level element, make certain we really are dealing with the same elements + left_id = left.get('id') + if left_id is not None: # Only primary elements have OVAL IDs + left_type = lib_repo.get_element_type_from_oval_id(left_id) + if left_type is not None and left_type in lib_repo.supported_element_types: + right_id = right.get('id') + right_type = lib_repo.get_element_type_from_oval_id(right_id) + if not right_id or right_id is None: + return -1 + if left_type != right_type: + return -1 + + # Compare text + if left.text is not None: + if right.text is not None: + if left.text.strip() != right.text.strip(): + return 2 + else: + return 2 + elif right.text is not None: + return 2 + + + # Compare tails? Do OVAL Elements even have tails? +# if left.tail != right.tail: +# return 2 + + # Compare attributes + # But don't compare the version attribute -- so we can't just use cmp() + try: + for key in left.keys(): + if not key in ignore_attributes: + lvalue = left.get(key) + rvalue = right.get(key) + if lvalue != rvalue: + if key in unimportant_attributes: + return 1 + else: + return 2 + except: + # An exception at this point probably indicates that the attributes don't match + return 2 + + # Compare children (recurse on this method) + # Be sure we are enumerating the same children in the same order + result = 0 + + left_children = list(left) + right_children = list(right) + if len(left_children) != len(right_children): + return 2 + + for index, lchild in enumerate(left_children): + # Get the matching right_child, if any + # and compare children + rchild = right_children[index] + sub_result = get_element_change_status(lchild, rchild) + if sub_result != 0 and sub_result != 1: # Anything else is reason to stop processing + return sub_result + elif sub_result == 1: + result = 1 + + return result + + except: + print (traceback.format_exc()) + return -2 + + +def load_standalone_element(path): + """Assuming that the path references a file that exists and the contents of that file is valid XXML + that contains a single OVAL Element, loads and returns that element or None if this could not be done. + + @type path: string + @param path: the path to the file + + @rtype: lxml.etree.element + @return: the XML element or None + """ + + if not path or path is None: + return None + + try: + tree = etree.parse(path) + return tree.getroot() + except Exception: + return None + + + + def get_definition_metadata(filepath): """ Takes a filepath for an OVAL definition file and returns dictionary of metadata. """ try: