diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index bb111b2b8..60fde0c81 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -23,6 +23,6 @@ jobs: VALIDATE_PYTHON_BLACK: true VALIDATE_ALL_CODEBASE: false VALIDATE_PYTHON_MYPY: true - + PYTHON_MYPY_CONFIG_FILE: .mypy.ini DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 000000000..a684f1dd6 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,2 @@ +[mypy] +python_version = 2.8 diff --git a/Procfile b/Procfile index 52d23bfd2..4c55e4074 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: gunicorn cre:app --log-file=- \ No newline at end of file +web: cp cres/db.sqlite . && gunicorn cre:app --log-file=- \ No newline at end of file diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index 844f37bbd..ac2410944 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -76,15 +76,13 @@ def register_standard( def register_cre(cre: defs.CRE, collection: db.Standard_collection) -> db.CRE: - dbcre: db.CRE = collection.add_cre(cre) - for link in cre.links: - if type(link.document).__name__ == defs.CRE.__name__: + if type(link.document) == defs.CRE: collection.add_internal_link( dbcre, register_cre(link.document, collection), type=link.ltype ) - elif type(link.document).__name__ == defs.Standard.__name__: + elif type(link.document) == defs.Standard: collection.add_link( cre=dbcre, standard=register_standard( @@ -98,7 +96,6 @@ def register_cre(cre: defs.CRE, collection: db.Standard_collection) -> db.CRE: def parse_file( filename: str, yamldocs: List[Dict[str, Any]], scollection: db.Standard_collection ) -> Optional[List[defs.Document]]: - """given yaml from export format deserialise to internal standards format and add standards to db""" resulting_objects = [] @@ -156,7 +153,6 @@ def parse_file( def parse_standards_from_spreadsheeet( cre_file: List[Dict[str, Any]], result: db.Standard_collection ) -> None: - """given a yaml with standards, build a list of standards in the db""" hi_lvl_CREs = {} cres = {} @@ -197,7 +193,6 @@ def get_standards_files_from_disk(cre_loc: str) -> Generator[str, None, None]: def add_from_spreadsheet(spreadsheet_url: str, cache_loc: str, cre_loc: str) -> None: - """--add --from_spreadsheet use the cre db in this repo import new mappings from @@ -218,7 +213,6 @@ def add_from_spreadsheet(spreadsheet_url: str, cache_loc: str, cre_loc: str) -> def add_from_disk(cache_loc: str, cre_loc: str) -> None: - """--add --cre_loc use the cre db in this repo import new mappings from @@ -236,7 +230,6 @@ def add_from_disk(cache_loc: str, cre_loc: str) -> None: def review_from_spreadsheet(cache: str, spreadsheet_url: str, share_with: str) -> None: - """--review --from_spreadsheet copy db to new temp dir, import new mappings from spreadsheet @@ -266,7 +259,6 @@ def review_from_spreadsheet(cache: str, spreadsheet_url: str, share_with: str) - def review_from_disk(cache: str, cre_file_loc: str, share_with: str) -> None: - """--review --cre_loc copy db to new temp dir, import new mappings from yaml files defined in @@ -298,7 +290,6 @@ def review_from_disk(cache: str, cre_file_loc: str, share_with: str) -> None: def print_graph() -> None: - """export db to single json object, pass to visualise.html so it can be shown in browser""" raise NotImplementedError @@ -347,7 +338,6 @@ def create_spreadsheet( title: str, share_with: List[str], ) -> Any: - """Reads cre docs exported from a standards_collection.export() dumps each doc into a workbook""" flat_dicts = sheet_utils.prepare_spreadsheet( diff --git a/application/database/db.py b/application/database/db.py index d93b7c844..ee4041773 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Tuple +from collections import Counter import networkx as nx # type: ignore import yaml from flask_sqlalchemy.model import DefaultMeta @@ -31,8 +32,7 @@ class Standard(BaseModel): # type: ignore # which subpart of are we linking to subsection = sqla.Column(sqla.String) tags = sqla.Column(sqla.String, default="") # coma separated tags - - version = sqla.Column(sqla.String) + version = sqla.Column(sqla.String, default="") # some external link to where this is, usually a URL with an anchor link = sqla.Column(sqla.String, default="") @@ -82,7 +82,7 @@ def __init__(self) -> None: def __load_cre_graph(self) -> nx.Graph: - graph = nx.Graph() + graph = nx.DiGraph() for il in self.session.query(InternalLinks).all(): graph.add_node(f"CRE: {il.group}") graph.add_node(f"CRE: {il.cre}") @@ -141,27 +141,14 @@ def get_standards_names(self) -> List[str]: return res def get_max_internal_connections(self) -> int: - count: Dict[int, int] = {} - - # TODO: (spyros) this should be made into a count(*) query q = self.session.query(InternalLinks).all() - for il in q: - if il.group in count.keys(): - count[il.group] += 1 - else: - count[il.group] = 1 - if il.cre in count.keys(): - count[il.cre] += 1 - else: - count[il.cre] = 1 - if count: - return max(count.values()) - else: - return 0 + grp_count = Counter([x.group for x in q]) or {0: 0} + cre_count = Counter([x.cre for x in q]) or {0: 0} + return max([max(cre_count.values()), max(grp_count.values())]) def find_cres_of_cre(self, cre: CRE) -> Optional[List[CRE]]: - - """returns the higher level CREs of the cre or none if no higher level cres link to it""" + """returns the higher level CREs of the cre or none + if no higher level cres link to it""" cre_id = self.session.query(CRE.id).filter(CRE.name == cre.name).first() links = ( self.session.query(InternalLinks).filter(InternalLinks.cre == cre_id).all() @@ -177,7 +164,8 @@ def find_cres_of_cre(self, cre: CRE) -> Optional[List[CRE]]: return None def find_cres_of_standard(self, standard: Standard) -> Optional[List[CRE]]: - """returns the CREs that link to this standard or none if none link to it""" + """returns the CREs that link to this standard or none + if none link to it""" if not standard.id: standard = ( self.session.query(Standard) @@ -202,11 +190,12 @@ def find_cres_of_standard(self, standard: Standard) -> Optional[List[CRE]]: return result or None def get_by_tags(self, tags: List[str]) -> List[cre_defs.Document]: - """Returns the cre_defs.Documents and their Links that are tagged with ALL of the tags provided """ - # TODO: (spyros), when we have useful tags this needs to be refactored so both standards and CREs become the same query and it gets paginated + # TODO: (spyros), when we have useful tags this needs to be refactored + # so both standards and CREs become the same query + # and it gets paginated standards_where_clause = [] cre_where_clause = [] documents = [] @@ -233,7 +222,8 @@ def get_by_tags(self, tags: List[str]) -> List[cre_defs.Document]: documents.extend(standard) else: logger.fatal( - "db.get_standard returned None for Standard %s:%s that exists, BUG!" + "db.get_standard returned None for" + "Standard %s:%s that exists, BUG!" % (standard.name, standard.section) ) @@ -329,8 +319,6 @@ def __get_standards_query__( page: int = 0, items_per_page: Optional[int] = None, ) -> sqla.Query: - - total_pages = 0 query = Standard.query.filter(Standard.name == name) if section: query = query.filter(Standard.section == section) @@ -348,7 +336,7 @@ def get_CRE( cre = None query = CRE.query if not external_id and not name: - logger.error(f"You need to search by either external_id or name") + logger.error("You need to search by either external_id or name") return None if external_id: @@ -364,7 +352,8 @@ def get_CRE( return None - # todo figure a way to return both the Standard and the link_type for that link + # todo figure a way to return both the Standard + # and the link_type for that link linked_standards = self.session.query(Links).filter(Links.cre == dbcre.id).all() for ls in linked_standards: cre.add_link( @@ -469,33 +458,29 @@ def export(self, dir: str) -> List[cre_defs.Document]: def add_cre(self, cre: cre_defs.CRE) -> CRE: entry: CRE - - if cre.id != None: - entry = ( - self.session.query(CRE) - .filter(CRE.name == cre.name, CRE.external_id == cre.id) - .first() - ) + query: sqla.Query = self.session.query(CRE).filter(CRE.name == cre.name) + if cre.id: + entry = query.filter(CRE.external_id == cre.id).first() else: - entry = ( - self.session.query(CRE) - .filter(CRE.name == cre.name, CRE.description == cre.description) - .first() - ) + entry = query.filter(CRE.description == cre.description).first() if entry is not None: logger.debug("knew of %s ,skipping" % cre.name) return entry else: logger.debug("did not know of %s ,adding" % cre.name) - entry = CRE(description=cre.description, name=cre.name, external_id=cre.id) + entry = CRE( + description=cre.description, + name=cre.name, + external_id=cre.id, + tags=",".join([str(t) for t in cre.tags]), + ) self.session.add(entry) self.session.commit() self.cre_graph.add_node(f"CRE: {entry.id}") return entry def add_standard(self, standard: cre_defs.Standard) -> Standard: - entry: Standard = Standard.query.filter( sqla.and_( Standard.name == standard.name, @@ -517,6 +502,7 @@ def add_standard(self, standard: cre_defs.Standard) -> Standard: subsection=standard.subsection, link=standard.hyperlink, version=standard.version, + tags=",".join([str(t) for t in standard.tags]), ) self.session.add(entry) self.session.commit() @@ -529,10 +515,14 @@ def __introduces_cycle(self, node_from: str, node_to: str) -> Any: existing_cycle = nx.find_cycle(self.cre_graph) if existing_cycle: logger.fatal( - f"Existing graph contains cycle, this not a recoverable error, manual database actions are required {existing_cycle}" + "Existing graph contains cycle," + "this not a recoverable error," + f" manual database actions are required {existing_cycle}" ) raise ValueError( - f"Existing graph contains cycle, this not a recoverable error, manual database actions are required {existing_cycle}" + "Existing graph contains cycle," + "this not a recoverable error," + f" manual database actions are required {existing_cycle}" ) except nx.exception.NetworkXNoCycle: pass # happy path, we don't want cycles @@ -547,8 +537,8 @@ def add_internal_link( self, group: CRE, cre: CRE, type: cre_defs.LinkTypes = cre_defs.LinkTypes.Same ) -> None: - if cre.id == None: - if cre.external_id == None: + if cre.id is None: + if cre.external_id is None: cre = ( self.session.query(CRE) .filter( @@ -568,8 +558,8 @@ def add_internal_link( ) .first() ) - if group.id == None: - if group.external_id == None: + if group.id is None: + if group.external_id is None: group = ( self.session.query(CRE) .filter( @@ -589,11 +579,11 @@ def add_internal_link( ) .first() ) - if cre == None or group == None: + if cre is None or group is None: logger.fatal( - "Tried to insert internal mapping with element that doesn't exist in db, this looks like a bug" + "Tried to insert internal mapping with element" + " that doesn't exist in db, this looks like a bug" ) - return None entry = ( @@ -610,7 +600,7 @@ def add_internal_link( ) .first() ) - if entry != None: + if entry is not None: logger.debug(f"knew of internal link {cre.name} == {group.name} ,updating") entry.type = type.value self.session.commit() @@ -619,7 +609,9 @@ def add_internal_link( else: logger.debug( - f"did not know of internal link {group.external_id}:{group.name} == {cre.external_id}:{cre.name} ,adding" + "did not know of internal link" + f" {group.external_id}:{group.name}" + f" == {cre.external_id}:{cre.name} ,adding" ) cycle = self.__introduces_cycle(f"CRE: {group.id}", f"CRE: {cre.id}") if not cycle: @@ -630,9 +622,10 @@ def add_internal_link( self.cre_graph.add_edge(f"CRE: {group.id}", f"CRE: {cre.id}") else: logger.warning( - f"A link between CREs {group.external_id} and {cre.external_id} would introduce a cycle, skipping" + f"A link between CREs {group.external_id} and" + f" {cre.external_id} " + f"would introduce cycle {cycle}, skipping" ) - logger.debug(cycle) def add_link( self, @@ -641,11 +634,11 @@ def add_link( type: cre_defs.LinkTypes = cre_defs.LinkTypes.Same, ) -> None: - if cre.id == None: + if cre.id is None: cre = ( self.session.query(CRE).filter(sqla.and_(CRE.name == cre.name)).first() ) - if standard.id == None: + if standard.id is None: standard = ( self.session.query(Standard) .filter( @@ -666,33 +659,45 @@ def add_link( ) if entry: logger.debug( - f"knew of link {standard.name}:{standard.section}=={cre.name} ,updating" + f"knew of link {standard.name}:{standard.section}" + f"=={cre.name} ,updating" ) entry.type = type.value self.session.commit() return else: - # Since multiple CREs need to link to the same standard, cycles are inevitable - # cycle = self.__introduces_cycle( - # f"CRE: {cre.id}", f"Standard: {str(standard.id)}") - # if not cycle: - logger.debug( - f"did not know of link {standard.id}){standard.name}:{standard.section}=={cre.id}){cre.name} ,adding" + cycle = self.__introduces_cycle( + f"CRE: {cre.id}", f"Standard: {str(standard.id)}" ) - self.session.add(Links(type=type.value, cre=cre.id, standard=standard.id)) - self.cre_graph.add_edge(f"CRE: {cre.id}", f"Standard: {str(standard.id)}") - # else: - # logger.warning(f"A link between CRE {cre.external_id} and Standard: {standard.name}:{standard.section}:{standard.subsection}" - # " would introduce a cycle, skipping") - # logger.debug(f"{cycle}") + if not cycle: + logger.debug( + f"did not know of link {standard.id})" + f"{standard.name}:{standard.section}=={cre.id}){cre.name}" + " ,adding" + ) + self.session.add( + Links(type=type.value, cre=cre.id, standard=standard.id) + ) + self.cre_graph.add_edge( + f"CRE: {cre.id}", f"Standard: {str(standard.id)}" + ) + else: + logger.warning( + f"A link between CRE {cre.external_id}" + f" and Standard: {standard.name}" + f":{standard.section}:{standard.subsection}" + f" would introduce cycle {cycle}, skipping" + ) + logger.debug(f"{cycle}") self.session.commit() def find_path_between_standards( self, standard_source_id: int, standard_destination_id: int ) -> bool: - """One line method to return paths in a graph, this starts getting complicated when we have more linktypes""" + """One line method to return paths in a graph, + this starts getting complicated when we have more linktypes""" res: bool = nx.has_path( - self.cre_graph, + self.cre_graph.to_undirected(), "Standard: " + str(standard_source_id), "Standard: " + str(standard_destination_id), ) @@ -700,8 +705,8 @@ def find_path_between_standards( return res def gap_analysis(self, standards: List[str]) -> List[cre_defs.Standard]: - - """Since the CRE structure is a tree-like graph with leaves being standards we can find the paths between standards + """Since the CRE structure is a tree-like graph with + leaves being standards we can find the paths between standards find_path_between_standards() is a graph-path-finding method """ processed_standards = [] diff --git a/application/frontend/src/pages/Search/components/BodyText.scss b/application/frontend/src/pages/Search/components/BodyText.scss new file mode 100644 index 000000000..28900ec79 --- /dev/null +++ b/application/frontend/src/pages/Search/components/BodyText.scss @@ -0,0 +1,6 @@ +.index-text{ + margin-top: 2rem; + width: 67%; + background: white; + padding: 1%; +} diff --git a/application/frontend/src/pages/Search/components/BodyText.tsx b/application/frontend/src/pages/Search/components/BodyText.tsx new file mode 100644 index 000000000..05c66598e --- /dev/null +++ b/application/frontend/src/pages/Search/components/BodyText.tsx @@ -0,0 +1,29 @@ +import React, { useState } from 'react'; +import { useHistory } from 'react-router-dom'; +import { Button, Dropdown, Form, Icon, Input } from 'semantic-ui-react'; + +import { CRE, STANDARD } from '../../../routes'; +import './BodyText.scss'; + +export const SearchBody = () => { + + return ( +
+

OPEN CRE

+Linking standards. By the community, for the community. +Open Common Requirement Enumeration links standards and guidelines at the level of topics that need to be taken into account when procuring, designing, building and testing great software. +

WHY?

+

CRE follows up on the ENISA recommendation for a repository to bring the fragmented landscape of security knowledge resources together. +This initiative was taken by an independent group of software security experts who got together to make it 1) easier to create and maintain standards and guidelines, and 2) easier to find relevant information. +No more endless searches for relevant rules and information, no more broken links, no more need to cover every topic everywhere - just link. +

+

WHAT?

+

The idea of CRE is to link each section of a resource to a shared topic identifier (a Common Requirement), instead of linking to just 1 or 2 other resources. Through this shared link, all resources map to eachother. This 1) enables standard and guideline makers to work efficiently, 2) it enables users to find the information they need, and 3) it facilitates a shared understanding in the industry of what cyber security is. The key element is self-maintainability: the CRE topic links in the resource can be automatically parsed and used to map everything together. No more manual mapping labour. Furthermore, because the topics are linked as well, users can quickly find more relevant material through related topics. +For example the topic of session time-out will take the user not only to relevant criteria in several standards, but also to testing guides, development tips, more technical detail, threat descriptions, articles etc. Plus, from there the user can navigate to resources about session management in general. +The CRE only links and therefore does not create a new standard of requirements. It is simply a standard on how standards can link together. +

WHEN?

+

Currently, CRE has linked several OWASP standards together, and included other standards (CWE, NIST-800 53, NIST-800 63b), as part of the OWASP Integration standard project. +Join the movement.

+
+ ); +}; diff --git a/application/tests/db_test.py b/application/tests/db_test.py index 02b5c399e..f4134069a 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -27,9 +27,11 @@ def setUp(self) -> None: self.collection = db.Standard_collection() collection = self.collection - dbcre = collection.add_cre(defs.CRE(description="CREdesc", name="CREname")) + dbcre = collection.add_cre( + defs.CRE(id="111-000", description="CREdesc", name="CREname") + ) dbgroup = collection.add_cre( - defs.CRE(description="Groupdesc", name="GroupName") + defs.CRE(id="111-001", description="Groupdesc", name="GroupName") ) dbstandard = collection.add_standard( defs.Standard( @@ -40,7 +42,7 @@ def setUp(self) -> None: ) ) - unlinked = collection.add_standard( + collection.add_standard( defs.Standard( subsection="4.5.6", section="Unlinked", @@ -50,7 +52,6 @@ def setUp(self) -> None: ) collection.session.add(dbcre) - collection.add_link(cre=dbcre, standard=dbstandard) collection.add_internal_link(cre=dbcre, group=dbgroup) @@ -81,6 +82,7 @@ def test_get_by_tags(self) -> None: section="tagsstand", name="tagsstand", link="https://example.com", + version="", tags="tag1, dots.5.5, space 6 , several spaces and newline 7 \n", ) standard = db.StandardFromDB(dbstandard) @@ -150,19 +152,26 @@ def test_export(self) -> None: loc = tempfile.mkdtemp() result = [ defs.CRE( + id="111-001", description="Groupdesc", name="GroupName", links=[ - defs.Link(document=defs.CRE(description="CREdesc", name="CREname")) + defs.Link( + document=defs.CRE( + id="111-000", description="CREdesc", name="CREname" + ) + ) ], ), defs.CRE( - id="", + id="111-000", description="CREdesc", name="CREname", links=[ defs.Link( - document=defs.CRE(description="Groupdesc", name="GroupName") + document=defs.CRE( + id="111-001", description="Groupdesc", name="GroupName" + ) ), defs.Link( document=defs.Standard( @@ -193,6 +202,7 @@ def test_export(self) -> None: doc = yaml.safe_load(f) self.assertDictEqual(group, doc) crename = result[1].name + ".yaml" + self.maxDiff = None with open(os.path.join(loc, crename), "r") as f: doc = yaml.safe_load(f) self.assertDictEqual(cre, doc) @@ -235,7 +245,6 @@ def test_CREfromDB(self) -> None: def test_add_cre(self) -> None: original_desc = str(uuid.uuid4()) name = str(uuid.uuid4()) - gname = str(uuid.uuid4()) c = defs.CRE( id="cid", doctype=defs.Credoctypes.CRE, description=original_desc, name=name @@ -264,9 +273,9 @@ def test_add_cre(self) -> None: self.collection.session.query(db.CRE).filter(db.CRE.name == c.name).first() ) # ensure original description - self.assertEqual(dbcre.description, str(original_desc)) + self.assertEqual(dbcre.description, original_desc) # ensure original description - self.assertEqual(newCRE.description, str(original_desc)) + self.assertEqual(newCRE.description, original_desc) def test_add_standard(self) -> None: original_section = str(uuid.uuid4()) diff --git a/application/utils/parsers.py b/application/utils/parsers.py index 2f8785461..0cde1089d 100644 --- a/application/utils/parsers.py +++ b/application/utils/parsers.py @@ -1,5 +1,6 @@ import logging import re +from copy import copy from pprint import pprint from typing import Any, Dict, List, Optional, Tuple @@ -26,6 +27,7 @@ def is_empty(value: Optional[str]) -> bool: or value.lower() == "no" or value.lower() == "tbd" or value.lower() == "see higher level topic" + or value.lower() == "tbd" ) @@ -423,7 +425,10 @@ def parse_hierarchical_export_format( higher_cre = i break if is_empty(name): - logger.warning("Found entry without a cre name, skipping") + logger.warning( + f'Found entry with ID {mapping.get("CRE ID")}' + " without a cre name, skipping" + ) continue if name in cres.keys(): @@ -553,24 +558,33 @@ def parse_hierarchical_export_format( cre.add_link(defs.Link(document=cheatsheet)) # link CRE to a higher level one - if f"CRE hierarchy {str(higher_cre)}" in mapping: - if cres.get(mapping[f"CRE hierarchy {str(higher_cre)}"]): - cre_hi = cres[mapping[f"CRE hierarchy {str(higher_cre)}"]] - - existing_link = [ - c - for c in cre_hi.links - if c.document.doctype == defs.Credoctypes.CRE - and c.document.name == name - ] - if existing_link: - cre_hi.links[ - cre_hi.links.index(existing_link[0]) - # ugliest way ever to write "update the object in that pointer" - ].document = cre - else: - cre_hi.add_link(defs.Link(document=cre)) - cres[cre_hi.name] = cre_hi + + if higher_cre and cres.get(mapping[f"CRE hierarchy {str(higher_cre)}"]): + cre_hi = cres[mapping[f"CRE hierarchy {str(higher_cre)}"]] + + existing_link = [ + c + for c in cre_hi.links + if c.document.doctype == defs.Credoctypes.CRE + and c.document.name == cre.name + ] + # pprint( + # f'state:{higher_cre},\n {cre_hi},\n\n {cre}, \n\n{existing_link}') + # input() + shallow_copy = copy(cre) # we only need a shallow copy here + shallow_copy.links = [] + # there is no need to capture the entirety of the cre tree, we just need to register this shallow relation + # the "cres" dict should contain the rest of the info + if existing_link: + cre_hi.links[ + cre_hi.links.index(existing_link[0]) + # ugliest way ever to write "update the object in that pointer" + ].document = shallow_copy + else: + cre_hi.add_link(defs.Link(document=shallow_copy)) + cres[cre_hi.name] = cre_hi + else: + pass # add the cre to cres and make the connection if cre: cres[cre.name] = cre return cres diff --git a/application/utils/spreadsheet.py b/application/utils/spreadsheet.py index 7c67dfbf2..fb15d9173 100644 --- a/application/utils/spreadsheet.py +++ b/application/utils/spreadsheet.py @@ -149,7 +149,6 @@ def prepare_spreadsheet( header[defs.ExportFormat.link_type_key(name)] = None maxgroups = collection.get_max_internal_connections() for i in range(0, maxgroups): - header[defs.ExportFormat.linked_cre_id_key(str(i))] = None header[defs.ExportFormat.linked_cre_name_key(str(i))] = None header[defs.ExportFormat.linked_cre_link_type_key(str(i))] = None diff --git a/cres/db.sqlite b/cres/db.sqlite index c093ed3b2..e64e59ad9 100644 Binary files a/cres/db.sqlite and b/cres/db.sqlite differ