# HG changeset patch # User Benoît Allard # Date 1413298176 -7200 # Node ID 8a89b7a591e657c3daef81465d887251219693e4 # Parent 8e23ba7d4167b654b7e6fe756ec73dc1beac98f0# Parent 90852c11fabd6c0b2addc042607bc864c8ebd7d8 merged diff -r 8e23ba7d4167 -r 8a89b7a591e6 CHANGES --- a/CHANGES Wed Sep 24 17:47:14 2014 +0200 +++ b/CHANGES Tue Oct 14 16:49:36 2014 +0200 @@ -1,3 +1,22 @@ +FarolLuz 0.1.1 (????-??-??) +=========================== + +This is the first patch release of FarolLuz 0.1 + +This release add support for reading / writing incomplete CVRF documents. + +Main changes since FarolLuz 0.1: +-------------------------------- +* Allow writing of incomplete CVRF documents. +* Allow parsing of incomplete CVRF documents. +* Add a method to extract a document ID. +* Add methods to extract Product references in a Document. +* Add method to get a Vulnerability Note per ordinal. +* Fix issue where Acknowledgment could only have one Name and Organization. +* Complete the CVRF template with missing elements +* Improve validation. + + FarolLuz 0.1 (2014-09-23) ========================= diff -r 8e23ba7d4167 -r 8a89b7a591e6 farolluz/cvrf.py --- a/farolluz/cvrf.py Wed Sep 24 17:47:14 2014 +0200 +++ b/farolluz/cvrf.py Tue Oct 14 16:49:36 2014 +0200 @@ -65,6 +65,9 @@ def addAlias(self, alias): self._aliases.append(alias) + def getId(self): + return self._id + def validate(self): if not self._id: raise ValidationError('Document ID cannot be left empty') @@ -92,6 +95,9 @@ def setGenerator(self, generator): self._generator = generator + def getId(self): + return self._identification.getId() + def validate(self): if self._identification is None: raise ValidationError('Document Tracking needs to have an Identification') @@ -225,18 +231,19 @@ class CVRFAcknowledgment(object): - def __init__(self, name=None, organization=None, description=None, + def __init__(self, names=[], organizations=[], description=None, url=None): - self._name = name - self._organization = organization + self._names = names + self._organizations = organizations self._description = description self._url = url def getTitle(self): - return "%s - %s" % (self._name, self._organization) + return "%s - %s" % (', '.join(self._names), + ', '.join(self._organizations)) def validate(self): - if (not self._name) and (not self._organization) and (not self._description): + if (not self._names) and (not self._organizations) and (not self._description): raise ValidationError('An Acknowledgment must have at least a Name, an Organization or a Description') @@ -249,18 +256,9 @@ self._products = [] self._groups = [] - def addBranch(self, branch): - parent = self.getBranch(branch.getParent().getPath()) - if parent is self: - self._branches.append(branch) - else: - parent._childs.append(branch) - def addProduct(self, product): - if product not in self._products: - self._products.append(product) - if product._parent is not self: - product._parent._product = product + """ Add to the product list """ + self._products.append(product) def addRelationship(self, rel): self._relationships.append(rel) @@ -396,9 +394,9 @@ def __init__(self, _type, name, parentbranch): self._type = _type self._name = name - self._parentbranch = parentbranch self._childs = [] self._product = None + self.link(parentbranch) def getParent(self): return self._parentbranch @@ -455,6 +453,15 @@ self.getParent()._childs.remove(self) self._parentbranch = None + def link(self, parent): + """ Actually, only set the parent """ + self._parentbranch = parent + if self.isRoot(): + parent._branches.append(self) + else: + parent._childs.append(self) + + def validate(self): if not self._type: raise ValidationError('A Branch must have a Type') @@ -475,10 +482,10 @@ def __init__(self, productid, name, parent, cpe=None): self._productid = productid self._name = name + self._cpe = cpe # Can be None (directly under the tree), a ProductBranch, or a # Relationship - self._parent = parent - self._cpe = cpe + self.link(parent) def isRoot(self): return isinstance(self._parent, CVRFProductTree) @@ -502,13 +509,18 @@ return None def unlink(self): - """ Unset our _parent, and remove us from the _parent._childs """ - if self.isRoot(): - self._parent._products.remove(self) - else: + """ Unset our _parent, and remove us from the _parent._childs + We are still in the product list. + """ + if not self.isRoot(): self._parent._product = None self._parent = None + def link(self, parent): + self._parent = parent + if not self.isRoot(): + parent._product = self + def validate(self): if not self._productid: raise ValidationError('A Product must have a ProductID') @@ -655,6 +667,46 @@ return self._id._value return "#%d" % self._ordinal + def getNote(self, ordinal): + for note in self._notes: + if note._ordinal == ordinal: + return note + return None + + def mentionsProdId(self, productid): + """ Returns in which sub element, self is mentioning the productid """ + for category in (self._productstatuses, self._threats, self._cvsss, self._remediations): + for subelem in category: + if productid in subelem._productids: + yield subelem + + def isMentioningProdId(self, productid): + """ Returns if self is mentioning the productid """ + for e in self.mentionsProdId(productid): + # We only need to know if the generator yield at least one elem. + return True + return False + + def mentionsGroupId(self, groupid): + for category in (self._threats, self._remediations): + for subelem in category: + if groupid in subelem._groupids: + yield subelem + + def isMentioningGroupId(self, groupids): + """ Make sure you call this with a list (not a generator or a tuple) + when wished """ + if not isinstance(groupids, list): + groupids = [groupids] + for groupid in groupids: + print "testing GroupId: ", groupid + for _ in self.mentionsGroupId(groupid): + # We only need to know if the generator yield at least one elem. + print 'True' + return True + print 'False' + return False + def validate(self, productids, groupids): if not self._ordinal: raise ValidationError('A Vulnerability must have an ordinal') @@ -672,10 +724,22 @@ cwe.validate() for status in self._productstatuses: status.validate(productids) + pids = set() + for status in self._productstatuses: + for pid in status._productids: + if pid in pids: + raise ValidationError('ProductID %s mentionned in two different ProductStatuses for Vulnerability %d' % (pid, self._ordinal)) + pids.add(pid) for threat in self._threats: threat.validate(productids, groupids) for cvss in self._cvsss: cvss.validate(productids) + pids = set() + for cvss in self._cvsss: + for pid in (cvss._productids or productids): + if pid in pids: + raise ValidationError('ProductID %s mentionned in two different CVSS Score Sets for Vulnerability %d' % (pid, self._ordinal)) + pids.add(pid) for remediation in self._remediations: remediation.validate(productids, groupids) for reference in self._references: @@ -684,7 +748,6 @@ acknowledgment.validate() - class CVRFInvolvement(object): PARTIES = CVRFPublisher.TYPES STATUSES = ('Open', 'Disputed', 'In Progress', 'Completed', @@ -726,6 +789,7 @@ class CVRFProductStatus(object): TYPES = ('First Affected', 'Known Affected', 'Known Not Affected', 'First Fixed', 'Fixed', 'Recommended', 'Last Affected') + NAME = "Product Status" def __init__(self, _type): self._type = _type self._productids = [] @@ -750,6 +814,7 @@ class CVRFThreat(object): TYPES = ('Impact', 'Exploit Status', 'Target Set') + NAME = "Threat" def __init__(self, _type, description): self._type = _type self._description = description @@ -792,6 +857,7 @@ 'C': {'N':0.0, 'P':0.275, 'C':0.66}, 'I': {'N':0.0, 'P':0.275, 'C':0.66}, 'A': {'N':0.0, 'P':0.275, 'C':0.66}} + NAME = "CVSS Score Set" def __init__(self, basescore): self._basescore = basescore self._temporalscore = None @@ -844,6 +910,7 @@ class CVRFRemediation(object): TYPES = ('Workaround', 'Mitigation', 'Vendor Fix', 'None Available', 'Will Not Fix') + NAME = "Remediation" def __init__(self, _type, description): self._type = _type self._description = description @@ -972,12 +1039,39 @@ products.add(productid) return set(self.getProductForID(p) for p in products) + def isProductOrphan(self, productid): + """ Returns if a productid is mentionned nowhere in the document """ + # We first look at the ProductTree + ptree = self._producttree + for relation in ptree._relationships: + if productid == relation._productreference: + return False + if productid == relation._relatestoproductreference: + return False + groupids = [g._groupid for g in ptree._groups if productid in g._productids] + if len(groupids) > 0: + return False + # Go through all the Vulnerabilities + for vulnerability in self._vulnerabilities: + if vulnerability.isMentioningProdId(productid): + return False + for groupid in groupids: + if vulnerability.isMentioningGroupId(groupid): + return False + return True + def getNote(self, ordinal): for note in self._notes: if note._ordinal == ordinal: return note return None + def getDocId(self): + if self._tracking is not None: + return self._tracking.getId() + # Make up something ... + return self._title.lower() + def validate(self): if not self._title: raise ValidationError('Document Title cannot be empty') diff -r 8e23ba7d4167 -r 8a89b7a591e6 farolluz/parsers/cvrf.py --- a/farolluz/parsers/cvrf.py Wed Sep 24 17:47:14 2014 +0200 +++ b/farolluz/parsers/cvrf.py Tue Oct 14 16:49:36 2014 +0200 @@ -94,9 +94,14 @@ def parseAcknowledgment(elem, ns='cvrf'): + names = [] + for cvrfname in elem.findall(UN(ns, 'Name')): + names.append(cvrfname.text.strip()) + orgs = [] + for cvrforg in elem.findall(UN(ns, 'Organization')): + orgs.append(cvrforg.text.strip()) return CVRFAcknowledgment( - elem.findtext(UN(ns, 'Name')), - elem.findtext(UN(ns, 'Organization')), + names, orgs, elem.findtext(UN(ns, 'Description')), elem.findtext(UN(ns, 'URL')), ) @@ -247,47 +252,51 @@ cvrfdoc.findtext(UN('cvrf', 'DocumentTitle')).strip(), cvrfdoc.findtext(UN('cvrf', 'DocumentType')).strip() ) + cvrfpub = cvrfdoc.find(UN('cvrf', 'DocumentPublisher')) - pub = CVRFPublisher(cvrfpub.attrib['Type'], cvrfpub.attrib.get('VendorID')) - doc.setPublisher(pub) - contact = cvrfpub.find(UN('cvrf', 'ContactDetails')) - if contact is not None: - pub.setContact(contact.text.strip()) - authority = cvrfpub.find(UN('cvrf', 'IssuingAuthority')) - if authority is not None: - pub.setAuthority(authority.text.strip()) + if cvrfpub is not None: + pub = CVRFPublisher(cvrfpub.attrib['Type'], cvrfpub.attrib.get('VendorID')) + doc.setPublisher(pub) + contact = cvrfpub.find(UN('cvrf', 'ContactDetails')) + if contact is not None: + pub.setContact(contact.text.strip()) + authority = cvrfpub.find(UN('cvrf', 'IssuingAuthority')) + if authority is not None: + pub.setAuthority(authority.text.strip()) + cvrftracking = cvrfdoc.find(UN('cvrf', 'DocumentTracking')) - identification = CVRFTrackingID( - cvrftracking.findtext('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'ID')])).strip() - ) - for cvrfalias in cvrftracking.findall('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'Alias')])): - identification.addAlias(cvrfalias.text.strip()) - tracking = CVRFTracking( - identification, - cvrftracking.findtext(UN('cvrf', 'Status')).strip(), - parseVersion(cvrftracking.findtext(UN('cvrf', 'Version')).strip()), - parseDate(cvrftracking.findtext(UN('cvrf', 'InitialReleaseDate')).strip()), - parseDate(cvrftracking.findtext(UN('cvrf', 'CurrentReleaseDate')).strip()) - ) - doc.setTracking(tracking) - for cvrfrev in cvrftracking.findall('/'.join([UN('cvrf', 'RevisionHistory'), UN('cvrf', 'Revision')])): - rev = CVRFRevision( - parseVersion(cvrfrev.findtext(UN('cvrf', 'Number')).strip()), - parseDate(cvrfrev.findtext(UN('cvrf', 'Date')).strip()), - cvrfrev.findtext(UN('cvrf', 'Description')).strip(), + if cvrftracking is not None: + identification = CVRFTrackingID( + cvrftracking.findtext('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'ID')])).strip() ) - tracking.addRevision(rev) + for cvrfalias in cvrftracking.findall('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'Alias')])): + identification.addAlias(cvrfalias.text.strip()) + tracking = CVRFTracking( + identification, + cvrftracking.findtext(UN('cvrf', 'Status')).strip(), + parseVersion(cvrftracking.findtext(UN('cvrf', 'Version')).strip()), + parseDate(cvrftracking.findtext(UN('cvrf', 'InitialReleaseDate')).strip()), + parseDate(cvrftracking.findtext(UN('cvrf', 'CurrentReleaseDate')).strip()) + ) + doc.setTracking(tracking) + for cvrfrev in cvrftracking.findall('/'.join([UN('cvrf', 'RevisionHistory'), UN('cvrf', 'Revision')])): + rev = CVRFRevision( + parseVersion(cvrfrev.findtext(UN('cvrf', 'Number')).strip()), + parseDate(cvrfrev.findtext(UN('cvrf', 'Date')).strip()), + cvrfrev.findtext(UN('cvrf', 'Description')).strip(), + ) + tracking.addRevision(rev) - xmlgenerator = cvrftracking.find(UN('cvrf', 'Generator')) - if xmlgenerator is not None: - generator = CVRFGenerator() - xmlengine = xmlgenerator.findtext(UN('cvrf', 'Engine')) - if xmlengine is not None: - generator.setEngine(xmlengine.strip()) - xmldate = xmlgenerator.findtext(UN('cvrf', 'Date')) - if xmldate is not None: - generator.setDate(parseDate(xmldate.strip())) - tracking.setGenerator(generator) + xmlgenerator = cvrftracking.find(UN('cvrf', 'Generator')) + if xmlgenerator is not None: + generator = CVRFGenerator() + xmlengine = xmlgenerator.findtext(UN('cvrf', 'Engine')) + if xmlengine is not None: + generator.setEngine(xmlengine.strip()) + xmldate = xmlgenerator.findtext(UN('cvrf', 'Date')) + if xmldate is not None: + generator.setDate(parseDate(xmldate.strip())) + tracking.setGenerator(generator) for cvrfnote in cvrfdoc.findall('/'.join([UN('cvrf', 'DocumentNotes'), UN('cvrf', 'Note')])): doc.addNote(parseNote(cvrfnote)) @@ -315,8 +324,8 @@ cvrfptree = cvrfdoc.find(UN('prod', 'ProductTree')) if cvrfptree is not None: producttree = doc.createProductTree() - for branch in parseProdBranch(cvrfptree, producttree): - producttree.addBranch(branch) + # We need to exhaust our generator ... + for _ in parseProdBranch(cvrfptree, producttree): pass for product in cvrfptree.findall(UN('prod', 'FullProductName')): producttree.addProduct(parseFullProductName(product, producttree)) diff -r 8e23ba7d4167 -r 8a89b7a591e6 farolluz/templates/cvrf.j2 --- a/farolluz/templates/cvrf.j2 Wed Sep 24 17:47:14 2014 +0200 +++ b/farolluz/templates/cvrf.j2 Tue Oct 14 16:49:36 2014 +0200 @@ -25,7 +25,7 @@ -{#- Some macros for producttree generation #} +{#- A macro for producttree generation #} {%- macro FullProductNames(producttree, parent) %} {%- for product in producttree._products %} {%- if product._parent is sameas parent %} @@ -36,15 +36,41 @@ {%- endfor %} {%- endmacro %} +{#- Some macros about more generic types #} {%- macro Note(note) -%} {{- note._note | escape -}} -{%- endmacro %} +{%- endmacro -%} + +{%- macro Reference(reference) -%} + + {{ reference._url }} + {{ reference._description }} + +{%- endmacro -%} + +{%- macro Acknowledgment(acknowledgment) -%} + + {%- for name in acknowledgment._names %} + {{ name }} + {%- endfor %} + {%- for organization in acknowledgment._organizations %} + {{ organization }} + {%- endfor %} + {%- if acknowledgment._description %} + {{ acknowledgment._description }} + {%- endif %} + {%- if acknowledgment._url %} + {{ acknowledgment._url }} + {%- endif %} + +{%- endmacro -%} + {{ cvrf._title }} {{ cvrf._type }} - {%- with publisher = cvrf._publisher %} + {%- with publisher = cvrf._publisher %}{% if publisher %} {%- if publisher._contact %} {{ publisher._contact }} @@ -53,8 +79,8 @@ {{ publisher._authority }} {%- endif %} - {%- endwith %} - {%- with tracking = cvrf._tracking %} + {%- endif %}{% endwith %} + {%- with tracking = cvrf._tracking %}{% if tracking %} {{ tracking._identification._id }} @@ -88,7 +114,7 @@ {%- endif %} - {%- endwith %} + {%- endif %}{% endwith %} {%- if cvrf._notes %} {%- for note in cvrf._notes %} @@ -107,30 +133,14 @@ {%- if cvrf._references %} {%- for reference in cvrf._references %} - - {{ reference._url }} - {{ reference._description }} - + {{ Reference(reference) }} {%- endfor %} {%- endif %} {%- if cvrf._acknowledgments %} {%- for acknowledgment in cvrf._acknowledgments %} - - {%- if acknowledgment._name %} - {{ acknowledgment._name }} - {%- endif %} - {%- if acknowledgment._organization %} - {{ acknowledgment._organization }} - {%- endif %} - {%- if acknowledgment._description %} - {{ acknowledgment._description }} - {%- endif %} - {%- if acknowledgment._url %} - {{ acknowledgment._url }} - {%- endif %} - + {{ Acknowledgment(acknowledgment) }} {%- endfor %} {%- endif %} @@ -143,8 +153,8 @@ {{- FullProductNames(producttree, branch) }} {%- endfor %} - {{ FullProductNames(producttree, producttree) }} - {%- for relationship in producttree._relationships -%} + {{- FullProductNames(producttree, producttree) }} + {%- for relationship in producttree._relationships %} {{- FullProductNames(producttree, relationship) }} @@ -152,7 +162,7 @@ {%- if producttree._groups %} {%- for group in producttree._groups %} - + {%- if group._description %} {{ group._description }} {%- endif %} @@ -215,6 +225,21 @@ {%- endfor %} {%- endif %} + {%- if vulnerability._threats %} + + {%- for threat in vulnerability._threats %} + + {{ threat._description }} + {%- for productid in threat._productids %} + {{ productid }} + {%- endfor %} + {%- for groupid in threat._groupids %} + {{ groupid }} + {%- endfor %} + + {%- endfor %} + + {%- endif %} {%- if vulnerability._cvsss %} {%- for cvss in vulnerability._cvsss %} @@ -230,7 +255,7 @@ {{ cvss._vector }} {%- endif %} {%- for productid in cvss._productids %} - {{productid}} + {{ productid }} {%- endfor %} {%- endfor %} @@ -257,6 +282,20 @@ {%- endfor %} {%- endif %} + {%- if vulnerability._references %} + + {%- for reference in vulnerability._references %} + {{ Reference(reference) }} + {%- endfor %} + + {%- endif %} + {%- if vulnerability._acknowledgments %} + + {%- for acknowledgment in vulnerability._acknowledgments %} + {{ Acknowledgment(acknowledgment) }} + {%- endfor %} + + {%- endif %} {%- endfor %}