benoit@0: # -*- coding: utf-8 -*-
benoit@0: # Description:
benoit@0: # Methods for parsing CVRF documents
benoit@0: #
benoit@0: # Authors:
benoit@0: # Benoît Allard <benoit.allard@greenbone.net>
benoit@0: #
benoit@0: # Copyright:
benoit@0: # Copyright (C) 2014 Greenbone Networks GmbH
benoit@0: #
benoit@0: # This program is free software; you can redistribute it and/or
benoit@0: # modify it under the terms of the GNU General Public License
benoit@0: # as published by the Free Software Foundation; either version 2
benoit@0: # of the License, or (at your option) any later version.
benoit@0: #
benoit@0: # This program is distributed in the hope that it will be useful,
benoit@0: # but WITHOUT ANY WARRANTY; without even the implied warranty of
benoit@0: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
benoit@0: # GNU General Public License for more details.
benoit@0: #
benoit@0: # You should have received a copy of the GNU General Public License
benoit@0: # along with this program; if not, write to the Free Software
benoit@0: # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
benoit@0: 
benoit@0: """\
benoit@0: Methods for parsing of CVRF Documents
benoit@0: """
benoit@0: 
benoit@0: from __future__ import print_function
benoit@0: 
benoit@0: import re
benoit@0: import textwrap
benoit@0: import xml.etree.ElementTree as ET
benoit@0: from datetime import datetime, timedelta
benoit@0: 
benoit@0: try:
benoit@0:     from datetime import timezone
benoit@0: except ImportError:
benoit@0:     from ..py2 import FixedTimeZone as timezone
benoit@0: 
benoit@0: from ..cvrf import (CVRF, CVRFPublisher, CVRFTracking, CVRFRevision, CVRFNote,
benoit@0:     CVRFAcknowledgment, CVRFProductBranch, CVRFFullProductName, CVRFGenerator,
benoit@0:     CVRFRelationship, CVRFVulnerability, CVRFVulnerabilityID, CVRFThreat,
benoit@0:     CVRFProductStatus, CVRFCVSSSet, CVRFReference, CVRFRemediation, CVRFGroup,
benoit@1:     CVRFInvolvement, CVRFCWE, CVRFTrackingID, CVRFAggregateSeverity)
benoit@0: 
benoit@0: NAMESPACES = {
benoit@0:     'cvrf': "http://www.icasi.org/CVRF/schema/cvrf/1.1",
benoit@0:     'prod': "http://www.icasi.org/CVRF/schema/prod/1.1",
benoit@0:     'vuln': "http://www.icasi.org/CVRF/schema/vuln/1.1",
benoit@0:     'xml':  "http://www.w3.org/XML/1998/namespace",
benoit@0: }
benoit@0: 
benoit@0: 
benoit@0: def UN(ns, name):
benoit@0:     """ UN for Universal Name """
benoit@0:     return "{%s}%s" % (NAMESPACES[ns], name)
benoit@0: 
benoit@0: 
benoit@0: def parseVersion(string):
benoit@0:     return tuple(int(i) for i in string.split('.'))
benoit@0: 
benoit@0: 
benoit@0: def parseDate(string):
benoit@0:     m = re.match('(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:([+-])(\d{2}):(\d{2})|(Z))?', string)
benoit@0:     if (m.group(7) is None) or (m.group(7) == 'Z'):
benoit@0:         tzhours = 0
benoit@0:         tzmin = 0
benoit@0:     else:
benoit@0:         tzhours = int(m.group(8))
benoit@0:         if m.group(7) == '-':
benoit@0:             tzhours = - tzhours
benoit@0:         tzmin = int(m.group(9))
benoit@0:     return datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5)), int(m.group(6)), tzinfo=timezone(timedelta(hours=tzhours, minutes=tzmin)))
benoit@0: 
benoit@0: 
benoit@0: def parseNote(elem):
benoit@0:     return CVRFNote(
benoit@0:         elem.attrib['Type'],
benoit@0:         int(elem.attrib['Ordinal']),
benoit@0:         textwrap.dedent(elem.text).strip(),
benoit@0:         elem.attrib.get('Title'),
benoit@0:         elem.attrib.get('Audience')
benoit@0:     )
benoit@0: 
benoit@0: 
benoit@0: def parseReference(elem, ns='cvrf'):
benoit@0:     """ ns is the current namespace """
benoit@0:     return CVRFReference(
benoit@0:         elem.findtext(UN(ns, 'URL')).strip(),
benoit@0:         textwrap.dedent(elem.findtext(UN(ns, 'Description'))).strip(),
benoit@0:         elem.attrib.get('Type')
benoit@0:     )
benoit@0: 
benoit@0: 
benoit@0: def parseAcknowledgment(elem, ns='cvrf'):
benoit@9:     names = []
benoit@9:     for cvrfname in elem.findall(UN(ns, 'Name')):
benoit@9:         names.append(cvrfname.text.strip())
benoit@9:     orgs = []
benoit@9:     for cvrforg in elem.findall(UN(ns, 'Organization')):
benoit@9:         orgs.append(cvrforg.text.strip())
benoit@0:     return CVRFAcknowledgment(
benoit@9:         names, orgs,
benoit@0:         elem.findtext(UN(ns, 'Description')),
benoit@0:         elem.findtext(UN(ns, 'URL')),
benoit@0:     )
benoit@0: 
benoit@0: 
benoit@0: def parseFullProductName(elem, parent):
benoit@0:     return CVRFFullProductName(
benoit@0:         elem.attrib['ProductID'],
benoit@0:         elem.text.strip(),
benoit@0:         parent,
benoit@0:         cpe=elem.attrib.get('CPE')
benoit@0:     )
benoit@0: 
benoit@0: 
benoit@0: def parseProdBranch(elem, ptree, parentbranch=None):
benoit@0:     """ Recursively parses the branches and the terminal productnames """
benoit@0:     fpncvrf = elem.find(UN('prod', 'FullProductName'))
benoit@0:     if (parentbranch is not None) and (fpncvrf is not None):
benoit@0:         # Don't process the products at the root of the tree
benoit@0:         prod = parseFullProductName(fpncvrf, parentbranch)
benoit@0:         ptree.addProduct(prod)
benoit@0: 
benoit@0:     if parentbranch is None:
benoit@0:         parentbranch = ptree
benoit@0:     for brcvrf in elem.findall(UN('prod', 'Branch')):
benoit@0:         br = CVRFProductBranch(brcvrf.attrib['Type'], brcvrf.attrib['Name'], parentbranch)
benoit@0:         # And go into recursion ...
benoit@0:         br._childs = list(parseProdBranch(brcvrf, ptree, br))
benoit@0:         yield br
benoit@0: 
benoit@0: 
benoit@0: def parseVulnerability(elem):
benoit@0:     vuln = CVRFVulnerability(int(elem.attrib['Ordinal']))
benoit@0: 
benoit@0:     xmltitle = elem.findtext(UN('vuln', 'Title'))
benoit@0:     if xmltitle is not None:
benoit@0:         vuln.setTitle(xmltitle.strip())
benoit@0: 
benoit@0:     xmlID = elem.find(UN('vuln', 'ID'))
benoit@0:     if xmlID is not None:
benoit@0:         vuln.setID(CVRFVulnerabilityID(xmlID.attrib['SystemName'], xmlID.text.strip()))
benoit@0: 
benoit@0:     for xmlnote in elem.findall('/'.join([UN('vuln', 'Notes'), UN('vuln', 'Note')])):
benoit@0:         vuln.addNote(parseNote(xmlnote))
benoit@0: 
benoit@0:     xmldiscoverydate = elem.findtext(UN('vuln', 'DiscoveryDate'))
benoit@0:     if xmldiscoverydate is not None:
benoit@0:         vuln.setDiscoveryDate(parseDate(xmldiscoverydate))
benoit@0:     xmlreleasedate = elem.findtext(UN('vuln', 'ReleaseDate'))
benoit@0:     if xmlreleasedate is not None:
benoit@0:         vuln.setReleaseDate(parseDate(xmlreleasedate))
benoit@0: 
benoit@0:     for xmlinv in elem.findall('/'.join([UN('vuln', 'Involvements'), UN('vuln', 'Involvement')])):
benoit@0:         involvement = CVRFInvolvement(
benoit@0:             xmlinv.attrib['Party'],
benoit@0:             xmlinv.attrib['Status']
benoit@0:         )
benoit@0:         xmldescr = xmlinv.findtext(UN('vuln', 'Description'))
benoit@0:         if xmldescr is not None:
benoit@0:             involvement.setDescription(textwrap.dedent(xmldescr).strip())
benoit@0:         vuln.addInvolvement(involvement)
benoit@0: 
benoit@0:     xmlcve = elem.findtext(UN('vuln', 'CVE'))
benoit@0:     if xmlcve is not None:
benoit@0:         vuln.setCVE(xmlcve.strip())
benoit@0: 
benoit@0:     for xmlcwe in elem.findall(UN('vuln', 'CWE')):
benoit@0:         vuln.addCWE(CVRFCWE(
benoit@0:             xmlcwe.attrib['ID'],
benoit@0:             xmlcwe.text.strip()
benoit@0:         ))
benoit@0: 
benoit@0:     for xmlstatus in elem.findall('/'.join([UN('vuln', 'ProductStatuses'), UN('vuln', 'Status')])):
benoit@0:         status = CVRFProductStatus(xmlstatus.attrib['Type'])
benoit@0:         for xmlproductid in xmlstatus.findall(UN('vuln', 'ProductID')):
benoit@0:             status.addProductID(xmlproductid.text.strip())
benoit@0: 
benoit@0:         vuln.addProductStatus(status)
benoit@0: 
benoit@0:     for xmlthreat in elem.findall('/'.join([UN('vuln', 'Threats'), UN('vuln', 'Threat')])):
benoit@0:         threat = CVRFThreat(
benoit@0:             xmlthreat.attrib['Type'],
benoit@0:             textwrap.dedent(xmlthreat.findtext(UN('vuln', 'Description'))).strip()
benoit@0:         )
benoit@0:         xmldate = xmlthreat.findtext(UN('vuln', 'Date'))
benoit@0:         if xmldate is not None:
benoit@0:             threat.setDate(parseDate(xmldate))
benoit@0:         for xmlpid in xmlthreat.findall(UN('vuln', 'ProductID')):
benoit@0:             threat.addProductID(xmlpid.text.strip())
benoit@0:         for xmlgid in xmlthreat.findall(UN('vuln', 'GroupID')):
benoit@0:             threat.addGroupID(xmlgid.text.strip())
benoit@0: 
benoit@0:         vuln.addThreat(threat)
benoit@0: 
benoit@0:     for xmlcvss in elem.findall('/'.join([UN('vuln', 'CVSSScoreSets'), UN('vuln', 'ScoreSet')])):
benoit@0:         cvss_set = CVRFCVSSSet(float(xmlcvss.findtext(UN('vuln', 'BaseScore')).strip()))
benoit@0:         xmltempscore = xmlcvss.findtext(UN('vuln', 'TemporalScore'))
benoit@0:         if xmltempscore is not None:
benoit@0:             cvss_set.setTemporalScore(float(xmltempscore.strip()))
benoit@0:         xmlenvscore = xmlcvss.findtext(UN('vuln', 'EnvironmentalScore'))
benoit@0:         if xmlenvscore is not None:
benoit@0:             cvss_set.setEnvironmentalScore(float(xmlenvscore.strip()))
benoit@0:         xmlvector = xmlcvss.findtext(UN('vuln', 'Vector'))
benoit@0:         if xmlvector is not None:
benoit@0:             cvss_set.setVector(xmlvector.strip())
benoit@0:         for xmlprodid in xmlcvss.findall(UN('vuln', 'ProductID')):
benoit@0:             cvss_set.addProductID(xmlprodid.text.strip())
benoit@0: 
benoit@0:         vuln.addCVSSSet(cvss_set)
benoit@0: 
benoit@0:     for xmlremediation in elem.findall('/'.join([UN('vuln', 'Remediations'), UN('vuln', 'Remediation')])):
benoit@0:         remediation = CVRFRemediation(
benoit@0:             xmlremediation.attrib['Type'],
benoit@0:             textwrap.dedent(xmlremediation.findtext(UN('vuln', 'Description'))).strip()
benoit@0:         )
benoit@0:         xmldate = xmlremediation.findtext(UN('vuln', 'Date'))
benoit@0:         if xmldate is not None:
benoit@0:             remediation.setDate(parseDate(xmldate))
benoit@0:         xmlentitlement = xmlremediation.findtext(UN('vuln', 'Entitlement'))
benoit@0:         if xmlentitlement is not None:
benoit@0:             remediation.setEntitlement(textwrap.dedent(xmlentitlement).strip())
benoit@0:         xmlurl = xmlremediation.findtext(UN('vuln', 'URL'))
benoit@0:         if xmlurl is not None:
benoit@0:             remediation.setURL(xmlurl.strip())
benoit@0:         for xmlpid in xmlremediation.findall(UN('vuln', 'ProductID')):
benoit@0:             remediation.addProductID(xmlpid.text.strip())
benoit@0:         for xmlgid in xmlremediation.findall(UN('vuln', 'GroupID')):
benoit@0:             remediation.addGroupID(xmlgid.text.strip())
benoit@0: 
benoit@0:         vuln.addRemediation(remediation)
benoit@0: 
benoit@0:     for xmlref in elem.findall('/'.join([UN('vuln', 'References'), UN('vuln', 'Reference')])):
benoit@0:         vuln.addReference(parseReference(xmlref, 'vuln'))
benoit@0: 
benoit@0:     for xmlack in elem.findall('/'.join([UN('vuln', 'Acknowledgments'), UN('vuln', 'Acknowledgment')])):
benoit@0:         vuln.addAcknowledgment(parseAcknowledgment(xmlack, 'vuln'))
benoit@0: 
benoit@0:     return vuln
benoit@0: 
benoit@0: 
benoit@0: def parse(xml):
benoit@0:     if hasattr(xml, 'read'):
benoit@0:         xml = xml.read()
benoit@0:     cvrfdoc = ET.fromstring(xml)
benoit@0:     if cvrfdoc.tag != UN('cvrf', 'cvrfdoc'):
benoit@0:         raise ValueError('Not a CVRF document !')
benoit@0:     doc = CVRF(
benoit@0:         cvrfdoc.findtext(UN('cvrf', 'DocumentTitle')).strip(),
benoit@0:         cvrfdoc.findtext(UN('cvrf', 'DocumentType')).strip()
benoit@0:     )
benoit@5: 
benoit@0:     cvrfpub = cvrfdoc.find(UN('cvrf', 'DocumentPublisher'))
benoit@5:     if cvrfpub is not None:
benoit@5:         pub = CVRFPublisher(cvrfpub.attrib['Type'], cvrfpub.attrib.get('VendorID'))
benoit@5:         doc.setPublisher(pub)
benoit@5:         contact = cvrfpub.find(UN('cvrf', 'ContactDetails'))
benoit@5:         if contact is not None:
benoit@5:             pub.setContact(contact.text.strip())
benoit@5:         authority = cvrfpub.find(UN('cvrf', 'IssuingAuthority'))
benoit@5:         if authority is not None:
benoit@5:             pub.setAuthority(authority.text.strip())
benoit@5: 
benoit@0:     cvrftracking = cvrfdoc.find(UN('cvrf', 'DocumentTracking'))
benoit@5:     if cvrftracking is not None:
benoit@5:         identification = CVRFTrackingID(
benoit@5:             cvrftracking.findtext('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'ID')])).strip()
benoit@0:         )
benoit@5:         for cvrfalias in cvrftracking.findall('/'.join([UN('cvrf', 'Identification'), UN('cvrf', 'Alias')])):
benoit@5:             identification.addAlias(cvrfalias.text.strip())
benoit@5:         tracking = CVRFTracking(
benoit@5:             identification,
benoit@5:             cvrftracking.findtext(UN('cvrf', 'Status')).strip(),
benoit@5:             parseVersion(cvrftracking.findtext(UN('cvrf', 'Version')).strip()),
benoit@5:             parseDate(cvrftracking.findtext(UN('cvrf', 'InitialReleaseDate')).strip()),
benoit@5:             parseDate(cvrftracking.findtext(UN('cvrf', 'CurrentReleaseDate')).strip())
benoit@5:         )
benoit@5:         doc.setTracking(tracking)
benoit@5:         for cvrfrev in cvrftracking.findall('/'.join([UN('cvrf', 'RevisionHistory'), UN('cvrf', 'Revision')])):
benoit@5:             rev = CVRFRevision(
benoit@5:                 parseVersion(cvrfrev.findtext(UN('cvrf', 'Number')).strip()),
benoit@5:                 parseDate(cvrfrev.findtext(UN('cvrf', 'Date')).strip()),
benoit@5:                 cvrfrev.findtext(UN('cvrf', 'Description')).strip(),
benoit@5:             )
benoit@5:             tracking.addRevision(rev)
benoit@0: 
benoit@5:         xmlgenerator = cvrftracking.find(UN('cvrf', 'Generator'))
benoit@5:         if xmlgenerator is not None:
benoit@5:             generator = CVRFGenerator()
benoit@5:             xmlengine = xmlgenerator.findtext(UN('cvrf', 'Engine'))
benoit@5:             if xmlengine is not None:
benoit@5:                 generator.setEngine(xmlengine.strip())
benoit@5:             xmldate = xmlgenerator.findtext(UN('cvrf', 'Date'))
benoit@5:             if xmldate is not None:
benoit@5:                 generator.setDate(parseDate(xmldate.strip()))
benoit@5:             tracking.setGenerator(generator)
benoit@0: 
benoit@0:     for cvrfnote in cvrfdoc.findall('/'.join([UN('cvrf', 'DocumentNotes'), UN('cvrf', 'Note')])):
benoit@0:         doc.addNote(parseNote(cvrfnote))
benoit@0: 
benoit@0:     distr = cvrfdoc.findtext(UN('cvrf', 'DocumentDistribution'))
benoit@0:     if distr is not None:
benoit@0:         doc.setDistribution(textwrap.dedent(distr).strip())
benoit@0: 
benoit@0:     # This is in a quite free format, not sure how to do something with it ...
benoit@0:     xmlaggsev = cvrfdoc.find(UN('cvrf', 'AggregateSeverity'))
benoit@1:     if xmlaggsev is not None:
benoit@1:         aggsev = CVRFAggregateSeverity(xmlaggsev.text.strip())
benoit@1:         if 'Namespace' in xmlaggsev.attrib:
benoit@1:             aggsev.setNamespace(xmlaggsev.attrib['Namespace'])
benoit@1:         doc.setAggregateSeverity(aggsev)
benoit@0: 
benoit@0:     for xmlref in cvrfdoc.findall('/'.join([UN('cvrf', 'DocumentReferences'), UN('cvrf', 'Reference')])):
benoit@0:         doc.addReference(parseReference(xmlref))
benoit@0: 
benoit@0:     for cvrfack in cvrfdoc.findall('/'.join([UN('cvrf', 'Acknowledgments'), UN('cvrf', 'Acknowledgment')])):
benoit@0:         doc.addAcknowledgment(parseAcknowledgment(cvrfack))
benoit@0: 
benoit@0:     # --- The ProductTree
benoit@0: 
benoit@0:     cvrfptree = cvrfdoc.find(UN('prod', 'ProductTree'))
benoit@0:     if cvrfptree is not None:
benoit@0:         producttree = doc.createProductTree()
benoit@15:         # We need to exhaust our generator ...
benoit@15:         for _ in parseProdBranch(cvrfptree, producttree): pass
benoit@0: 
benoit@0:         for product in cvrfptree.findall(UN('prod', 'FullProductName')):
benoit@0:             producttree.addProduct(parseFullProductName(product, producttree))
benoit@0: 
benoit@0:         for cvrfrel in cvrfptree.findall(UN('prod', 'Relationship')):
benoit@0:             rel = CVRFRelationship(
benoit@0:                 cvrfrel.attrib['ProductReference'],
benoit@0:                 cvrfrel.attrib['RelationType'],
benoit@0:                 cvrfrel.attrib['RelatesToProductReference']
benoit@0:             )
benoit@0:             producttree.addRelationship(rel)
benoit@0:             producttree.addProduct(parseFullProductName(cvrfrel.find(UN('prod', 'FullProductName')), rel))
benoit@0: 
benoit@0:         for xmlgroup in cvrfptree.findall('/'.join([UN('prod', 'ProductGroups'), UN('prod', 'Group')])):
benoit@0:             group = CVRFGroup(xmlgroup.attrib['GroupID'])
benoit@0:             xmldescr = xmlgroup.findtext(UN('prod', 'Description'))
benoit@0:             if xmldescr is not None:
benoit@0:                 group.setDescription(textwrap.dedent(xmldescr).strip())
benoit@0:             for xmlpid in xmlgroup.findall(UN('prod', 'ProductID')):
benoit@0:                 group.addProductID(xmlpid.text.strip())
benoit@0:             producttree.addGroup(group)
benoit@0: 
benoit@0:     # --- The Vulnerabilities
benoit@0: 
benoit@0:     for cvrfvuln in cvrfdoc.findall(UN('vuln', 'Vulnerability')):
benoit@0:         doc.addVulnerability(parseVulnerability(cvrfvuln))
benoit@0: 
benoit@0:     return doc
benoit@0: 
benoit@0: 
benoit@0: if __name__ == "__main__":
benoit@0:     import sys
benoit@0:     with open(sys.argv[1], 'rt') as f:
benoit@0:         cvrf = parse(f)
benoit@0:     cvrf.validate()
benoit@0:     print(cvrf)
benoit@0:     print(cvrf.getHighestCVSS()._vector)
benoit@0:     print(cvrf.getProductList())
benoit@0:     print(cvrf._producttree._branches)
benoit@0: #    print(cvrf._producttree._branches[0]._childs)