Mercurial > dive4elements > river
changeset 5006:769593a84606 dami
Importer: Behold, Logging!
| author | Andre Heinecke <aheinecke@intevation.de> |
|---|---|
| date | Fri, 15 Feb 2013 16:22:13 +0100 |
| parents | eb2d6609387c |
| children | 4e5d209253dd |
| files | flys-backend/contrib/shpimporter/hws.py flys-backend/contrib/shpimporter/importer.py flys-backend/contrib/shpimporter/shpimporter.py flys-backend/contrib/shpimporter/utils.py |
| diffstat | 4 files changed, 64 insertions(+), 56 deletions(-) [+] |
line wrap: on
line diff
--- a/flys-backend/contrib/shpimporter/hws.py Fri Feb 15 16:19:52 2013 +0100 +++ b/flys-backend/contrib/shpimporter/hws.py Fri Feb 15 16:22:13 2013 +0100 @@ -9,6 +9,9 @@ from importer import Importer import utils +import logging +logger = logging.getLogger("Hochwasserschutzanlagen") + PATH="Hydrologie/HW-Schutzanlagen" NAME="HWS" @@ -93,7 +96,7 @@ self.handled(artname) kind_id = HWS_KIND.get(feat.GetField(artname).lower()) if not kind_id: - print ("Unknown Art: %s" % \ + logger.warn("Unknown Art: %s" % \ feat.GetField(artname)) else: newFeat.SetField("kind_id", kind_id) @@ -104,7 +107,7 @@ fed_id = FED_STATES.get(feat.GetField(fname).lower()) if not fed_id: - print ("Unknown Bundesland: %s" % \ + logger.warn("Unknown Bundesland: %s" % \ feat.GetField("Bundesland")) else: newFeat.SetField("fed_state_id", fed_id) @@ -123,7 +126,7 @@ if self.IsFieldSet(feat, fname): self.handled(fname) if feat.GetField(fname) != self.river_id: - print ("River_id mismatch between shapefile and" + logger.warn("River_id mismatch between shapefile and" " importer parameter.") newFeat.SetField("river_id", feat.GetField(fname)) else:
--- a/flys-backend/contrib/shpimporter/importer.py Fri Feb 15 16:19:52 2013 +0100 +++ b/flys-backend/contrib/shpimporter/importer.py Fri Feb 15 16:22:13 2013 +0100 @@ -4,6 +4,9 @@ import ogr, osr import utils import re +import logging + +logger = logging.getLogger("importer") class Importer: @@ -78,20 +81,20 @@ shp = ogr.Open(shape[1]) if shp is None: - shpimporter.ERROR("Shapefile '%s' could not be opened!" % path) + logger.error("Shapefile '%s' could not be opened!" % path) return if not self.isShapeRelevant(name, path): - shpimporter.INFO("Skip shapefile: '%s' of Type: %s" % (path, + logger.info("Skip shapefile: '%s' of Type: %s" % (path, utils.getWkbString(shp.GetLayerByName(name).GetGeomType()))) return - shpimporter.INFO("Processing shapefile '%s'" % path) + logger.info("Processing shapefile '%s'" % path) srcLayer = shp.GetLayerByName(name) if srcLayer is None: - shpimporter.ERROR("Layer '%s' was not found!" % name) + logger.error("Layer '%s' was not found!" % name) return return self.shape2Database(srcLayer, name, path) @@ -101,7 +104,7 @@ src_srs = geometry.GetSpatialReference() if src_srs is None: - shpimporter.ERROR("No source SRS given! No transformation possible!") + logger.error("No source SRS given! No transformation possible!") return feat transformer = osr.CoordinateTransformation(src_srs, self.dest_srs) @@ -158,15 +161,15 @@ destLayer = self.dbconn.GetLayerByName(self.getTablename()) if srcLayer is None: - shpimporter.ERROR("Shapefile is None!") + logger.error("Shapefile is None!") return -1 if destLayer is None: - shpimporter.ERROR("No destination layer given!") + logger.error("No destination layer given!") return -1 count = srcLayer.GetFeatureCount() - shpimporter.DEBUG("Try to add %i features to database." % count) + logger.debug("Try to add %i features to database." % count) srcLayer.ResetReading() self.srcLayer = srcLayer @@ -181,7 +184,7 @@ geom = feat.GetGeometryRef() if geom is None: - shpimporter.DEBUG("Unkown Geometry reference for feature") + logger.debug("Unkown Geometry reference for feature") continue geomType = geom.GetGeometryType() @@ -198,11 +201,11 @@ if newFeat: res = destLayer.CreateFeature(newFeat) if res is None or res > 0: - shpimporter.ERROR("Unable to insert feature. Error: %r" % res) + logger.error("Unable to insert feature. Error: %r" % res) else: success = success + 1 else: - shpimporter.ERROR("Could not transform feature: %s " % feat.GetFID()) + logger.error("Could not transform feature: %s " % feat.GetFID()) creationFailed += 1 else: creationFailed = creationFailed + 1 @@ -210,10 +213,10 @@ unsupported[utils.getWkbString(geomType)] = \ unsupported.get(utils.getWkbString(geomType), 0) + 1 - shpimporter.INFO("Inserted %i features" % success) - shpimporter.INFO("Failed to create %i features" % creationFailed) + logger.info("Inserted %i features" % success) + logger.info("Failed to create %i features" % creationFailed) for key, value in unsupported.items(): - shpimporter.INFO("Found %i unsupported features of type: %s" % (value, key)) + logger.info("Found %i unsupported features of type: %s" % (value, key)) if self.tracking_import: unhandled = [] @@ -223,7 +226,7 @@ unhandled.append(act_field) if len(unhandled): - shpimporter.INFO("Did not import values from fields: %s " % \ + logger.info("Did not import values from fields: %s " % \ " ".join(unhandled)) try: @@ -231,6 +234,6 @@ return geomType destLayer.CommitTransaction() except e: - shpimporter.ERROR("Exception while committing transaction.") + logger.error("Exception while committing transaction.") return geomType
--- a/flys-backend/contrib/shpimporter/shpimporter.py Fri Feb 15 16:19:52 2013 +0100 +++ b/flys-backend/contrib/shpimporter/shpimporter.py Fri Feb 15 16:22:13 2013 +0100 @@ -6,6 +6,7 @@ import utils, optparse import sys import os +import logging from uesg import UESG from axis import Axis @@ -20,25 +21,16 @@ from catchments import Catchment from dgm import insertRiverDgm - -VERBOSE_DEBUG=2 -VERBOSE_INFO=1 - +logger = logging.getLogger("shpimporter") -def DEBUG(msg): - config = getConfig() - if config.verbose >= VERBOSE_DEBUG: - print "DEBUG: %s" % msg - -def INFO(msg): - config = getConfig() - if config.verbose >= VERBOSE_INFO: - print "INFO: %s" % msg - -def ERROR(msg): - config = getConfig() - print "ERROR: %s" % msg - +def initialize_logging(level): + """Initializes the logging system""" + root = logging.getLogger() + root.setLevel(level) + hdlr = logging.StreamHandler() + fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s") + hdlr.setFormatter(fmt) + root.addHandler(hdlr) def getImporters(river_id, dbconn, dry_run): return [ @@ -84,18 +76,25 @@ parser.add_option("--skip_dgm", type="int") (config, args) = parser.parse_args() + if config.verbose > 1: + initialize_logging(logging.DEBUG) + elif config.verbose == 1: + initialize_logging(logging.INFO) + else: + initialize_logging(logging.WARN) + if config.directory == None: - ERROR("No river directory specified!") + logger.error("No river directory specified!") raise Exception("Invalid config") if not config.ogr_connection: if not config.host: - ERROR("No database host specified!") + logger.error("No database host specified!") raise Exception("Invalid config") if not config.user: - ERROR("No databaser user specified!") + logger.error("No databaser user specified!") raise Exception("Invalid config") if not config.password: - ERROR("No password specified!") + logger.error("No password specified!") raise Exception("Invalid config") return config @@ -139,11 +138,11 @@ return -1 if config == None: - ERROR("Unable to read config from command line!") + logger.error("Unable to read config from command line!") return if config.dry_run > 0: - INFO("You enable 'dry_run'. No database transaction will take place!") + logger.info("You enable 'dry_run'. No database transaction will take place!") if config.ogr_connection: connstr = config.ogr_connection @@ -155,7 +154,7 @@ import cx_Oracle as dbapi raw_connstr=connstr.replace("OCI:", "") except ImportErrror: - ERROR("Module cx_Oracle not found in: %s\n" + logger.error("Module cx_Oracle not found in: %s\n" "Neccessary to connect to a Oracle Database.\n" "Please refer to the installation " "documentation." % sys.path) @@ -165,7 +164,7 @@ import psycopg2 as dbapi raw_connstr=connstr.replace("PG:", "") except ImportError: - ERROR("Module psycopg2 not found in: %s\n" + logger.error("Module psycopg2 not found in: %s\n" "Neccessary to connect to a Posgresql Database.\n" "Please refer to the installation " "documentation." % sys.path) @@ -174,7 +173,7 @@ dbconn = ogr.Open(connstr) if dbconn == None: - ERROR("Could not connect to database %s" % connstr) + logger.error("Could not connect to database %s" % connstr) return -1 types = {} @@ -197,21 +196,21 @@ river_id = utils.getRiverId(dbconn_raw, river_name) if not river_id: - INFO("Could not find river in database. Skipping: %s" + logger.info("Could not find river in database. Skipping: %s" % river_name) continue else: - INFO("Importing River: %s" % river_name) + logger.info("Importing River: %s" % river_name) for importer in getImporters(river_id, dbconn, config.dry_run): if skip_importer(config, importer): - INFO("Skip import of '%s'" % importer.getName()) + logger.info("Skip import of '%s'" % importer.getName()) continue - INFO("Start import of '%s'" % importer.getName()) + logger.info("Start import of '%s'" % importer.getName()) shapes = utils.findShapefiles(importer.getPath(config.directory)) - DEBUG("Found %i Shapefiles" % len(shapes)) + logger.debug("Found %i Shapefiles" % len(shapes)) for shpTuple in shapes: geomType = importer.walkOverShapes(shpTuple) @@ -223,17 +222,17 @@ types[geomType] = 1 for key in types: - DEBUG("%i x geometry type %s" % (types[key], key)) + logger.debug("%i x geometry type %s" % (types[key], key)) if not config.skip_dgm: dgmfilename = os.path.join( config.directory, "..", "DGMs.csv") if not os.access(dgmfilename, os.R_OK) or not \ os.path.isfile(dgmfilename): - INFO("Could not find or access DGM file: %s \n" + logger.info("Could not find or access DGM file: %s \n" "Skipping DGM import." % dgmfilename) else: - INFO("Inserting DGM meta information in 'dem' table.") + logger.info("Inserting DGM meta information in 'dem' table.") insertRiverDgm(dbconn_raw, dgmfilename, river_name, config.dry_run) if __name__ == '__main__':
--- a/flys-backend/contrib/shpimporter/utils.py Fri Feb 15 16:19:52 2013 +0100 +++ b/flys-backend/contrib/shpimporter/utils.py Fri Feb 15 16:22:13 2013 +0100 @@ -1,11 +1,14 @@ import os import sys -from shpimporter import DEBUG, INFO, ERROR +import logging + try: from osgeo import ogr except ImportErrror: import ogr +logger = logging.getLogger("utils") + SHP='.shp' SQL_SELECT_RIVER_ID="SELECT id FROM rivers WHERE name = %s" @@ -16,7 +19,7 @@ if len(files) == 0: continue - DEBUG("Processing directory '%s' with %i files " % (root, len(files))) + logger.debug("Processing directory '%s' with %i files " % (root, len(files))) for f in files: idx = f.find(SHP)