changeset 5006:769593a84606 dami

Importer: Behold, Logging!
author Andre Heinecke <aheinecke@intevation.de>
date Fri, 15 Feb 2013 16:22:13 +0100
parents eb2d6609387c
children 4e5d209253dd
files flys-backend/contrib/shpimporter/hws.py flys-backend/contrib/shpimporter/importer.py flys-backend/contrib/shpimporter/shpimporter.py flys-backend/contrib/shpimporter/utils.py
diffstat 4 files changed, 64 insertions(+), 56 deletions(-) [+]
line wrap: on
line diff
--- a/flys-backend/contrib/shpimporter/hws.py	Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/hws.py	Fri Feb 15 16:22:13 2013 +0100
@@ -9,6 +9,9 @@
 from importer import Importer
 import utils
 
+import logging
+logger = logging.getLogger("Hochwasserschutzanlagen")
+
 PATH="Hydrologie/HW-Schutzanlagen"
 NAME="HWS"
 
@@ -93,7 +96,7 @@
             self.handled(artname)
             kind_id = HWS_KIND.get(feat.GetField(artname).lower())
             if not kind_id:
-                print ("Unknown Art: %s" % \
+                logger.warn("Unknown Art: %s" % \
                         feat.GetField(artname))
             else:
                 newFeat.SetField("kind_id", kind_id)
@@ -104,7 +107,7 @@
             fed_id = FED_STATES.get(feat.GetField(fname).lower())
 
             if not fed_id:
-                print ("Unknown Bundesland: %s" % \
+                logger.warn("Unknown Bundesland: %s" % \
                         feat.GetField("Bundesland"))
             else:
                 newFeat.SetField("fed_state_id", fed_id)
@@ -123,7 +126,7 @@
         if self.IsFieldSet(feat, fname):
             self.handled(fname)
             if feat.GetField(fname) != self.river_id:
-                print ("River_id mismatch between shapefile and"
+                logger.warn("River_id mismatch between shapefile and"
                      " importer parameter.")
             newFeat.SetField("river_id", feat.GetField(fname))
         else:
--- a/flys-backend/contrib/shpimporter/importer.py	Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/importer.py	Fri Feb 15 16:22:13 2013 +0100
@@ -4,6 +4,9 @@
     import ogr, osr
 import utils
 import re
+import logging
+
+logger = logging.getLogger("importer")
 
 class Importer:
 
@@ -78,20 +81,20 @@
 
         shp = ogr.Open(shape[1])
         if shp is None:
-            shpimporter.ERROR("Shapefile '%s' could not be opened!" % path)
+            logger.error("Shapefile '%s' could not be opened!" % path)
             return
 
         if not self.isShapeRelevant(name, path):
-            shpimporter.INFO("Skip shapefile: '%s' of Type: %s" % (path,
+            logger.info("Skip shapefile: '%s' of Type: %s" % (path,
                 utils.getWkbString(shp.GetLayerByName(name).GetGeomType())))
             return
 
 
-        shpimporter.INFO("Processing shapefile '%s'" % path)
+        logger.info("Processing shapefile '%s'" % path)
         srcLayer = shp.GetLayerByName(name)
 
         if srcLayer is None:
-            shpimporter.ERROR("Layer '%s' was not found!" % name)
+            logger.error("Layer '%s' was not found!" % name)
             return
 
         return self.shape2Database(srcLayer, name, path)
@@ -101,7 +104,7 @@
         src_srs  = geometry.GetSpatialReference()
 
         if src_srs is None:
-            shpimporter.ERROR("No source SRS given! No transformation possible!")
+            logger.error("No source SRS given! No transformation possible!")
             return feat
 
         transformer = osr.CoordinateTransformation(src_srs, self.dest_srs)
@@ -158,15 +161,15 @@
         destLayer = self.dbconn.GetLayerByName(self.getTablename())
 
         if srcLayer is None:
-            shpimporter.ERROR("Shapefile is None!")
+            logger.error("Shapefile is None!")
             return -1
 
         if destLayer is None:
-            shpimporter.ERROR("No destination layer given!")
+            logger.error("No destination layer given!")
             return -1
 
         count = srcLayer.GetFeatureCount()
-        shpimporter.DEBUG("Try to add %i features to database." % count)
+        logger.debug("Try to add %i features to database." % count)
 
         srcLayer.ResetReading()
         self.srcLayer = srcLayer
@@ -181,7 +184,7 @@
             geom     = feat.GetGeometryRef()
 
             if geom is None:
-                shpimporter.DEBUG("Unkown Geometry reference for feature")
+                logger.debug("Unkown Geometry reference for feature")
                 continue
 
             geomType = geom.GetGeometryType()
@@ -198,11 +201,11 @@
                     if newFeat:
                         res = destLayer.CreateFeature(newFeat)
                         if res is None or res > 0:
-                            shpimporter.ERROR("Unable to insert feature. Error: %r" % res)
+                            logger.error("Unable to insert feature. Error: %r" % res)
                         else:
                             success = success + 1
                     else:
-                        shpimporter.ERROR("Could not transform feature: %s " % feat.GetFID())
+                        logger.error("Could not transform feature: %s " % feat.GetFID())
                         creationFailed += 1
                 else:
                     creationFailed = creationFailed + 1
@@ -210,10 +213,10 @@
                 unsupported[utils.getWkbString(geomType)] = \
                         unsupported.get(utils.getWkbString(geomType), 0) + 1
 
-        shpimporter.INFO("Inserted %i features" % success)
-        shpimporter.INFO("Failed to create %i features" % creationFailed)
+        logger.info("Inserted %i features" % success)
+        logger.info("Failed to create %i features" % creationFailed)
         for key, value in unsupported.items():
-            shpimporter.INFO("Found %i unsupported features of type: %s" % (value, key))
+            logger.info("Found %i unsupported features of type: %s" % (value, key))
 
         if self.tracking_import:
             unhandled = []
@@ -223,7 +226,7 @@
                     unhandled.append(act_field)
 
             if len(unhandled):
-                shpimporter.INFO("Did not import values from fields: %s " % \
+                logger.info("Did not import values from fields: %s " % \
                         " ".join(unhandled))
 
         try:
@@ -231,6 +234,6 @@
                 return geomType
             destLayer.CommitTransaction()
         except e:
-            shpimporter.ERROR("Exception while committing transaction.")
+            logger.error("Exception while committing transaction.")
 
         return geomType
--- a/flys-backend/contrib/shpimporter/shpimporter.py	Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/shpimporter.py	Fri Feb 15 16:22:13 2013 +0100
@@ -6,6 +6,7 @@
 import utils, optparse
 import sys
 import os
+import logging
 
 from uesg  import UESG
 from axis  import Axis
@@ -20,25 +21,16 @@
 from catchments import Catchment
 from dgm import insertRiverDgm
 
-
-VERBOSE_DEBUG=2
-VERBOSE_INFO=1
-
+logger = logging.getLogger("shpimporter")
 
-def DEBUG(msg):
-    config = getConfig()
-    if config.verbose >= VERBOSE_DEBUG:
-        print "DEBUG: %s" % msg
-
-def INFO(msg):
-    config = getConfig()
-    if config.verbose >= VERBOSE_INFO:
-        print "INFO: %s" % msg
-
-def ERROR(msg):
-    config = getConfig()
-    print "ERROR: %s" % msg
-
+def initialize_logging(level):
+    """Initializes the logging system"""
+    root = logging.getLogger()
+    root.setLevel(level)
+    hdlr = logging.StreamHandler()
+    fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
+    hdlr.setFormatter(fmt)
+    root.addHandler(hdlr)
 
 def getImporters(river_id, dbconn, dry_run):
     return [
@@ -84,18 +76,25 @@
     parser.add_option("--skip_dgm", type="int")
     (config, args) = parser.parse_args()
 
+    if config.verbose > 1:
+        initialize_logging(logging.DEBUG)
+    elif config.verbose == 1:
+        initialize_logging(logging.INFO)
+    else:
+        initialize_logging(logging.WARN)
+
     if config.directory == None:
-        ERROR("No river directory specified!")
+        logger.error("No river directory specified!")
         raise Exception("Invalid config")
     if not config.ogr_connection:
         if not config.host:
-            ERROR("No database host specified!")
+            logger.error("No database host specified!")
             raise Exception("Invalid config")
         if not config.user:
-            ERROR("No databaser user specified!")
+            logger.error("No databaser user specified!")
             raise Exception("Invalid config")
         if not config.password:
-            ERROR("No password specified!")
+            logger.error("No password specified!")
             raise Exception("Invalid config")
 
     return config
@@ -139,11 +138,11 @@
         return -1
 
     if config == None:
-        ERROR("Unable to read config from command line!")
+        logger.error("Unable to read config from command line!")
         return
 
     if config.dry_run > 0:
-        INFO("You enable 'dry_run'. No database transaction will take place!")
+        logger.info("You enable 'dry_run'. No database transaction will take place!")
 
     if config.ogr_connection:
         connstr = config.ogr_connection
@@ -155,7 +154,7 @@
             import cx_Oracle as dbapi
             raw_connstr=connstr.replace("OCI:", "")
         except ImportErrror:
-            ERROR("Module cx_Oracle not found in: %s\n"
+            logger.error("Module cx_Oracle not found in: %s\n"
                   "Neccessary to connect to a Oracle Database.\n"
                   "Please refer to the installation "
                   "documentation." % sys.path)
@@ -165,7 +164,7 @@
             import psycopg2 as dbapi
             raw_connstr=connstr.replace("PG:", "")
         except ImportError:
-            ERROR("Module psycopg2 not found in: %s\n"
+            logger.error("Module psycopg2 not found in: %s\n"
                   "Neccessary to connect to a Posgresql Database.\n"
                   "Please refer to the installation "
                   "documentation." % sys.path)
@@ -174,7 +173,7 @@
     dbconn = ogr.Open(connstr)
 
     if dbconn == None:
-        ERROR("Could not connect to database %s" % connstr)
+        logger.error("Could not connect to database %s" % connstr)
         return -1
 
     types = {}
@@ -197,21 +196,21 @@
         river_id = utils.getRiverId(dbconn_raw, river_name)
 
         if not river_id:
-            INFO("Could not find river in database. Skipping: %s"
+            logger.info("Could not find river in database. Skipping: %s"
                   % river_name)
             continue
         else:
-            INFO("Importing River: %s" % river_name)
+            logger.info("Importing River: %s" % river_name)
 
         for importer in getImporters(river_id, dbconn, config.dry_run):
             if skip_importer(config, importer):
-                INFO("Skip import of '%s'" % importer.getName())
+                logger.info("Skip import of '%s'" % importer.getName())
                 continue
 
-            INFO("Start import of '%s'" % importer.getName())
+            logger.info("Start import of '%s'" % importer.getName())
 
             shapes = utils.findShapefiles(importer.getPath(config.directory))
-            DEBUG("Found %i Shapefiles" % len(shapes))
+            logger.debug("Found %i Shapefiles" % len(shapes))
 
             for shpTuple in shapes:
                 geomType = importer.walkOverShapes(shpTuple)
@@ -223,17 +222,17 @@
                     types[geomType] = 1
 
         for key in types:
-            DEBUG("%i x geometry type %s" % (types[key], key))
+            logger.debug("%i x geometry type %s" % (types[key], key))
 
         if not config.skip_dgm:
             dgmfilename = os.path.join(
                     config.directory, "..", "DGMs.csv")
             if not os.access(dgmfilename, os.R_OK) or not \
                     os.path.isfile(dgmfilename):
-                INFO("Could not find or access DGM file: %s \n"
+                logger.info("Could not find or access DGM file: %s \n"
                      "Skipping DGM import." % dgmfilename)
             else:
-                INFO("Inserting DGM meta information in 'dem' table.")
+                logger.info("Inserting DGM meta information in 'dem' table.")
                 insertRiverDgm(dbconn_raw, dgmfilename, river_name, config.dry_run)
 
 if __name__ == '__main__':
--- a/flys-backend/contrib/shpimporter/utils.py	Fri Feb 15 16:19:52 2013 +0100
+++ b/flys-backend/contrib/shpimporter/utils.py	Fri Feb 15 16:22:13 2013 +0100
@@ -1,11 +1,14 @@
 import os
 import sys
-from shpimporter import DEBUG, INFO, ERROR
+import logging
+
 try:
     from osgeo import ogr
 except ImportErrror:
     import ogr
 
+logger = logging.getLogger("utils")
+
 SHP='.shp'
 SQL_SELECT_RIVER_ID="SELECT id FROM rivers WHERE name = %s"
 
@@ -16,7 +19,7 @@
         if len(files) == 0:
             continue
 
-        DEBUG("Processing directory '%s' with %i files " % (root, len(files)))
+        logger.debug("Processing directory '%s' with %i files " % (root, len(files)))
 
         for f in files:
             idx = f.find(SHP)

http://dive4elements.wald.intevation.org