--- a/flys-backend/contrib/shpimporter/importer.py	Wed Mar 06 14:14:15 2013 +0100
+++ b/flys-backend/contrib/shpimporter/importer.py	Fri Mar 22 11:25:54 2013 +0100
@@ -1,21 +1,24 @@
 try:
-    from osgeo import ogr
-except ImportErrror:
-    import ogr
-import osr
-import shpimporter
+    from osgeo import ogr, osr
+except ImportError:
+    import ogr, osr
 import utils
+import re
+import logging
+
+logger = logging.getLogger("importer")
 
 class Importer:
 
-    def __init__(self, config, dbconn):
-        self.config = config
+    def __init__(self, river_id, dbconn, dry_run):
+        self.river_id = river_id
         self.dbconn = dbconn
-        self.river_id = config.river_id
+        self.dry_run = dry_run
         self.dest_srs = osr.SpatialReference()
-        self.dest_srs.ImportFromEPSG(config.target_srs)
+        self.dest_srs.ImportFromEPSG(31467)
         self.handled_fields = []
         self.tracking_import = False
+        self.srcLayer = None
 
     def getKind(self, path):
         raise NotImplementedError("Importer.getKind is abstract!")
@@ -27,13 +30,48 @@
         raise NotImplementedError("Importer.getTablename is abstract!")
 
     def getName(self):
-        raise NotImplementedError("Importer.getTablename is abstract!")
+        raise NotImplementedError("Importer.getName is abstract!")
+
+    def isGeometryValid(self, geomType):
+        raise NotImplementedError("Importer.isGeometryValid is abstract!")
+
+    def createNewFeature(self, featureDef, feat, **args):
+        raise NotImplementedError("Importer.createNewFeature is abstract!")
 
     def IsFieldSet(self, feat, name):
+        if not name:
+            return False
         if feat.GetFieldIndex(name) == -1:
             return False # Avoids an Error in IsFieldSet
         return feat.IsFieldSet(feat.GetFieldIndex(name))
 
+    def searchField(self, regex):
+        """
+        Searches for a field in the current src layer that matches
+        the expression regex.
+        Throws an exception if more than one field matches
+        @param feat: The feature to search for attributes
+        @param regex: The regex to look for
+
+        @returns: The field name as a string
+        """
+
+        if not hasattr(self.srcLayer, "fieldnames"):
+            self.srcLayer.fieldnames = []
+            for i in range(0, self.srcLayer.GetLayerDefn().GetFieldCount()):
+                self.srcLayer.fieldnames.append(
+                    self.srcLayer.GetLayerDefn().GetFieldDefn(i).GetNameRef())
+
+        result = None
+        for name in self.srcLayer.fieldnames:
+            match = re.match(regex, name, re.IGNORECASE)
+            if match:
+                if result:
+                    raise Exception("More than one field matches: %s" % regex)
+                else:
+                    result = match.group(0)
+        return result
+
     def IsDoubleFieldSet(self, feat, name):
         try:
             isset = feat.GetFieldAsDouble(name)
@@ -46,20 +84,23 @@
 
     def walkOverShapes(self, shape):
         (name, path) = shape
-        if not self.isShapeRelevant(name, path):
-            shpimporter.INFO("Skip shapefile '%s'" % path)
-            return
 
         shp = ogr.Open(shape[1])
         if shp is None:
-            shpimporter.ERROR("Shapefile '%s' could not be opened!" % path)
+            logger.error("Shapefile '%s' could not be opened!" % path)
             return
 
-        shpimporter.INFO("Processing shapefile '%s'" % path)
+        if not self.isShapeRelevant(name, path):
+            logger.info("Skip shapefile: '%s' of Type: %s" % (path,
+                utils.getWkbString(shp.GetLayerByName(name).GetGeomType())))
+            return
+
+
+        logger.info("Processing shapefile '%s'" % path)
         srcLayer = shp.GetLayerByName(name)
 
         if srcLayer is None:
-            shpimporter.ERROR("Layer '%s' was not found!" % name)
+            logger.error("Layer '%s' was not found!" % name)
             return
 
         return self.shape2Database(srcLayer, name, path)
@@ -69,11 +110,12 @@
         src_srs  = geometry.GetSpatialReference()
 
         if src_srs is None:
-            shpimporter.ERROR("No source SRS given! No transformation possible!")
+            logger.error("No source SRS given! No transformation possible!")
             return feat
 
         transformer = osr.CoordinateTransformation(src_srs, self.dest_srs)
-        geometry.Transform(transformer)
+        if geometry.Transform(transformer):
+            return None
 
         return feat
 
@@ -90,15 +132,19 @@
         """
         Checks the mapping dictonary for key value pairs to
         copy from the source to the destination feature.
+        The keys can be reguar expressions that are matched
+        agains the source fieldnames
 
         The Key is the attribute of the source feature to be copied
         into the target attribute named by the dict's value.
         """
         self.tracking_import = True
-        self.handled_fields.extend(mapping.keys())
         for key, value in mapping.items():
-            if src.GetFieldIndex(key) == -1:
+            realname = self.searchField(key)
+            if realname == None:
                 continue
+            if not realname in self.handled_fields:
+                self.handled_fields.append(realname)
             # 0 OFTInteger, Simple 32bit integer
             # 1 OFTIntegerList, List of 32bit integers
             # 2 OFTReal, Double Precision floating point
@@ -111,31 +157,32 @@
             # 9 OFTDate, Date
             # 10 OFTTime, Time
             # 11 OFTDateTime, Date and Time
-            if src.IsFieldSet(src.GetFieldIndex(key)):
-                if src.GetFieldType(key) == 2:
-                    target.SetField(value, src.GetFieldAsDouble(key))
+            if src.IsFieldSet(src.GetFieldIndex(realname)):
+                if src.GetFieldType(realname) == 2:
+                    target.SetField(value, src.GetFieldAsDouble(realname))
                 else:
-                    target.SetField(value, src.GetField(key))
+                    target.SetField(value, utils.getUTF8(src.GetField(realname)))
 
     def shape2Database(self, srcLayer, name, path):
         destLayer = self.dbconn.GetLayerByName(self.getTablename())
 
         if srcLayer is None:
-            shpimporter.ERROR("Shapefile is None!")
+            logger.error("Shapefile is None!")
             return -1
 
         if destLayer is None:
-            shpimporter.ERROR("No destination layer given!")
+            logger.error("No destination layer given!")
             return -1
 
         count = srcLayer.GetFeatureCount()
-        shpimporter.DEBUG("Try to add %i features to database." % count)
+        logger.debug("Try to add %i features to database." % count)
 
         srcLayer.ResetReading()
+        self.srcLayer = srcLayer
 
         geomType    = -1
         success     = 0
-        unsupported = 0
+        unsupported = {}
         creationFailed = 0
         featureDef  = destLayer.GetLayerDefn()
 
@@ -143,7 +190,7 @@
             geom     = feat.GetGeometryRef()
 
             if geom is None:
-                shpimporter.DEBUG("Unkown Geometry reference for feature")
+                logger.debug("Unkown Geometry reference for feature")
                 continue
 
             geomType = geom.GetGeometryType()
@@ -151,25 +198,31 @@
             if self.isGeometryValid(geomType):
                 newFeat = self.createNewFeature(featureDef,
                                                 feat,
-                                                name=name,
+                                                name=utils.getUTF8(name),
                                                 path=path)
 
                 if newFeat is not None:
                     newFeat.SetField("path", utils.getUTF8Path(path))
                     newFeat = self.transform(newFeat)
-                    res = destLayer.CreateFeature(newFeat)
-                    if res is None or res > 0:
-                        shpimporter.ERROR("Unable to insert feature. Error: %r" % res)
+                    if newFeat:
+                        res = destLayer.CreateFeature(newFeat)
+                        if res is None or res > 0:
+                            logger.error("Unable to insert feature. Error: %r" % res)
+                        else:
+                            success = success + 1
                     else:
-                        success = success + 1
+                        logger.error("Could not transform feature: %s " % feat.GetFID())
+                        creationFailed += 1
                 else:
                     creationFailed = creationFailed + 1
             else:
-                unsupported = unsupported + 1
+                unsupported[utils.getWkbString(geomType)] = \
+                        unsupported.get(utils.getWkbString(geomType), 0) + 1
 
-        shpimporter.INFO("Inserted %i features" % success)
-        shpimporter.INFO("Failed to create %i features" % creationFailed)
-        shpimporter.INFO("Found %i unsupported features" % unsupported)
+        logger.info("Inserted %i features" % success)
+        logger.info("Failed to create %i features" % creationFailed)
+        for key, value in unsupported.items():
+            logger.info("Found %i unsupported features of type: %s" % (value, key))
 
         if self.tracking_import:
             unhandled = []
@@ -179,14 +232,14 @@
                     unhandled.append(act_field)
 
             if len(unhandled):
-                shpimporter.INFO("Did not import values from fields: %s " % \
+                logger.info("Did not import values from fields: %s " % \
                         " ".join(unhandled))
 
         try:
-            if self.config.dry_run > 0:
+            if self.dry_run:
                 return geomType
             destLayer.CommitTransaction()
-        except e:
-            shpimporter.ERROR("Exception while committing transaction.")
+        except:
+            logger.error("Exception while committing transaction.")
 
         return geomType