Mercurial > dive4elements > river

changeset 2853:bd9e76e0b55d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Improved the python shapefile importer. flys-backend/trunk@4313 c6561f87-3c4e-4783-a992-168aeb5c3f6f
author Ingo Weinzierl <ingo.weinzierl@intevation.de>
date Fri, 27 Apr 2012 06:39:12 +0000
parents 875a87b8489f
children 89b097c8ebd7
files flys-backend/ChangeLog flys-backend/contrib/shpimporter/axis.py flys-backend/contrib/shpimporter/buildings.py flys-backend/contrib/shpimporter/crosssectiontracks.py flys-backend/contrib/shpimporter/fixpoints.py flys-backend/contrib/shpimporter/importer.py flys-backend/contrib/shpimporter/km.py flys-backend/contrib/shpimporter/lines.py flys-backend/contrib/shpimporter/shpimporter.py flys-backend/contrib/shpimporter/uesg.py
diffstat 10 files changed, 460 insertions(+), 82 deletions(-) [+]
line wrap: on
line diff
--- a/flys-backend/ChangeLog	Thu Apr 26 13:01:02 2012 +0000
+++ b/flys-backend/ChangeLog	Fri Apr 27 06:39:12 2012 +0000
@@ -1,3 +1,21 @@
+2012-04-27  Ingo Weinzierl <ingo@intevation.de>
+
+	* contrib/shpimporter/lines.py,
+	  contrib/shpimporter/buildings.py,
+	  contrib/shpimporter/importer.py,
+	  contrib/shpimporter/fixpoints.py,
+	  contrib/shpimporter/axis.py,
+	  contrib/shpimporter/crosssectiontracks.py,
+	  contrib/shpimporter/km.py: New classes for importing specific
+	  shapefiles. Each of this imports defines its target db tablename and a
+	  directory path to the shapefiles it should import.
+
+	* contrib/shpimporter/uesg.py: Some modifications necessary to streamline
+	  the import process of shapefiles.
+
+	* contrib/shpimporter/shpimporter.py: Use all available imports for the
+	  import process.
+
 2012-04-26  Ingo Weinzierl <ingo@intevation.de>
 
 	* doc/schema/oracle-spatial.sql: Repaired broken schema.
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/axis.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,45 @@
+import ogr
+
+from importer import Importer
+
+TABLE_NAME="river_axes"
+PATH="Geodaesie/Flussachse+km"
+
+
+class Axis(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
+
+
+    def isGeometryValid(self, geomType):
+        return geomType == 2
+
+
+    def isShapeRelevant(self, name, path):
+        return name == "achse"
+
+
+    def createNewFeature(self, featureDef, feat, **args):
+        newFeat = ogr.Feature(featureDef)
+        newFeat.SetGeometry(feat.GetGeometryRef())
+
+        if self.IsFieldSet(feat, "river_id"):
+            riverId = feat.GetField("river_id")
+        else:
+            riverId = self.river_id
+
+        if self.IsFieldSet(feat, "kind"):
+            kind = feat.GetField("kind")
+        else:
+            kind = 0
+
+        newFeat.SetField("river_id", riverId)
+        newFeat.SetField("kind", kind)
+
+        return newFeat
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/buildings.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,42 @@
+import ogr
+
+from importer import Importer
+
+TABLE_NAME="buildings"
+PATH="Geodaesie/Bauwerke"
+
+
+class Building(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
+
+
+    def isGeometryValid(self, geomType):
+        return geomType == 2
+
+
+    def isShapeRelevant(self, name, path):
+        return True
+
+
+    def createNewFeature(self, featureDef, feat, **args):
+        newFeat = ogr.Feature(featureDef)
+        newFeat.SetGeometry(feat.GetGeometryRef())
+
+        if self.IsFieldSet(feat, "river_id"):
+            newFeat.SetField("river_id", feat.GetField("river_id"))
+        else:
+            newFeat.SetField("river_id", self.river_id)
+
+        if self.IsFieldSet(feat, "Name"):
+            name = feat.GetField("Name")
+            print "Building name '%s'" % name
+            newFeat.SetField("name", name)
+
+        return newFeat
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/crosssectiontracks.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,49 @@
+import ogr
+
+from importer import Importer
+
+TABLE_NAME="cross_section_tracks"
+PATH="Geodaesie/Querprofile"
+
+
+class CrosssectionTrack(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
+
+
+    def isGeometryValid(self, geomType):
+        return geomType == 2
+
+
+    def isShapeRelevant(self, name, path):
+        return True
+
+
+    def createNewFeature(self, featureDef, feat, **args):
+        newFeat = ogr.Feature(featureDef)
+        newFeat.SetGeometry(feat.GetGeometryRef())
+
+        if self.IsFieldSet(feat, "river_id"):
+            newFeat.SetField("river_id", feat.GetField("river_id"))
+        else:
+            newFeat.SetField("river_id", self.river_id)
+
+        if self.IsFieldSet(feat, "KILOMETER"):
+            newFeat.SetField("km", feat.GetFieldAsDouble("KILOMETER"))
+        elif self.IsFieldSet(feat, "KM"):
+            newFeat.SetField("km", feat.GetFieldAsDouble("KM"))
+        else:
+            return None
+
+        if self.IsFieldSet(feat, "ELEVATION"):
+            newFeat.SetField("z", feat.GetFieldAsDouble("ELEVATION"))
+        else:
+            newFeat.SetField("z", 0)
+
+        return newFeat
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/fixpoints.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,51 @@
+import ogr
+
+from importer import Importer
+
+TABLE_NAME="fixpoints"
+PATH="Geodaesie/Festpunkte"
+
+
+class Fixpoint(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
+
+
+    def isGeometryValid(self, geomType):
+        return geomType == 1
+
+
+    def isShapeRelevant(self, name, path):
+        return True
+
+
+    def createNewFeature(self, featureDef, feat, **args):
+        newFeat = ogr.Feature(featureDef)
+        newFeat.SetGeometry(feat.GetGeometryRef())
+
+        if self.IsFieldSet(feat, "river_id"):
+            newFeat.SetField("river_id", feat.GetField("river_id"))
+        else:
+            newFeat.SetField("river_id", self.river_id)
+
+        if self.IsFieldSet(feat, "KM"):
+            newFeat.SetField("km", feat.GetFieldAsDouble("KM"))
+        else:
+            return None
+
+        if self.IsFieldSet(feat, "X"):
+            newFeat.SetField("x", feat.GetFieldAsDouble("X"))
+
+        if self.IsFieldSet(feat, "Y"):
+            newFeat.SetField("y", feat.GetFieldAsDouble("Y"))
+
+        if self.IsFieldSet(feat, "HPGP"):
+            newFeat.SetField("HPGP", feat.GetField("HPGP"))
+
+        return newFeat
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/importer.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,113 @@
+import ogr
+
+class Importer:
+
+    def __init__(self, dbconn, river_id):
+        self.dbconn   = dbconn
+        self.river_id = river_id
+
+
+    def getKind(self, path):
+        raise NotImplementedError("Importer.getKind is abstract!")
+
+
+    def getPath(self, base):
+        raise NotImplementedError("Importer.getPath is abstract!")
+
+
+    def getTablename(self):
+        raise NotImplementedError("Importer.getTablename is abstract!")
+
+
+    def IsFieldSet(self, feat, name):
+        try:
+            isset = feat.GetField(name)
+            return isset is not None
+        except:
+            return False
+
+
+    def IsDoubleFieldSet(self, feat, name):
+        try:
+            isset = feat.GetFieldAsDouble(name)
+            return isset is not None
+        except:
+            return False
+
+
+    def isShapeRelevant(self, name, path):
+        return True
+
+
+    def walkOverShapes(self, shape):
+        (name, path) = shape
+        if not self.isShapeRelevant(name, path):
+            print "Skip shapefile '%s'" % name
+            return
+
+        shp = ogr.Open(shape[1])
+        if shp is None:
+            print "Shapefile '%s' could not be opened!" % path
+            return
+
+        print "Opened shapefile '%s'" % path
+        srcLayer = shp.GetLayerByName(name)
+
+        if srcLayer is None:
+            print "Layer '%s' was not found!" % name
+            return
+
+        return self.shape2Database(srcLayer, name, path)
+
+
+    def shape2Database(self, srcLayer, name, path):
+        table     = ogr.Open(self.dbconn)
+        destLayer = table.GetLayerByName(self.getTablename())
+
+        if srcLayer is None:
+            print "Shapefile is None!"
+            return -1
+
+        if destLayer is None:
+            print "No destination layer given!"
+            return -1
+
+        count = srcLayer.GetFeatureCount()
+        print "Try to add %i features to database." % count
+
+        srcLayer.ResetReading()
+
+        geomType    = -1
+        success     = 0
+        unsupported = 0
+        featureDef  = destLayer.GetLayerDefn()
+
+        for feat in srcLayer:
+            geom     = feat.GetGeometryRef()
+            geomType = geom.GetGeometryType()
+
+            if self.isGeometryValid(geomType):
+                newFeat = self.createNewFeature(featureDef,
+                                                feat,
+                                                name=name,
+                                                path=path)
+
+                if newFeat is not None:
+                    res = destLayer.CreateFeature(newFeat)
+                    if res is None or res > 0:
+                        print "Error while inserting feature: %r" % res
+                    else:
+                        success = success + 1
+            else:
+                unsupported = unsupported + 1
+
+        print "Inserted %i features" % success
+        print "Found %i unsupported features" % unsupported
+
+        try:
+            destLayer.CommitTransaction()
+        except e:
+            print "Exception while committing transaction."
+
+        return geomType
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/km.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,44 @@
+import ogr
+
+from importer import Importer
+
+TABLE_NAME="river_axes_km"
+PATH="Geodaesie/Flussachse+km"
+
+
+class KM(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
+
+
+    def isGeometryValid(self, geomType):
+        return geomType == 1
+
+
+    def isShapeRelevant(self, name, path):
+        return name == "km"
+
+
+    def createNewFeature(self, featureDef, feat, **args):
+        newFeat = ogr.Feature(featureDef)
+        newFeat.SetGeometry(feat.GetGeometryRef())
+
+        if self.IsFieldSet(feat, "river_id"):
+            newFeat.SetField("river_id", feat.GetField("river_id"))
+        else:
+            newFeat.SetField("river_id", self.river_id)
+
+        if self.IsDoubleFieldSet(feat, "km"):
+            newFeat.SetField("km", feat.GetFieldAsDouble("km"))
+        elif self.IsDoubleFieldSet(feat, "KM"):
+            newFeat.SetField("km", feat.GetFieldAsDouble("KM"))
+        else:
+            return None
+
+        return newFeat
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/flys-backend/contrib/shpimporter/lines.py	Fri Apr 27 06:39:12 2012 +0000
@@ -0,0 +1,47 @@
+import ogr
+
+from importer import Importer
+
+TABLE_NAME="lines"
+PATH="Geodaesie/Linien"
+
+
+class Line(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
+
+
+    def isGeometryValid(self, geomType):
+        return geomType == 2
+
+
+    def isShapeRelevant(self, name, path):
+        return True
+
+
+    def createNewFeature(self, featureDef, feat, **args):
+        newFeat = ogr.Feature(featureDef)
+        newFeat.SetGeometry(feat.GetGeometryRef())
+
+        if self.IsFieldSet(feat, "river_id"):
+            newFeat.SetField("river_id", feat.GetField("river_id"))
+        else:
+            newFeat.SetField("river_id", self.river_id)
+
+        if self.IsFieldSet(feat, "TYP"):
+            newFeat.SetField("kind", feat.GetFieldAsDouble("TYP"))
+        else:
+            return None
+
+        if self.IsFieldSet(feat, "Z"):
+            newFeat.SetField("z", feat.GetFieldAsDouble("Z"))
+        else:
+            newFeat.SetField("z", 0)
+
+        return newFeat
+
--- a/flys-backend/contrib/shpimporter/shpimporter.py	Thu Apr 26 13:01:02 2012 +0000
+++ b/flys-backend/contrib/shpimporter/shpimporter.py	Fri Apr 27 06:39:12 2012 +0000
@@ -1,29 +1,46 @@
 import ogr
+
 import utils
-from uesg import UESG
+from uesg  import UESG
+from axis  import Axis
+from km    import KM
+from lines import Line
+from fixpoints import Fixpoint
+from buildings import Building
+from crosssectiontracks import CrosssectionTrack
 
-DBCONN='PG:dbname=flystest1 host=localhost user=flys password=flys port=5432'
-PATH='/**/**/**/flys3-testdaten/Saar/Hydrologie/UeSG/Berechnung'
+DBCONN='OCI:user/pass@host'
+PATH='/path/to/Gewaesser/Elbe'
+RIVER_ID=the_river_id
 
 
-def getImporter():
-    return UESG(DBCONN)
+def getImporters():
+    return [
+        Axis(DBCONN, RIVER_ID),
+        KM(DBCONN, RIVER_ID),
+        CrosssectionTrack(DBCONN, RIVER_ID),
+        Line(DBCONN, RIVER_ID),
+        Fixpoint(DBCONN, RIVER_ID),
+        Building(DBCONN, RIVER_ID),
+        UESG(DBCONN, RIVER_ID)]
 
 
 if __name__ == '__main__':
-    shapes = utils.findShapefiles(PATH)
-    print "Found %i Shapefiles" % len(shapes)
+    importers = getImporters()
 
     types = {}
-    importer = getImporter()
 
-    for shpTuple in shapes:
-        geomType = importer.walkOverShapes(shpTuple)
-        try:
-            num = types[geomType]
-            types[geomType] = num+1
-        except:
-            types[geomType] = 1
+    for importer in importers:
+        shapes = utils.findShapefiles(importer.getPath(PATH))
+        print "Found %i Shapefiles" % len(shapes)
+
+        for shpTuple in shapes:
+            geomType = importer.walkOverShapes(shpTuple)
+            try:
+                num = types[geomType]
+                types[geomType] = num+1
+            except:
+                types[geomType] = 1
 
     for key in types:
         print "%i x geometry type %s" % (types[key], key)
--- a/flys-backend/contrib/shpimporter/uesg.py	Thu Apr 26 13:01:02 2012 +0000
+++ b/flys-backend/contrib/shpimporter/uesg.py	Fri Apr 27 06:39:12 2012 +0000
@@ -1,11 +1,20 @@
 import ogr
 
+from importer import Importer
+
+
 TABLE_NAME="floodmaps"
+PATH="Hydrologie/UeSG/Berechnung"
 
-class UESG:
 
-    def __init__(self, dbconn):
-        self.dbconn = dbconn
+class UESG(Importer):
+
+    def getPath(self, base):
+        return "%s/%s" % (base, PATH)
+
+
+    def getTablename(self):
+        return TABLE_NAME
 
 
     def isGeometryValid(self, geomType):
@@ -35,75 +44,16 @@
         return kind
 
 
-    def walkOverShapes(self, shape):
-        (name, path) = shape
-        shp = ogr.Open(shape[1])
-        if shp is None:
-            print "Shapefile '%s' could not be opened!" % path
-            return
-
-        print "Opened shapefile '%s'" % path
-        srcLayer = shp.GetLayerByName(name)
-
-        if srcLayer is None:
-            print "Layer '%s' was not found!" % name
-            return
-
-        return self.shape2Database(srcLayer, name, self.getKind(path))
-
-
-    def shape2Database(self, srcLayer, name, kind):
-        table = ogr.Open(self.dbconn)
-        destLayer = table.GetLayerByName(TABLE_NAME)
-
-        if srcLayer is None:
-            print "Shapefile is None!"
-            return -1
-
-        if destLayer is None:
-            print "No destination layer given!"
-            return -1
+    def createNewFeature(self, featureDef, feat, **args):
+        kind  = self.getKind(args['path'])
 
-        count = srcLayer.GetFeatureCount()
-        print "Try to add %i features to database." % count
-
-        srcLayer.ResetReading()
-
-        geomType = -1
-
-        featureDef = destLayer.GetLayerDefn()
-
-        for feat in srcLayer:
-            geom = feat.GetGeometryRef()
-            geomType = geom.GetGeometryType()
-
-            newFeat = self.createNewFeature(featureDef, feat)
-            newFeat.SetField("kind", kind)
-            newFeat.SetField("name", name)
-
-            if self.isGeometryValid(geomType):
-                res = destLayer.CreateFeature(newFeat)
-                if res is None or res > 0:
-                    print "Error while inserting feature: %r" % res
-            else:
-                print "Geometry type not supported: %i" % geomType
-
-        try:
-            destLayer.CommitTransaction()
-        except e:
-            print "Exception while committing transaction."
-
-        return geomType
-
-
-    def createNewFeature(self, featureDef, feat):
         newFeat = ogr.Feature(featureDef)
         newFeat.SetGeometry(feat.GetGeometryRef())
 
-        if feat.IsFieldSet("river_id"):
-            riverId = feat.GetField("river_id")
+        if self.IsFieldSet(feat, "river_id"):
+            riverId = feat.GetField(feat)
         else:
-            riverId = 1
+            riverId = self.river_id
 
         if feat.IsFieldSet("diff"):
             diff = feat.GetFieldAsDouble("diff")
@@ -132,6 +82,8 @@
         newFeat.SetField("count", count)
         newFeat.SetField("area", area)
         newFeat.SetField("perimeter", perimeter)
+        newFeat.SetField("kind", kind)
+        newFeat.SetField("name", args['name'])
 
         return newFeat
http://dive4elements.wald.intevation.org