# HG changeset patch # User Ingo Weinzierl # Date 1335508752 0 # Node ID bd9e76e0b55dd4c413f0886d60afd297c68e058b # Parent 875a87b8489f21c29d130e5d9524d7de00678a5e Improved the python shapefile importer. flys-backend/trunk@4313 c6561f87-3c4e-4783-a992-168aeb5c3f6f diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/ChangeLog --- a/flys-backend/ChangeLog Thu Apr 26 13:01:02 2012 +0000 +++ b/flys-backend/ChangeLog Fri Apr 27 06:39:12 2012 +0000 @@ -1,3 +1,21 @@ +2012-04-27 Ingo Weinzierl + + * contrib/shpimporter/lines.py, + contrib/shpimporter/buildings.py, + contrib/shpimporter/importer.py, + contrib/shpimporter/fixpoints.py, + contrib/shpimporter/axis.py, + contrib/shpimporter/crosssectiontracks.py, + contrib/shpimporter/km.py: New classes for importing specific + shapefiles. Each of this imports defines its target db tablename and a + directory path to the shapefiles it should import. + + * contrib/shpimporter/uesg.py: Some modifications necessary to streamline + the import process of shapefiles. + + * contrib/shpimporter/shpimporter.py: Use all available imports for the + import process. + 2012-04-26 Ingo Weinzierl * doc/schema/oracle-spatial.sql: Repaired broken schema. diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/axis.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/axis.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,45 @@ +import ogr + +from importer import Importer + +TABLE_NAME="river_axes" +PATH="Geodaesie/Flussachse+km" + + +class Axis(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME + + + def isGeometryValid(self, geomType): + return geomType == 2 + + + def isShapeRelevant(self, name, path): + return name == "achse" + + + def createNewFeature(self, featureDef, feat, **args): + newFeat = ogr.Feature(featureDef) + newFeat.SetGeometry(feat.GetGeometryRef()) + + if self.IsFieldSet(feat, "river_id"): + riverId = feat.GetField("river_id") + else: + riverId = self.river_id + + if self.IsFieldSet(feat, "kind"): + kind = feat.GetField("kind") + else: + kind = 0 + + newFeat.SetField("river_id", riverId) + newFeat.SetField("kind", kind) + + return newFeat + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/buildings.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/buildings.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,42 @@ +import ogr + +from importer import Importer + +TABLE_NAME="buildings" +PATH="Geodaesie/Bauwerke" + + +class Building(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME + + + def isGeometryValid(self, geomType): + return geomType == 2 + + + def isShapeRelevant(self, name, path): + return True + + + def createNewFeature(self, featureDef, feat, **args): + newFeat = ogr.Feature(featureDef) + newFeat.SetGeometry(feat.GetGeometryRef()) + + if self.IsFieldSet(feat, "river_id"): + newFeat.SetField("river_id", feat.GetField("river_id")) + else: + newFeat.SetField("river_id", self.river_id) + + if self.IsFieldSet(feat, "Name"): + name = feat.GetField("Name") + print "Building name '%s'" % name + newFeat.SetField("name", name) + + return newFeat + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/crosssectiontracks.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/crosssectiontracks.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,49 @@ +import ogr + +from importer import Importer + +TABLE_NAME="cross_section_tracks" +PATH="Geodaesie/Querprofile" + + +class CrosssectionTrack(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME + + + def isGeometryValid(self, geomType): + return geomType == 2 + + + def isShapeRelevant(self, name, path): + return True + + + def createNewFeature(self, featureDef, feat, **args): + newFeat = ogr.Feature(featureDef) + newFeat.SetGeometry(feat.GetGeometryRef()) + + if self.IsFieldSet(feat, "river_id"): + newFeat.SetField("river_id", feat.GetField("river_id")) + else: + newFeat.SetField("river_id", self.river_id) + + if self.IsFieldSet(feat, "KILOMETER"): + newFeat.SetField("km", feat.GetFieldAsDouble("KILOMETER")) + elif self.IsFieldSet(feat, "KM"): + newFeat.SetField("km", feat.GetFieldAsDouble("KM")) + else: + return None + + if self.IsFieldSet(feat, "ELEVATION"): + newFeat.SetField("z", feat.GetFieldAsDouble("ELEVATION")) + else: + newFeat.SetField("z", 0) + + return newFeat + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/fixpoints.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/fixpoints.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,51 @@ +import ogr + +from importer import Importer + +TABLE_NAME="fixpoints" +PATH="Geodaesie/Festpunkte" + + +class Fixpoint(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME + + + def isGeometryValid(self, geomType): + return geomType == 1 + + + def isShapeRelevant(self, name, path): + return True + + + def createNewFeature(self, featureDef, feat, **args): + newFeat = ogr.Feature(featureDef) + newFeat.SetGeometry(feat.GetGeometryRef()) + + if self.IsFieldSet(feat, "river_id"): + newFeat.SetField("river_id", feat.GetField("river_id")) + else: + newFeat.SetField("river_id", self.river_id) + + if self.IsFieldSet(feat, "KM"): + newFeat.SetField("km", feat.GetFieldAsDouble("KM")) + else: + return None + + if self.IsFieldSet(feat, "X"): + newFeat.SetField("x", feat.GetFieldAsDouble("X")) + + if self.IsFieldSet(feat, "Y"): + newFeat.SetField("y", feat.GetFieldAsDouble("Y")) + + if self.IsFieldSet(feat, "HPGP"): + newFeat.SetField("HPGP", feat.GetField("HPGP")) + + return newFeat + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/importer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/importer.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,113 @@ +import ogr + +class Importer: + + def __init__(self, dbconn, river_id): + self.dbconn = dbconn + self.river_id = river_id + + + def getKind(self, path): + raise NotImplementedError("Importer.getKind is abstract!") + + + def getPath(self, base): + raise NotImplementedError("Importer.getPath is abstract!") + + + def getTablename(self): + raise NotImplementedError("Importer.getTablename is abstract!") + + + def IsFieldSet(self, feat, name): + try: + isset = feat.GetField(name) + return isset is not None + except: + return False + + + def IsDoubleFieldSet(self, feat, name): + try: + isset = feat.GetFieldAsDouble(name) + return isset is not None + except: + return False + + + def isShapeRelevant(self, name, path): + return True + + + def walkOverShapes(self, shape): + (name, path) = shape + if not self.isShapeRelevant(name, path): + print "Skip shapefile '%s'" % name + return + + shp = ogr.Open(shape[1]) + if shp is None: + print "Shapefile '%s' could not be opened!" % path + return + + print "Opened shapefile '%s'" % path + srcLayer = shp.GetLayerByName(name) + + if srcLayer is None: + print "Layer '%s' was not found!" % name + return + + return self.shape2Database(srcLayer, name, path) + + + def shape2Database(self, srcLayer, name, path): + table = ogr.Open(self.dbconn) + destLayer = table.GetLayerByName(self.getTablename()) + + if srcLayer is None: + print "Shapefile is None!" + return -1 + + if destLayer is None: + print "No destination layer given!" + return -1 + + count = srcLayer.GetFeatureCount() + print "Try to add %i features to database." % count + + srcLayer.ResetReading() + + geomType = -1 + success = 0 + unsupported = 0 + featureDef = destLayer.GetLayerDefn() + + for feat in srcLayer: + geom = feat.GetGeometryRef() + geomType = geom.GetGeometryType() + + if self.isGeometryValid(geomType): + newFeat = self.createNewFeature(featureDef, + feat, + name=name, + path=path) + + if newFeat is not None: + res = destLayer.CreateFeature(newFeat) + if res is None or res > 0: + print "Error while inserting feature: %r" % res + else: + success = success + 1 + else: + unsupported = unsupported + 1 + + print "Inserted %i features" % success + print "Found %i unsupported features" % unsupported + + try: + destLayer.CommitTransaction() + except e: + print "Exception while committing transaction." + + return geomType + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/km.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/km.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,44 @@ +import ogr + +from importer import Importer + +TABLE_NAME="river_axes_km" +PATH="Geodaesie/Flussachse+km" + + +class KM(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME + + + def isGeometryValid(self, geomType): + return geomType == 1 + + + def isShapeRelevant(self, name, path): + return name == "km" + + + def createNewFeature(self, featureDef, feat, **args): + newFeat = ogr.Feature(featureDef) + newFeat.SetGeometry(feat.GetGeometryRef()) + + if self.IsFieldSet(feat, "river_id"): + newFeat.SetField("river_id", feat.GetField("river_id")) + else: + newFeat.SetField("river_id", self.river_id) + + if self.IsDoubleFieldSet(feat, "km"): + newFeat.SetField("km", feat.GetFieldAsDouble("km")) + elif self.IsDoubleFieldSet(feat, "KM"): + newFeat.SetField("km", feat.GetFieldAsDouble("KM")) + else: + return None + + return newFeat + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/lines.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/lines.py Fri Apr 27 06:39:12 2012 +0000 @@ -0,0 +1,47 @@ +import ogr + +from importer import Importer + +TABLE_NAME="lines" +PATH="Geodaesie/Linien" + + +class Line(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME + + + def isGeometryValid(self, geomType): + return geomType == 2 + + + def isShapeRelevant(self, name, path): + return True + + + def createNewFeature(self, featureDef, feat, **args): + newFeat = ogr.Feature(featureDef) + newFeat.SetGeometry(feat.GetGeometryRef()) + + if self.IsFieldSet(feat, "river_id"): + newFeat.SetField("river_id", feat.GetField("river_id")) + else: + newFeat.SetField("river_id", self.river_id) + + if self.IsFieldSet(feat, "TYP"): + newFeat.SetField("kind", feat.GetFieldAsDouble("TYP")) + else: + return None + + if self.IsFieldSet(feat, "Z"): + newFeat.SetField("z", feat.GetFieldAsDouble("Z")) + else: + newFeat.SetField("z", 0) + + return newFeat + diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/shpimporter.py --- a/flys-backend/contrib/shpimporter/shpimporter.py Thu Apr 26 13:01:02 2012 +0000 +++ b/flys-backend/contrib/shpimporter/shpimporter.py Fri Apr 27 06:39:12 2012 +0000 @@ -1,29 +1,46 @@ import ogr + import utils -from uesg import UESG +from uesg import UESG +from axis import Axis +from km import KM +from lines import Line +from fixpoints import Fixpoint +from buildings import Building +from crosssectiontracks import CrosssectionTrack -DBCONN='PG:dbname=flystest1 host=localhost user=flys password=flys port=5432' -PATH='/**/**/**/flys3-testdaten/Saar/Hydrologie/UeSG/Berechnung' +DBCONN='OCI:user/pass@host' +PATH='/path/to/Gewaesser/Elbe' +RIVER_ID=the_river_id -def getImporter(): - return UESG(DBCONN) +def getImporters(): + return [ + Axis(DBCONN, RIVER_ID), + KM(DBCONN, RIVER_ID), + CrosssectionTrack(DBCONN, RIVER_ID), + Line(DBCONN, RIVER_ID), + Fixpoint(DBCONN, RIVER_ID), + Building(DBCONN, RIVER_ID), + UESG(DBCONN, RIVER_ID)] if __name__ == '__main__': - shapes = utils.findShapefiles(PATH) - print "Found %i Shapefiles" % len(shapes) + importers = getImporters() types = {} - importer = getImporter() - for shpTuple in shapes: - geomType = importer.walkOverShapes(shpTuple) - try: - num = types[geomType] - types[geomType] = num+1 - except: - types[geomType] = 1 + for importer in importers: + shapes = utils.findShapefiles(importer.getPath(PATH)) + print "Found %i Shapefiles" % len(shapes) + + for shpTuple in shapes: + geomType = importer.walkOverShapes(shpTuple) + try: + num = types[geomType] + types[geomType] = num+1 + except: + types[geomType] = 1 for key in types: print "%i x geometry type %s" % (types[key], key) diff -r 875a87b8489f -r bd9e76e0b55d flys-backend/contrib/shpimporter/uesg.py --- a/flys-backend/contrib/shpimporter/uesg.py Thu Apr 26 13:01:02 2012 +0000 +++ b/flys-backend/contrib/shpimporter/uesg.py Fri Apr 27 06:39:12 2012 +0000 @@ -1,11 +1,20 @@ import ogr +from importer import Importer + + TABLE_NAME="floodmaps" +PATH="Hydrologie/UeSG/Berechnung" -class UESG: - def __init__(self, dbconn): - self.dbconn = dbconn +class UESG(Importer): + + def getPath(self, base): + return "%s/%s" % (base, PATH) + + + def getTablename(self): + return TABLE_NAME def isGeometryValid(self, geomType): @@ -35,75 +44,16 @@ return kind - def walkOverShapes(self, shape): - (name, path) = shape - shp = ogr.Open(shape[1]) - if shp is None: - print "Shapefile '%s' could not be opened!" % path - return - - print "Opened shapefile '%s'" % path - srcLayer = shp.GetLayerByName(name) - - if srcLayer is None: - print "Layer '%s' was not found!" % name - return - - return self.shape2Database(srcLayer, name, self.getKind(path)) - - - def shape2Database(self, srcLayer, name, kind): - table = ogr.Open(self.dbconn) - destLayer = table.GetLayerByName(TABLE_NAME) - - if srcLayer is None: - print "Shapefile is None!" - return -1 - - if destLayer is None: - print "No destination layer given!" - return -1 + def createNewFeature(self, featureDef, feat, **args): + kind = self.getKind(args['path']) - count = srcLayer.GetFeatureCount() - print "Try to add %i features to database." % count - - srcLayer.ResetReading() - - geomType = -1 - - featureDef = destLayer.GetLayerDefn() - - for feat in srcLayer: - geom = feat.GetGeometryRef() - geomType = geom.GetGeometryType() - - newFeat = self.createNewFeature(featureDef, feat) - newFeat.SetField("kind", kind) - newFeat.SetField("name", name) - - if self.isGeometryValid(geomType): - res = destLayer.CreateFeature(newFeat) - if res is None or res > 0: - print "Error while inserting feature: %r" % res - else: - print "Geometry type not supported: %i" % geomType - - try: - destLayer.CommitTransaction() - except e: - print "Exception while committing transaction." - - return geomType - - - def createNewFeature(self, featureDef, feat): newFeat = ogr.Feature(featureDef) newFeat.SetGeometry(feat.GetGeometryRef()) - if feat.IsFieldSet("river_id"): - riverId = feat.GetField("river_id") + if self.IsFieldSet(feat, "river_id"): + riverId = feat.GetField(feat) else: - riverId = 1 + riverId = self.river_id if feat.IsFieldSet("diff"): diff = feat.GetFieldAsDouble("diff") @@ -132,6 +82,8 @@ newFeat.SetField("count", count) newFeat.SetField("area", area) newFeat.SetField("perimeter", perimeter) + newFeat.SetField("kind", kind) + newFeat.SetField("name", args['name']) return newFeat