Mercurial > dive4elements > river
changeset 3654:59ca5dab2782
Shape importer: use python's OptionParse to read user specific configuration from command line.
flys-backend/trunk@5231 c6561f87-3c4e-4783-a992-168aeb5c3f6f
| author | Ingo Weinzierl <ingo.weinzierl@intevation.de> |
|---|---|
| date | Wed, 22 Aug 2012 11:55:55 +0000 |
| parents | 20d249e1589f |
| children | 8654e4480fc6 |
| files | flys-backend/ChangeLog flys-backend/contrib/shpimporter/axis.py flys-backend/contrib/shpimporter/boundaries.py flys-backend/contrib/shpimporter/buildings.py flys-backend/contrib/shpimporter/catchments.py flys-backend/contrib/shpimporter/crosssectiontracks.py flys-backend/contrib/shpimporter/fixpoints.py flys-backend/contrib/shpimporter/floodplains.py flys-backend/contrib/shpimporter/gauges.py flys-backend/contrib/shpimporter/hws.py flys-backend/contrib/shpimporter/importer.py flys-backend/contrib/shpimporter/km.py flys-backend/contrib/shpimporter/lines.py flys-backend/contrib/shpimporter/run.sh flys-backend/contrib/shpimporter/shpimporter.py flys-backend/contrib/shpimporter/uesg.py flys-backend/contrib/shpimporter/utils.py |
| diffstat | 17 files changed, 262 insertions(+), 29 deletions(-) [+] |
line wrap: on
line diff
--- a/flys-backend/ChangeLog Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/ChangeLog Wed Aug 22 11:55:55 2012 +0000 @@ -1,3 +1,29 @@ +2012-08-22 Ingo Weinzierl <ingo@intevation.de> + + * contrib/shpimporter/shpimporter.py: Use OptionParse to read command line + options. Now, it is no longer necessary to adjust user specific settings + in python files. Users should adapt run.sh script for specific settings. + + * contrib/shpimporter/catchments.py, + contrib/shpimporter/boundaries.py, + contrib/shpimporter/hws.py, + contrib/shpimporter/importer.py, + contrib/shpimporter/uesg.py, + contrib/shpimporter/axis.py, + contrib/shpimporter/km.py, + contrib/shpimporter/floodplains.py, + contrib/shpimporter/lines.py, + contrib/shpimporter/gauges.py, + contrib/shpimporter/buildings.py, + contrib/shpimporter/fixpoints.py, + contrib/shpimporter/crosssectiontracks.py: Added new method getName(). + + * contrib/shpimporter/utils.py: Use shpimporter functions to print debug + messages. + + * contrib/shpimporter/run.sh: New shell script to run the shape importer + with a default configuration. + 2012-08-09 Sascha L. Teichmann <sascha.teichmann@intevation.de> * src/main/java/de/intevation/flys/model/FastAnnotations.java:
--- a/flys-backend/contrib/shpimporter/axis.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/axis.py Wed Aug 22 11:55:55 2012 +0000 @@ -2,6 +2,7 @@ from importer import Importer +NAME="Axis" TABLE_NAME="river_axes" PATH="Geodaesie/Flussachse+km" @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 2
--- a/flys-backend/contrib/shpimporter/boundaries.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/boundaries.py Wed Aug 22 11:55:55 2012 +0000 @@ -5,6 +5,7 @@ TABLE_NAME="hydr_boundaries" TABLE_NAME_POLY="hydr_boundaries_poly" PATH="Hydrologie/Hydr.Grenzen/Linien" +NAME="Hydr. Boundaries" class HydrBoundary(Importer): @@ -17,6 +18,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 2 @@ -58,6 +63,10 @@ return TABLE_NAME_POLY + def getName(self): + return "%s (Polygons)" % NAME + + def isGeometryValid(self, geomType): return geomType == 3 or geomType == 6
--- a/flys-backend/contrib/shpimporter/buildings.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/buildings.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="buildings" PATH="Geodaesie/Bauwerke" +NAME="Buildings" class Building(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 2
--- a/flys-backend/contrib/shpimporter/catchments.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/catchments.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="catchment" PATH="Hydrologie/Einzugsgebiet" +NAME="Catchments" class Catchment(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 3 or geomType == 6
--- a/flys-backend/contrib/shpimporter/crosssectiontracks.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/crosssectiontracks.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="cross_section_tracks" PATH="Geodaesie/Querprofile" +NAME="Crosssections" class CrosssectionTrack(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 2
--- a/flys-backend/contrib/shpimporter/fixpoints.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/fixpoints.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="fixpoints" PATH="Geodaesie/Festpunkte" +NAME="Fixpoints" class Fixpoint(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 1
--- a/flys-backend/contrib/shpimporter/floodplains.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/floodplains.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="floodplain" PATH="Hydrologie/Hydr.Grenzen" +NAME="Floodplains" class Floodplain(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 3 or geomType == 6
--- a/flys-backend/contrib/shpimporter/gauges.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/gauges.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="gauge_location" PATH="Hydrologie/Streckendaten" +NAME="Gauge locations" class GaugeLocation(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 1
--- a/flys-backend/contrib/shpimporter/hws.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/hws.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="hws" PATH="Hydrologie/HW-Schutzanlagen" +NAME="HWS" class HWS(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 2
--- a/flys-backend/contrib/shpimporter/importer.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/importer.py Wed Aug 22 11:55:55 2012 +0000 @@ -2,11 +2,12 @@ class Importer: - def __init__(self, dbconn, river_id, dest_srs): - self.dbconn = dbconn - self.river_id = river_id + def __init__(self, config): + self.config = config + self.dbconn = 'OCI:%s/%s@%s' % (config.user, config.password, config.host) + self.river_id = config.river_id self.dest_srs = osr.SpatialReference() - self.dest_srs.ImportFromEPSG(dest_srs) + self.dest_srs.ImportFromEPSG(config.target_srs) def getKind(self, path): @@ -21,6 +22,10 @@ raise NotImplementedError("Importer.getTablename is abstract!") + def getName(self): + raise NotImplementedError("Importer.getTablename is abstract!") + + def IsFieldSet(self, feat, name): try: isset = feat.GetField(name)
--- a/flys-backend/contrib/shpimporter/km.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/km.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="river_axes_km" PATH="Geodaesie/Flussachse+km" +NAME="KMS" class KM(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 1
--- a/flys-backend/contrib/shpimporter/lines.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/lines.py Wed Aug 22 11:55:55 2012 +0000 @@ -4,6 +4,7 @@ TABLE_NAME="lines" PATH="Geodaesie/Linien" +NAME="Lines" class Line(Importer): @@ -16,6 +17,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): return geomType == 2 or geomType == -2147483646
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/flys-backend/contrib/shpimporter/run.sh Wed Aug 22 11:55:55 2012 +0000 @@ -0,0 +1,30 @@ +#!/bin/sh + +RIVER_PATH="/path/to/rivers/river" +RIVER_ID=1 +TARGET_SRS=31466 +HOST=localhost +USER=the_user +PASS=the_pass + +exec python shpimporter.py \ + --directory $RIVER_PATH \ + --river_id $RIVER_ID \ + --target_srs $TARGET_SRS \ + --host $HOST \ + --user $USER \ + --password $PASS \ + --verbose 1 \ + --skip_axis 0 \ + --skip_kms 0 \ + --skip_crosssections 0 \ + --skip_lines 0 \ + --skip_fixpoints 0 \ + --skip_buildings 0 \ + --skip_floodplains 0 \ + --skip_hydr_boundaries 0 \ + --skip_hws 0 \ + --skip_gauge_locations 0 \ + --skip_catchments 0 \ + --skip_uesgs 0 +
--- a/flys-backend/contrib/shpimporter/shpimporter.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/shpimporter.py Wed Aug 22 11:55:55 2012 +0000 @@ -1,6 +1,7 @@ import ogr -import utils +import utils, optparse + from uesg import UESG from axis import Axis from km import KM @@ -14,38 +15,141 @@ from gauges import GaugeLocation from catchments import Catchment -DBCONN='OCI:user/pass@host' -PATH='/path/to/Gewaesser/Elbe' -RIVER_ID=the_river_id -DEST_SRS=31467 + +VERBOSE_DEBUG=2 +VERBOSE_INFO=1 -def getImporters(): +def DEBUG(msg): + config = getConfig() + if config.verbose >= VERBOSE_DEBUG: + print "DEBUG: %s" % msg + +def INFO(msg): + config = getConfig() + if config.verbose >= VERBOSE_INFO: + print "INFO: %s" % msg + +def ERROR(msg): + config = getConfig() + print "ERROR: %s" % msg + + +def getImporters(config): return [ - Axis(DBCONN, RIVER_ID, DEST_SRS), - KM(DBCONN, RIVER_ID, DEST_SRS), - CrosssectionTrack(DBCONN, RIVER_ID, DEST_SRS), - Line(DBCONN, RIVER_ID, DEST_SRS), - Fixpoint(DBCONN, RIVER_ID, DEST_SRS), - Building(DBCONN, RIVER_ID, DEST_SRS), - Floodplain(DBCONN, RIVER_ID, DEST_SRS), - HydrBoundary(DBCONN, RIVER_ID, DEST_SRS), - HydrBoundaryPoly(DBCONN, RIVER_ID, DEST_SRS), - HWS(DBCONN, RIVER_ID, DEST_SRS), - GaugeLocation(DBCONN, RIVER_ID, DEST_SRS), - Catchment(DBCONN, RIVER_ID, DEST_SRS), - UESG(DBCONN, RIVER_ID, DEST_SRS) + Axis(config), + KM(config), + CrosssectionTrack(config), + Line(config), + Fixpoint(config), + Building(config), + Floodplain(config), + HydrBoundary(config), + HydrBoundaryPoly(config), + HWS(config), + GaugeLocation(config), + Catchment(config), + UESG(config) ] -if __name__ == '__main__': - importers = getImporters() +def getConfig(): + parser = optparse.OptionParser() + parser.add_option("--directory", type="string") + parser.add_option("--target_srs", type="int") + parser.add_option("--host", type="string") + parser.add_option("--user", type="string") + parser.add_option("--password", type="string") + parser.add_option("--river_id", type="int") + parser.add_option("--verbose", type="int", default=1) + parser.add_option("--skip_axis", type="int") + parser.add_option("--skip_hydr_boundaries", type="int") + parser.add_option("--skip_buildings", type="int") + parser.add_option("--skip_crosssections", type="int") + parser.add_option("--skip_lines", type="int") + parser.add_option("--skip_fixpoints", type="int") + parser.add_option("--skip_floodplains", type="int") + parser.add_option("--skip_hws", type="int") + parser.add_option("--skip_gauge_locations", type="int") + parser.add_option("--skip_catchments", type="int") + parser.add_option("--skip_kms", type="int") + parser.add_option("--skip_uesgs", type="int") + (config, args) = parser.parse_args() + if config.directory == None: + ERROR("No river directory specified!") + raise Exception("Invalid config") + elif config.host == None: + ERROR("No database host specified!") + raise Exception("Invalid config") + elif config.user == None: + ERROR("No databaser user specified!") + raise Exception("Invalid config") + elif config.password == None: + ERROR("No password specified!") + raise Exception("Invalid config") + elif config.river_id == None: + ERROR("No river id specified!") + raise Exception("Invalid config") + + return config + + +def skip_importer(config, importer): + if config.skip_axis == 1 and isinstance(importer, Axis): + return True + elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundary): + return True + elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundaryPoly): + return True + elif config.skip_buildings == 1 and isinstance(importer, Building): + return True + elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack): + return True + elif config.skip_lines == 1 and isinstance(importer, Line): + return True + elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint): + return True + elif config.skip_floodplains == 1 and isinstance(importer, Floodplain): + return True + elif config.skip_hws == 1 and isinstance(importer, HWS): + return True + elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation): + return True + elif config.skip_catchments == 1 and isinstance(importer, Catchment): + return True + elif config.skip_kms == 1 and isinstance(importer, KM): + return True + elif config.skip_uesgs == 1 and isinstance(importer, UESG): + return True + + return False + + +def parse(): + config=None + try: + config = getConfig() + except: + return + + if config == None: + print "No Config" + ERROR("Unable to read config from command line!") + return + + importers = getImporters(config) types = {} for importer in importers: - shapes = utils.findShapefiles(importer.getPath(PATH)) - print "Found %i Shapefiles" % len(shapes) + if skip_importer(config, importer): + INFO("Skip import of '%s'" % importer.getName()) + continue + + INFO("Start import of '%s'" % importer.getName()) + + shapes = utils.findShapefiles(importer.getPath(config.directory)) + INFO("Found %i Shapefiles" % len(shapes)) for shpTuple in shapes: geomType = importer.walkOverShapes(shpTuple) @@ -56,5 +160,8 @@ types[geomType] = 1 for key in types: - print "%i x geometry type %s" % (types[key], key) + DEBUG("%i x geometry type %s" % (types[key], key)) + +if __name__ == '__main__': + parse()
--- a/flys-backend/contrib/shpimporter/uesg.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/uesg.py Wed Aug 22 11:55:55 2012 +0000 @@ -5,6 +5,7 @@ TABLE_NAME="floodmaps" PATH="Hydrologie/UeSG/Berechnung" +NAME="UESG" class UESG(Importer): @@ -17,6 +18,10 @@ return TABLE_NAME + def getName(self): + return NAME + + def isGeometryValid(self, geomType): if geomType == 3 or geomType == 6: return True
--- a/flys-backend/contrib/shpimporter/utils.py Thu Aug 09 11:17:44 2012 +0000 +++ b/flys-backend/contrib/shpimporter/utils.py Wed Aug 22 11:55:55 2012 +0000 @@ -1,4 +1,5 @@ import os +from shpimporter import DEBUG, INFO, ERROR SHP='.shp' @@ -9,7 +10,7 @@ if len(files) == 0: continue - print "Processing directory '%s' with %i files " % (root, len(files)) + DEBUG("Processing directory '%s' with %i files " % (root, len(files))) for f in files: idx = f.find(SHP)