# HG changeset patch # User Andre Heinecke # Date 1360753350 -3600 # Node ID 174fbaa3d344e87b635656dcab4e3fe385672d38 # Parent 1469066cc7d92783c5baed57295ba82462711b7f Add handling of River Names and remove target_src parameter This is the first step to make the shpimporter into a more generic geo importer that communicates directly (not only over ogr) with the database. Untested with Oracle diff -r 1469066cc7d9 -r 174fbaa3d344 flys-backend/contrib/shpimporter/importer.py --- a/flys-backend/contrib/shpimporter/importer.py Mon Feb 04 17:54:39 2013 +0100 +++ b/flys-backend/contrib/shpimporter/importer.py Wed Feb 13 12:02:30 2013 +0100 @@ -9,12 +9,12 @@ class Importer: - def __init__(self, config, dbconn): - self.config = config + def __init__(self, river_id, dbconn, dry_run): + self.river_id = river_id self.dbconn = dbconn - self.river_id = config.river_id + self.dry_run = dry_run self.dest_srs = osr.SpatialReference() - self.dest_srs.ImportFromEPSG(config.target_srs) + self.dest_srs.ImportFromEPSG(31467) self.handled_fields = [] self.tracking_import = False self.srcLayer = None @@ -219,7 +219,7 @@ " ".join(unhandled)) try: - if self.config.dry_run > 0: + if self.dry_run: return geomType destLayer.CommitTransaction() except e: diff -r 1469066cc7d9 -r 174fbaa3d344 flys-backend/contrib/shpimporter/shpimporter.py --- a/flys-backend/contrib/shpimporter/shpimporter.py Mon Feb 04 17:54:39 2013 +0100 +++ b/flys-backend/contrib/shpimporter/shpimporter.py Wed Feb 13 12:02:30 2013 +0100 @@ -4,6 +4,8 @@ import ogr import utils, optparse +import sys +import os from uesg import UESG from axis import Axis @@ -38,22 +40,22 @@ print "ERROR: %s" % msg -def getImporters(config, dbconn): +def getImporters(river_id, dbconn, dry_run): return [ - Axis(config, dbconn), - KM(config, dbconn), - CrosssectionTrack(config, dbconn), - Line(config, dbconn), - Fixpoint(config, dbconn), - Building(config, dbconn), - Floodplain(config, dbconn), - HydrBoundary(config, dbconn), - HydrBoundaryPoly(config, dbconn), - HWSLines(config, dbconn), - HWSPoints(config, dbconn), - GaugeLocation(config, dbconn), - Catchment(config, dbconn), - UESG(config, dbconn) + Axis(river_id, dbconn, dry_run), + KM(river_id, dbconn, dry_run), + CrosssectionTrack(river_id, dbconn, dry_run), + Line(river_id, dbconn, dry_run), + Fixpoint(river_id, dbconn, dry_run), + Building(river_id, dbconn, dry_run), + Floodplain(river_id, dbconn, dry_run), + HydrBoundary(river_id, dbconn, dry_run), + HydrBoundaryPoly(river_id, dbconn, dry_run), + HWSLines(river_id, dbconn, dry_run), + HWSPoints(river_id, dbconn, dry_run), + GaugeLocation(river_id, dbconn, dry_run), + Catchment(river_id, dbconn, dry_run), + UESG(river_id, dbconn, dry_run) ] @@ -64,7 +66,7 @@ parser.add_option("--host", type="string") parser.add_option("--user", type="string") parser.add_option("--password", type="string") - parser.add_option("--river_id", type="int") + parser.add_option("--river_name", type="string") parser.add_option("--verbose", type="int", default=1) parser.add_option("--dry_run", type="int", default=0) parser.add_option("--ogr_connection", type="string") @@ -96,9 +98,6 @@ if not config.password: ERROR("No password specified!") raise Exception("Invalid config") - if config.river_id == None: - ERROR("No river id specified!") - raise Exception("Invalid config") return config @@ -135,7 +134,6 @@ return False - def main(): config=None try: @@ -155,33 +153,75 @@ else: connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host) + if 'OCI:' in connstr: + try: + import cx_Oracle as dbapi + raw_connstr=connstr.replace("OCI:", "") + except ImportErrror: + ERROR("Module cx_Oracle not found in: %s\n" + "Neccessary to connect to a Oracle Database.\n" + "Please refer to the installation " + "documentation." % sys.path) + + else: # Currently only support for oracle and postgres + try: + import psycopg2 as dbapi + raw_connstr=connstr.replace("PG:", "") + except ImportErrror: + ERROR("Module psycopg2 not found in: %s\n" + "Neccessary to connect to a Posgresql Database.\n" + "Please refer to the installation " + "documentation." % sys.path) + + dbconn_raw = dbapi.connect(raw_connstr) dbconn = ogr.Open(connstr) if dbconn == None: ERROR("Could not connect to database %s" % connstr) return -1 - importers = getImporters(config, dbconn) types = {} - for importer in importers: - if skip_importer(config, importer): - INFO("Skip import of '%s'" % importer.getName()) - continue - - INFO("Start import of '%s'" % importer.getName()) + directories = [] + if not config.river_name: + for file in os.listdir(config.directory): + if os.path.isdir(file): + directories.append(os.path.join(file)) + else: + directories.append(config.directory) - shapes = utils.findShapefiles(importer.getPath(config.directory)) - DEBUG("Found %i Shapefiles" % len(shapes)) + for directory in directories: + if not config.river_name: + river_name = os.path.basename(os.path.normpath(directory)) + else: + river_name = config.river_name + river_id = utils.getRiverId(dbconn_raw, river_name) - for shpTuple in shapes: - geomType = importer.walkOverShapes(shpTuple) - try: - if geomType is not None: - num = types[geomType] - types[geomType] = num+1 - except: - types[geomType] = 1 + if not river_id: + INFO("Could not find river in database. Skipping: %s" + % river_name) + continue + else: + INFO("Importing River: %s" % river_name) + + for importer in getImporters(river_id, dbconn, config.dry_run): + if skip_importer(config, importer): + INFO("Skip import of '%s'" % importer.getName()) + continue + + INFO("Start import of '%s'" % importer.getName()) + + shapes = utils.findShapefiles(importer.getPath(config.directory)) + DEBUG("Found %i Shapefiles" % len(shapes)) + + for shpTuple in shapes: + geomType = importer.walkOverShapes(shpTuple) + try: + if geomType is not None: + num = types[geomType] + types[geomType] = num+1 + except: + types[geomType] = 1 for key in types: DEBUG("%i x geometry type %s" % (types[key], key)) diff -r 1469066cc7d9 -r 174fbaa3d344 flys-backend/contrib/shpimporter/utils.py --- a/flys-backend/contrib/shpimporter/utils.py Mon Feb 04 17:54:39 2013 +0100 +++ b/flys-backend/contrib/shpimporter/utils.py Wed Feb 13 12:02:30 2013 +0100 @@ -1,8 +1,13 @@ import os import sys from shpimporter import DEBUG, INFO, ERROR +try: + from osgeo import ogr +except ImportErrror: + import ogr SHP='.shp' +SQL_SELECT_ID="SELECT id FROM %s WHERE %s = '%s'" def findShapefiles(path): shapes = [] @@ -20,6 +25,20 @@ return shapes +def getRiverId(dbconn, name): + """ + Returns the id of the river "name" + Dbconn must be a python database connection api compliant object + """ + cur = dbconn.cursor() + select_stmt = SQL_SELECT_ID % ("rivers", "name", name) + cur.execute(select_stmt) + row = cur.fetchone() + if row: + return row[0] + else: + return 0 + def getUTF8(string): """ Tries to convert the string to a UTF-8 encoding by first checking if it