Mercurial > dive4elements > river

changeset 4970:174fbaa3d344 dami

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add handling of River Names and remove target_src parameter This is the first step to make the shpimporter into a more generic geo importer that communicates directly (not only over ogr) with the database. Untested with Oracle
author Andre Heinecke <aheinecke@intevation.de>
date Wed, 13 Feb 2013 12:02:30 +0100
parents 1469066cc7d9
children fc7ebf104779
files flys-backend/contrib/shpimporter/importer.py flys-backend/contrib/shpimporter/shpimporter.py flys-backend/contrib/shpimporter/utils.py
diffstat 3 files changed, 101 insertions(+), 42 deletions(-) [+]
line wrap: on
line diff
--- a/flys-backend/contrib/shpimporter/importer.py	Mon Feb 04 17:54:39 2013 +0100
+++ b/flys-backend/contrib/shpimporter/importer.py	Wed Feb 13 12:02:30 2013 +0100
@@ -9,12 +9,12 @@
 
 class Importer:
 
-    def __init__(self, config, dbconn):
-        self.config = config
+    def __init__(self, river_id, dbconn, dry_run):
+        self.river_id = river_id
         self.dbconn = dbconn
-        self.river_id = config.river_id
+        self.dry_run = dry_run
         self.dest_srs = osr.SpatialReference()
-        self.dest_srs.ImportFromEPSG(config.target_srs)
+        self.dest_srs.ImportFromEPSG(31467)
         self.handled_fields = []
         self.tracking_import = False
         self.srcLayer = None
@@ -219,7 +219,7 @@
                         " ".join(unhandled))
 
         try:
-            if self.config.dry_run > 0:
+            if self.dry_run:
                 return geomType
             destLayer.CommitTransaction()
         except e:
--- a/flys-backend/contrib/shpimporter/shpimporter.py	Mon Feb 04 17:54:39 2013 +0100
+++ b/flys-backend/contrib/shpimporter/shpimporter.py	Wed Feb 13 12:02:30 2013 +0100
@@ -4,6 +4,8 @@
     import ogr
 
 import utils, optparse
+import sys
+import os
 
 from uesg  import UESG
 from axis  import Axis
@@ -38,22 +40,22 @@
     print "ERROR: %s" % msg
 
 
-def getImporters(config, dbconn):
+def getImporters(river_id, dbconn, dry_run):
     return [
-        Axis(config, dbconn),
-        KM(config, dbconn),
-        CrosssectionTrack(config, dbconn),
-        Line(config, dbconn),
-        Fixpoint(config, dbconn),
-        Building(config, dbconn),
-        Floodplain(config, dbconn),
-        HydrBoundary(config, dbconn),
-        HydrBoundaryPoly(config, dbconn),
-        HWSLines(config, dbconn),
-        HWSPoints(config, dbconn),
-        GaugeLocation(config, dbconn),
-        Catchment(config, dbconn),
-        UESG(config, dbconn)
+        Axis(river_id, dbconn, dry_run),
+        KM(river_id, dbconn, dry_run),
+        CrosssectionTrack(river_id, dbconn, dry_run),
+        Line(river_id, dbconn, dry_run),
+        Fixpoint(river_id, dbconn, dry_run),
+        Building(river_id, dbconn, dry_run),
+        Floodplain(river_id, dbconn, dry_run),
+        HydrBoundary(river_id, dbconn, dry_run),
+        HydrBoundaryPoly(river_id, dbconn, dry_run),
+        HWSLines(river_id, dbconn, dry_run),
+        HWSPoints(river_id, dbconn, dry_run),
+        GaugeLocation(river_id, dbconn, dry_run),
+        Catchment(river_id, dbconn, dry_run),
+        UESG(river_id, dbconn, dry_run)
         ]
 
 
@@ -64,7 +66,7 @@
     parser.add_option("--host", type="string")
     parser.add_option("--user", type="string")
     parser.add_option("--password", type="string")
-    parser.add_option("--river_id", type="int")
+    parser.add_option("--river_name", type="string")
     parser.add_option("--verbose", type="int", default=1)
     parser.add_option("--dry_run", type="int", default=0)
     parser.add_option("--ogr_connection", type="string")
@@ -96,9 +98,6 @@
         if not config.password:
             ERROR("No password specified!")
             raise Exception("Invalid config")
-    if config.river_id == None:
-        ERROR("No river id specified!")
-        raise Exception("Invalid config")
 
     return config
 
@@ -135,7 +134,6 @@
 
     return False
 
-
 def main():
     config=None
     try:
@@ -155,33 +153,75 @@
     else:
         connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
 
+    if 'OCI:' in connstr:
+        try:
+            import cx_Oracle as dbapi
+            raw_connstr=connstr.replace("OCI:", "")
+        except ImportErrror:
+            ERROR("Module cx_Oracle not found in: %s\n"
+                  "Neccessary to connect to a Oracle Database.\n"
+                  "Please refer to the installation "
+                  "documentation." % sys.path)
+
+    else: # Currently only support for oracle and postgres
+        try:
+            import psycopg2 as dbapi
+            raw_connstr=connstr.replace("PG:", "")
+        except ImportErrror:
+            ERROR("Module psycopg2 not found in: %s\n"
+                  "Neccessary to connect to a Posgresql Database.\n"
+                  "Please refer to the installation "
+                  "documentation." % sys.path)
+
+    dbconn_raw = dbapi.connect(raw_connstr)
     dbconn = ogr.Open(connstr)
 
     if dbconn == None:
         ERROR("Could not connect to database %s" % connstr)
         return -1
 
-    importers = getImporters(config, dbconn)
     types = {}
 
-    for importer in importers:
-        if skip_importer(config, importer):
-            INFO("Skip import of '%s'" % importer.getName())
-            continue
-
-        INFO("Start import of '%s'" % importer.getName())
+    directories = []
+    if not config.river_name:
+        for file in os.listdir(config.directory):
+            if os.path.isdir(file):
+                directories.append(os.path.join(file))
+    else:
+        directories.append(config.directory)
 
-        shapes = utils.findShapefiles(importer.getPath(config.directory))
-        DEBUG("Found %i Shapefiles" % len(shapes))
+    for directory in directories:
+        if not config.river_name:
+            river_name = os.path.basename(os.path.normpath(directory))
+        else:
+            river_name = config.river_name
+        river_id = utils.getRiverId(dbconn_raw, river_name)
 
-        for shpTuple in shapes:
-            geomType = importer.walkOverShapes(shpTuple)
-            try:
-                if geomType is not None:
-                    num = types[geomType]
-                    types[geomType] = num+1
-            except:
-                types[geomType] = 1
+        if not river_id:
+            INFO("Could not find river in database. Skipping: %s"
+                  % river_name)
+            continue
+        else:
+            INFO("Importing River: %s" % river_name)
+
+        for importer in getImporters(river_id, dbconn, config.dry_run):
+            if skip_importer(config, importer):
+                INFO("Skip import of '%s'" % importer.getName())
+                continue
+
+            INFO("Start import of '%s'" % importer.getName())
+
+            shapes = utils.findShapefiles(importer.getPath(config.directory))
+            DEBUG("Found %i Shapefiles" % len(shapes))
+
+            for shpTuple in shapes:
+                geomType = importer.walkOverShapes(shpTuple)
+                try:
+                    if geomType is not None:
+                        num = types[geomType]
+                        types[geomType] = num+1
+                except:
+                    types[geomType] = 1
 
     for key in types:
         DEBUG("%i x geometry type %s" % (types[key], key))
--- a/flys-backend/contrib/shpimporter/utils.py	Mon Feb 04 17:54:39 2013 +0100
+++ b/flys-backend/contrib/shpimporter/utils.py	Wed Feb 13 12:02:30 2013 +0100
@@ -1,8 +1,13 @@
 import os
 import sys
 from shpimporter import DEBUG, INFO, ERROR
+try:
+    from osgeo import ogr
+except ImportErrror:
+    import ogr
 
 SHP='.shp'
+SQL_SELECT_ID="SELECT id FROM %s WHERE %s = '%s'"
 
 def findShapefiles(path):
     shapes = []
@@ -20,6 +25,20 @@
 
     return shapes
 
+def getRiverId(dbconn, name):
+    """
+    Returns the id of the river "name"
+    Dbconn must be a python database connection api compliant object
+    """
+    cur = dbconn.cursor()
+    select_stmt = SQL_SELECT_ID % ("rivers", "name", name)
+    cur.execute(select_stmt)
+    row = cur.fetchone()
+    if row:
+        return row[0]
+    else:
+        return 0
+
 def getUTF8(string):
     """
     Tries to convert the string to a UTF-8 encoding by first checking if it
http://dive4elements.wald.intevation.org