--- a/flys-backend/contrib/shpimporter/shpimporter.py	Thu Feb 28 11:06:20 2013 +0100
+++ b/flys-backend/contrib/shpimporter/shpimporter.py	Thu Feb 28 11:48:17 2013 +0100
@@ -1,14 +1,16 @@
 try:
     from osgeo import ogr
-except ImportErrror:
+except ImportError:
     import ogr
 
 import utils, optparse
+import sys
+import os
+import logging
 
 from uesg  import UESG
 from axis  import Axis
 from km    import KM
-from lines import Line
 from fixpoints import Fixpoint
 from buildings import Building
 from crosssectiontracks import CrosssectionTrack
@@ -16,44 +18,33 @@
 from boundaries import HydrBoundary, HydrBoundaryPoly
 from hws import HWSLines, HWSPoints
 from gauges import GaugeLocation
-from catchments import Catchment
-
-
-VERBOSE_DEBUG=2
-VERBOSE_INFO=1
-
-
-def DEBUG(msg):
-    config = getConfig()
-    if config.verbose >= VERBOSE_DEBUG:
-        print "DEBUG: %s" % msg
-
-def INFO(msg):
-    config = getConfig()
-    if config.verbose >= VERBOSE_INFO:
-        print "INFO: %s" % msg
+from dgm import insertRiverDgm
 
-def ERROR(msg):
-    config = getConfig()
-    print "ERROR: %s" % msg
+logger = logging.getLogger("shpimporter")
 
+def initialize_logging(level):
+    """Initializes the logging system"""
+    root = logging.getLogger()
+    root.setLevel(level)
+    hdlr = logging.StreamHandler()
+    fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
+    hdlr.setFormatter(fmt)
+    root.addHandler(hdlr)
 
-def getImporters(config, dbconn):
+def getImporters(river_id, dbconn, dry_run):
     return [
-        Axis(config, dbconn),
-        KM(config, dbconn),
-        CrosssectionTrack(config, dbconn),
-        Line(config, dbconn),
-        Fixpoint(config, dbconn),
-        Building(config, dbconn),
-        Floodplain(config, dbconn),
-        HydrBoundary(config, dbconn),
-        HydrBoundaryPoly(config, dbconn),
-        HWSLines(config, dbconn),
-        HWSPoints(config, dbconn),
-        GaugeLocation(config, dbconn),
-        Catchment(config, dbconn),
-        UESG(config, dbconn)
+        Axis(river_id, dbconn, dry_run),
+        KM(river_id, dbconn, dry_run),
+        CrosssectionTrack(river_id, dbconn, dry_run),
+        Fixpoint(river_id, dbconn, dry_run),
+        Building(river_id, dbconn, dry_run),
+        Floodplain(river_id, dbconn, dry_run),
+        HydrBoundary(river_id, dbconn, dry_run),
+        HydrBoundaryPoly(river_id, dbconn, dry_run),
+        HWSLines(river_id, dbconn, dry_run),
+        HWSPoints(river_id, dbconn, dry_run),
+        GaugeLocation(river_id, dbconn, dry_run),
+        UESG(river_id, dbconn, dry_run)
         ]
 
 
@@ -64,7 +55,7 @@
     parser.add_option("--host", type="string")
     parser.add_option("--user", type="string")
     parser.add_option("--password", type="string")
-    parser.add_option("--river_id", type="int")
+    parser.add_option("--river_name", type="string")
     parser.add_option("--verbose", type="int", default=1)
     parser.add_option("--dry_run", type="int", default=0)
     parser.add_option("--ogr_connection", type="string")
@@ -72,33 +63,36 @@
     parser.add_option("--skip_hydr_boundaries", type="int")
     parser.add_option("--skip_buildings", type="int")
     parser.add_option("--skip_crosssections", type="int")
-    parser.add_option("--skip_lines", type="int")
     parser.add_option("--skip_fixpoints", type="int")
     parser.add_option("--skip_floodplains", type="int")
     parser.add_option("--skip_hws_lines", type="int")
     parser.add_option("--skip_hws_points", type="int")
     parser.add_option("--skip_gauge_locations", type="int")
-    parser.add_option("--skip_catchments", type="int")
     parser.add_option("--skip_kms", type="int")
     parser.add_option("--skip_uesgs", type="int")
+    parser.add_option("--skip_dgm", type="int")
     (config, args) = parser.parse_args()
 
+    if config.verbose > 1:
+        initialize_logging(logging.DEBUG)
+    elif config.verbose == 1:
+        initialize_logging(logging.INFO)
+    else:
+        initialize_logging(logging.WARN)
+
     if config.directory == None:
-        ERROR("No river directory specified!")
+        logger.error("No river directory specified!")
         raise Exception("Invalid config")
     if not config.ogr_connection:
         if not config.host:
-            ERROR("No database host specified!")
+            logger.error("No database host specified!")
             raise Exception("Invalid config")
         if not config.user:
-            ERROR("No databaser user specified!")
+            logger.error("No databaser user specified!")
             raise Exception("Invalid config")
         if not config.password:
-            ERROR("No password specified!")
+            logger.error("No password specified!")
             raise Exception("Invalid config")
-    if config.river_id == None:
-        ERROR("No river id specified!")
-        raise Exception("Invalid config")
 
     return config
 
@@ -114,20 +108,17 @@
         return True
     elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack):
         return True
-    elif config.skip_lines == 1 and isinstance(importer, Line):
-        return True
     elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint):
         return True
     elif config.skip_floodplains == 1 and isinstance(importer, Floodplain):
         return True
-    elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints):
-        return True
     elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines):
         return True
+    elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints) and \
+            not isinstance(importer, HWSLines):
+        return True
     elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation):
         return True
-    elif config.skip_catchments == 1 and isinstance(importer, Catchment):
-        return True
     elif config.skip_kms == 1 and isinstance(importer, KM):
         return True
     elif config.skip_uesgs == 1 and isinstance(importer, UESG):
@@ -135,7 +126,6 @@
 
     return False
 
-
 def main():
     config=None
     try:
@@ -144,48 +134,109 @@
         return -1
 
     if config == None:
-        ERROR("Unable to read config from command line!")
+        logger.error("Unable to read config from command line!")
         return
 
     if config.dry_run > 0:
-        INFO("You enable 'dry_run'. No database transaction will take place!")
+        logger.info("You enable 'dry_run'. No database transaction will take place!")
 
     if config.ogr_connection:
         connstr = config.ogr_connection
     else:
         connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
 
+    oracle = False # Marker if oracle is used.
+    if 'OCI:' in connstr:
+        oracle = True
+        try:
+            import cx_Oracle as dbapi
+            raw_connstr=connstr.replace("OCI:", "")
+        except ImportError:
+            logger.error("Module cx_Oracle not found in: %s\n"
+                  "Neccessary to connect to a Oracle Database.\n"
+                  "Please refer to the installation "
+                  "documentation." % sys.path)
+            return -1
+
+    else: # Currently only support for oracle and postgres
+        try:
+            import psycopg2 as dbapi
+            raw_connstr=connstr.replace("PG:", "")
+        except ImportError:
+            logger.error("Module psycopg2 not found in: %s\n"
+                  "Neccessary to connect to a Posgresql Database.\n"
+                  "Please refer to the installation "
+                  "documentation." % sys.path)
+            return -1
+
+    dbconn_raw = dbapi.connect(raw_connstr)
     dbconn = ogr.Open(connstr)
 
     if dbconn == None:
-        ERROR("Could not connect to database %s" % connstr)
+        logger.error("Could not connect to database %s" % connstr)
         return -1
 
-    importers = getImporters(config, dbconn)
     types = {}
 
-    for importer in importers:
-        if skip_importer(config, importer):
-            INFO("Skip import of '%s'" % importer.getName())
-            continue
-
-        INFO("Start import of '%s'" % importer.getName())
-
-        shapes = utils.findShapefiles(importer.getPath(config.directory))
-        DEBUG("Found %i Shapefiles" % len(shapes))
+    directories = []
+    if not config.river_name:
+        for file in [os.path.join(config.directory, d) for d in \
+                os.listdir(config.directory)]:
+            if os.path.isdir(file):
+                directories.append(file)
+    else:
+        directories.append(config.directory)
 
-        for shpTuple in shapes:
-            geomType = importer.walkOverShapes(shpTuple)
-            try:
-                if geomType is not None:
-                    num = types[geomType]
-                    types[geomType] = num+1
-            except:
-                types[geomType] = 1
+    for directory in directories:
+        if not config.river_name:
+            river_name = utils.getUTF8Path(
+                    os.path.basename(os.path.normpath(directory)))
+        else:
+            river_name = config.river_name
+        river_id = utils.getRiverId(dbconn_raw, river_name, oracle)
 
-    for key in types:
-        DEBUG("%i x geometry type %s" % (types[key], key))
+        if not river_id:
+            logger.info("Could not find river in database. Skipping: %s"
+                  % river_name)
+            continue
+        else:
+            logger.info("Importing River: %s" % river_name)
 
+        for importer in getImporters(river_id, dbconn, config.dry_run):
+            if skip_importer(config, importer):
+                logger.info("Skip import of '%s'" % importer.getName())
+                continue
+
+            logger.info("Start import of '%s'" % importer.getName())
+
+            shapes = utils.findShapefiles(importer.getPath(config.directory))
+            logger.debug("Found %i Shapefiles" % len(shapes))
+
+            for shpTuple in shapes:
+                geomType = importer.walkOverShapes(shpTuple)
+                try:
+                    if geomType is not None:
+                        num = types[geomType]
+                        types[geomType] = num+1
+                except:
+                    types[geomType] = 1
+
+        for key in types:
+            logger.debug("%i x geometry type %s" % (types[key], key))
+
+        if not config.skip_dgm:
+            dgmfilename = os.path.join(
+                    config.directory, "..", "DGMs.csv")
+            if not os.access(dgmfilename, os.R_OK) or not \
+                    os.path.isfile(dgmfilename):
+                logger.info("Could not find or access DGM file: %s \n"
+                     "Skipping DGM import." % dgmfilename)
+            else:
+                logger.info("Inserting DGM meta information in 'dem' table.")
+                insertRiverDgm(dbconn_raw, dgmfilename, river_name,
+                        config.dry_run, oracle)
+        else:
+            logger.info("Skip import of DGM.")
 
 if __name__ == '__main__':
     main()