Mercurial > dive4elements > river

comparison flys-backend/contrib/shpimporter/shpimporter.py @ 5006:769593a84606 dami

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer: Behold, Logging!
author Andre Heinecke <aheinecke@intevation.de>
date Fri, 15 Feb 2013 16:22:13 +0100
parents eb2d6609387c
children 514948fdae54
comparison
equal deleted inserted replaced
5005:eb2d6609387c 5006:769593a84606
4 import ogr 4 import ogr
5 5
6 import utils, optparse 6 import utils, optparse
7 import sys 7 import sys
8 import os 8 import os
9 import logging
9 10
10 from uesg import UESG 11 from uesg import UESG
11 from axis import Axis 12 from axis import Axis
12 from km import KM 13 from km import KM
13 from fixpoints import Fixpoint 14 from fixpoints import Fixpoint
18 from hws import HWSLines, HWSPoints 19 from hws import HWSLines, HWSPoints
19 from gauges import GaugeLocation 20 from gauges import GaugeLocation
20 from catchments import Catchment 21 from catchments import Catchment
21 from dgm import insertRiverDgm 22 from dgm import insertRiverDgm
22 23
23 24 logger = logging.getLogger("shpimporter")
24 VERBOSE_DEBUG=2 25
25 VERBOSE_INFO=1 26 def initialize_logging(level):
26 27 """Initializes the logging system"""
27 28 root = logging.getLogger()
28 def DEBUG(msg): 29 root.setLevel(level)
29 config = getConfig() 30 hdlr = logging.StreamHandler()
30 if config.verbose >= VERBOSE_DEBUG: 31 fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
31 print "DEBUG: %s" % msg 32 hdlr.setFormatter(fmt)
32 33 root.addHandler(hdlr)
33 def INFO(msg):
34 config = getConfig()
35 if config.verbose >= VERBOSE_INFO:
36 print "INFO: %s" % msg
37
38 def ERROR(msg):
39 config = getConfig()
40 print "ERROR: %s" % msg
41
42 34
43 def getImporters(river_id, dbconn, dry_run): 35 def getImporters(river_id, dbconn, dry_run):
44 return [ 36 return [
45 Axis(river_id, dbconn, dry_run), 37 Axis(river_id, dbconn, dry_run),
46 KM(river_id, dbconn, dry_run), 38 KM(river_id, dbconn, dry_run),
82 parser.add_option("--skip_kms", type="int") 74 parser.add_option("--skip_kms", type="int")
83 parser.add_option("--skip_uesgs", type="int") 75 parser.add_option("--skip_uesgs", type="int")
84 parser.add_option("--skip_dgm", type="int") 76 parser.add_option("--skip_dgm", type="int")
85 (config, args) = parser.parse_args() 77 (config, args) = parser.parse_args()
86 78
79 if config.verbose > 1:
80 initialize_logging(logging.DEBUG)
81 elif config.verbose == 1:
82 initialize_logging(logging.INFO)
83 else:
84 initialize_logging(logging.WARN)
85
87 if config.directory == None: 86 if config.directory == None:
88 ERROR("No river directory specified!") 87 logger.error("No river directory specified!")
89 raise Exception("Invalid config") 88 raise Exception("Invalid config")
90 if not config.ogr_connection: 89 if not config.ogr_connection:
91 if not config.host: 90 if not config.host:
92 ERROR("No database host specified!") 91 logger.error("No database host specified!")
93 raise Exception("Invalid config") 92 raise Exception("Invalid config")
94 if not config.user: 93 if not config.user:
95 ERROR("No databaser user specified!") 94 logger.error("No databaser user specified!")
96 raise Exception("Invalid config") 95 raise Exception("Invalid config")
97 if not config.password: 96 if not config.password:
98 ERROR("No password specified!") 97 logger.error("No password specified!")
99 raise Exception("Invalid config") 98 raise Exception("Invalid config")
100 99
101 return config 100 return config
102 101
103 102
137 config = getConfig() 136 config = getConfig()
138 except: 137 except:
139 return -1 138 return -1
140 139
141 if config == None: 140 if config == None:
142 ERROR("Unable to read config from command line!") 141 logger.error("Unable to read config from command line!")
143 return 142 return
144 143
145 if config.dry_run > 0: 144 if config.dry_run > 0:
146 INFO("You enable 'dry_run'. No database transaction will take place!") 145 logger.info("You enable 'dry_run'. No database transaction will take place!")
147 146
148 if config.ogr_connection: 147 if config.ogr_connection:
149 connstr = config.ogr_connection 148 connstr = config.ogr_connection
150 else: 149 else:
151 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host) 150 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
153 if 'OCI:' in connstr: 152 if 'OCI:' in connstr:
154 try: 153 try:
155 import cx_Oracle as dbapi 154 import cx_Oracle as dbapi
156 raw_connstr=connstr.replace("OCI:", "") 155 raw_connstr=connstr.replace("OCI:", "")
157 except ImportErrror: 156 except ImportErrror:
158 ERROR("Module cx_Oracle not found in: %s\n" 157 logger.error("Module cx_Oracle not found in: %s\n"
159 "Neccessary to connect to a Oracle Database.\n" 158 "Neccessary to connect to a Oracle Database.\n"
160 "Please refer to the installation " 159 "Please refer to the installation "
161 "documentation." % sys.path) 160 "documentation." % sys.path)
162 161
163 else: # Currently only support for oracle and postgres 162 else: # Currently only support for oracle and postgres
164 try: 163 try:
165 import psycopg2 as dbapi 164 import psycopg2 as dbapi
166 raw_connstr=connstr.replace("PG:", "") 165 raw_connstr=connstr.replace("PG:", "")
167 except ImportError: 166 except ImportError:
168 ERROR("Module psycopg2 not found in: %s\n" 167 logger.error("Module psycopg2 not found in: %s\n"
169 "Neccessary to connect to a Posgresql Database.\n" 168 "Neccessary to connect to a Posgresql Database.\n"
170 "Please refer to the installation " 169 "Please refer to the installation "
171 "documentation." % sys.path) 170 "documentation." % sys.path)
172 171
173 dbconn_raw = dbapi.connect(raw_connstr) 172 dbconn_raw = dbapi.connect(raw_connstr)
174 dbconn = ogr.Open(connstr) 173 dbconn = ogr.Open(connstr)
175 174
176 if dbconn == None: 175 if dbconn == None:
177 ERROR("Could not connect to database %s" % connstr) 176 logger.error("Could not connect to database %s" % connstr)
178 return -1 177 return -1
179 178
180 types = {} 179 types = {}
181 180
182 directories = [] 181 directories = []
195 else: 194 else:
196 river_name = config.river_name 195 river_name = config.river_name
197 river_id = utils.getRiverId(dbconn_raw, river_name) 196 river_id = utils.getRiverId(dbconn_raw, river_name)
198 197
199 if not river_id: 198 if not river_id:
200 INFO("Could not find river in database. Skipping: %s" 199 logger.info("Could not find river in database. Skipping: %s"
201 % river_name) 200 % river_name)
202 continue 201 continue
203 else: 202 else:
204 INFO("Importing River: %s" % river_name) 203 logger.info("Importing River: %s" % river_name)
205 204
206 for importer in getImporters(river_id, dbconn, config.dry_run): 205 for importer in getImporters(river_id, dbconn, config.dry_run):
207 if skip_importer(config, importer): 206 if skip_importer(config, importer):
208 INFO("Skip import of '%s'" % importer.getName()) 207 logger.info("Skip import of '%s'" % importer.getName())
209 continue 208 continue
210 209
211 INFO("Start import of '%s'" % importer.getName()) 210 logger.info("Start import of '%s'" % importer.getName())
212 211
213 shapes = utils.findShapefiles(importer.getPath(config.directory)) 212 shapes = utils.findShapefiles(importer.getPath(config.directory))
214 DEBUG("Found %i Shapefiles" % len(shapes)) 213 logger.debug("Found %i Shapefiles" % len(shapes))
215 214
216 for shpTuple in shapes: 215 for shpTuple in shapes:
217 geomType = importer.walkOverShapes(shpTuple) 216 geomType = importer.walkOverShapes(shpTuple)
218 try: 217 try:
219 if geomType is not None: 218 if geomType is not None:
221 types[geomType] = num+1 220 types[geomType] = num+1
222 except: 221 except:
223 types[geomType] = 1 222 types[geomType] = 1
224 223
225 for key in types: 224 for key in types:
226 DEBUG("%i x geometry type %s" % (types[key], key)) 225 logger.debug("%i x geometry type %s" % (types[key], key))
227 226
228 if not config.skip_dgm: 227 if not config.skip_dgm:
229 dgmfilename = os.path.join( 228 dgmfilename = os.path.join(
230 config.directory, "..", "DGMs.csv") 229 config.directory, "..", "DGMs.csv")
231 if not os.access(dgmfilename, os.R_OK) or not \ 230 if not os.access(dgmfilename, os.R_OK) or not \
232 os.path.isfile(dgmfilename): 231 os.path.isfile(dgmfilename):
233 INFO("Could not find or access DGM file: %s \n" 232 logger.info("Could not find or access DGM file: %s \n"
234 "Skipping DGM import." % dgmfilename) 233 "Skipping DGM import." % dgmfilename)
235 else: 234 else:
236 INFO("Inserting DGM meta information in 'dem' table.") 235 logger.info("Inserting DGM meta information in 'dem' table.")
237 insertRiverDgm(dbconn_raw, dgmfilename, river_name, config.dry_run) 236 insertRiverDgm(dbconn_raw, dgmfilename, river_name, config.dry_run)
238 237
239 if __name__ == '__main__': 238 if __name__ == '__main__':
240 main() 239 main()
http://dive4elements.wald.intevation.org