Mercurial > dive4elements > river

comparison flys-backend/contrib/shpimporter/shpimporter.py @ 4970:174fbaa3d344 dami

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add handling of River Names and remove target_src parameter This is the first step to make the shpimporter into a more generic geo importer that communicates directly (not only over ogr) with the database. Untested with Oracle
author Andre Heinecke <aheinecke@intevation.de>
date Wed, 13 Feb 2013 12:02:30 +0100
parents 1469066cc7d9
children 9eea3cd22ee7
comparison
equal deleted inserted replaced
4963:1469066cc7d9 4970:174fbaa3d344
2 from osgeo import ogr 2 from osgeo import ogr
3 except ImportErrror: 3 except ImportErrror:
4 import ogr 4 import ogr
5 5
6 import utils, optparse 6 import utils, optparse
7 import sys
8 import os
7 9
8 from uesg import UESG 10 from uesg import UESG
9 from axis import Axis 11 from axis import Axis
10 from km import KM 12 from km import KM
11 from lines import Line 13 from lines import Line
36 def ERROR(msg): 38 def ERROR(msg):
37 config = getConfig() 39 config = getConfig()
38 print "ERROR: %s" % msg 40 print "ERROR: %s" % msg
39 41
40 42
41 def getImporters(config, dbconn): 43 def getImporters(river_id, dbconn, dry_run):
42 return [ 44 return [
43 Axis(config, dbconn), 45 Axis(river_id, dbconn, dry_run),
44 KM(config, dbconn), 46 KM(river_id, dbconn, dry_run),
45 CrosssectionTrack(config, dbconn), 47 CrosssectionTrack(river_id, dbconn, dry_run),
46 Line(config, dbconn), 48 Line(river_id, dbconn, dry_run),
47 Fixpoint(config, dbconn), 49 Fixpoint(river_id, dbconn, dry_run),
48 Building(config, dbconn), 50 Building(river_id, dbconn, dry_run),
49 Floodplain(config, dbconn), 51 Floodplain(river_id, dbconn, dry_run),
50 HydrBoundary(config, dbconn), 52 HydrBoundary(river_id, dbconn, dry_run),
51 HydrBoundaryPoly(config, dbconn), 53 HydrBoundaryPoly(river_id, dbconn, dry_run),
52 HWSLines(config, dbconn), 54 HWSLines(river_id, dbconn, dry_run),
53 HWSPoints(config, dbconn), 55 HWSPoints(river_id, dbconn, dry_run),
54 GaugeLocation(config, dbconn), 56 GaugeLocation(river_id, dbconn, dry_run),
55 Catchment(config, dbconn), 57 Catchment(river_id, dbconn, dry_run),
56 UESG(config, dbconn) 58 UESG(river_id, dbconn, dry_run)
57 ] 59 ]
58 60
59 61
60 def getConfig(): 62 def getConfig():
61 parser = optparse.OptionParser() 63 parser = optparse.OptionParser()
62 parser.add_option("--directory", type="string") 64 parser.add_option("--directory", type="string")
63 parser.add_option("--target_srs", type="int") 65 parser.add_option("--target_srs", type="int")
64 parser.add_option("--host", type="string") 66 parser.add_option("--host", type="string")
65 parser.add_option("--user", type="string") 67 parser.add_option("--user", type="string")
66 parser.add_option("--password", type="string") 68 parser.add_option("--password", type="string")
67 parser.add_option("--river_id", type="int") 69 parser.add_option("--river_name", type="string")
68 parser.add_option("--verbose", type="int", default=1) 70 parser.add_option("--verbose", type="int", default=1)
69 parser.add_option("--dry_run", type="int", default=0) 71 parser.add_option("--dry_run", type="int", default=0)
70 parser.add_option("--ogr_connection", type="string") 72 parser.add_option("--ogr_connection", type="string")
71 parser.add_option("--skip_axis", type="int") 73 parser.add_option("--skip_axis", type="int")
72 parser.add_option("--skip_hydr_boundaries", type="int") 74 parser.add_option("--skip_hydr_boundaries", type="int")
94 ERROR("No databaser user specified!") 96 ERROR("No databaser user specified!")
95 raise Exception("Invalid config") 97 raise Exception("Invalid config")
96 if not config.password: 98 if not config.password:
97 ERROR("No password specified!") 99 ERROR("No password specified!")
98 raise Exception("Invalid config") 100 raise Exception("Invalid config")
99 if config.river_id == None:
100 ERROR("No river id specified!")
101 raise Exception("Invalid config")
102 101
103 return config 102 return config
104 103
105 104
106 def skip_importer(config, importer): 105 def skip_importer(config, importer):
132 return True 131 return True
133 elif config.skip_uesgs == 1 and isinstance(importer, UESG): 132 elif config.skip_uesgs == 1 and isinstance(importer, UESG):
134 return True 133 return True
135 134
136 return False 135 return False
137
138 136
139 def main(): 137 def main():
140 config=None 138 config=None
141 try: 139 try:
142 config = getConfig() 140 config = getConfig()
153 if config.ogr_connection: 151 if config.ogr_connection:
154 connstr = config.ogr_connection 152 connstr = config.ogr_connection
155 else: 153 else:
156 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host) 154 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
157 155
156 if 'OCI:' in connstr:
157 try:
158 import cx_Oracle as dbapi
159 raw_connstr=connstr.replace("OCI:", "")
160 except ImportErrror:
161 ERROR("Module cx_Oracle not found in: %s\n"
162 "Neccessary to connect to a Oracle Database.\n"
163 "Please refer to the installation "
164 "documentation." % sys.path)
165
166 else: # Currently only support for oracle and postgres
167 try:
168 import psycopg2 as dbapi
169 raw_connstr=connstr.replace("PG:", "")
170 except ImportErrror:
171 ERROR("Module psycopg2 not found in: %s\n"
172 "Neccessary to connect to a Posgresql Database.\n"
173 "Please refer to the installation "
174 "documentation." % sys.path)
175
176 dbconn_raw = dbapi.connect(raw_connstr)
158 dbconn = ogr.Open(connstr) 177 dbconn = ogr.Open(connstr)
159 178
160 if dbconn == None: 179 if dbconn == None:
161 ERROR("Could not connect to database %s" % connstr) 180 ERROR("Could not connect to database %s" % connstr)
162 return -1 181 return -1
163 182
164 importers = getImporters(config, dbconn)
165 types = {} 183 types = {}
166 184
167 for importer in importers: 185 directories = []
168 if skip_importer(config, importer): 186 if not config.river_name:
169 INFO("Skip import of '%s'" % importer.getName()) 187 for file in os.listdir(config.directory):
188 if os.path.isdir(file):
189 directories.append(os.path.join(file))
190 else:
191 directories.append(config.directory)
192
193 for directory in directories:
194 if not config.river_name:
195 river_name = os.path.basename(os.path.normpath(directory))
196 else:
197 river_name = config.river_name
198 river_id = utils.getRiverId(dbconn_raw, river_name)
199
200 if not river_id:
201 INFO("Could not find river in database. Skipping: %s"
202 % river_name)
170 continue 203 continue
171 204 else:
172 INFO("Start import of '%s'" % importer.getName()) 205 INFO("Importing River: %s" % river_name)
173 206
174 shapes = utils.findShapefiles(importer.getPath(config.directory)) 207 for importer in getImporters(river_id, dbconn, config.dry_run):
175 DEBUG("Found %i Shapefiles" % len(shapes)) 208 if skip_importer(config, importer):
176 209 INFO("Skip import of '%s'" % importer.getName())
177 for shpTuple in shapes: 210 continue
178 geomType = importer.walkOverShapes(shpTuple) 211
179 try: 212 INFO("Start import of '%s'" % importer.getName())
180 if geomType is not None: 213
181 num = types[geomType] 214 shapes = utils.findShapefiles(importer.getPath(config.directory))
182 types[geomType] = num+1 215 DEBUG("Found %i Shapefiles" % len(shapes))
183 except: 216
184 types[geomType] = 1 217 for shpTuple in shapes:
218 geomType = importer.walkOverShapes(shpTuple)
219 try:
220 if geomType is not None:
221 num = types[geomType]
222 types[geomType] = num+1
223 except:
224 types[geomType] = 1
185 225
186 for key in types: 226 for key in types:
187 DEBUG("%i x geometry type %s" % (types[key], key)) 227 DEBUG("%i x geometry type %s" % (types[key], key))
188 228
189 229
http://dive4elements.wald.intevation.org