Mercurial > dive4elements > river

comparison flys-backend/contrib/shpimporter/shpimporter.py @ 5379:61bf64b102bc mapgenfix

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge with default branch
author Christian Lins <christian.lins@intevation.de>
date Fri, 22 Mar 2013 11:25:54 +0100
parents d086ce6c13d3
children ed149d5d7fb7
comparison
equal deleted inserted replaced
5175:cfc5540a4eec 5379:61bf64b102bc
1 try: 1 try:
2 from osgeo import ogr 2 from osgeo import ogr
3 except ImportErrror: 3 except ImportError:
4 import ogr 4 import ogr
5 5
6 import utils, optparse 6 import utils, optparse
7 import sys
8 import os
9 import logging
7 10
8 from uesg import UESG 11 from uesg import UESG
9 from axis import Axis 12 from axis import Axis
10 from km import KM 13 from km import KM
11 from lines import Line
12 from fixpoints import Fixpoint 14 from fixpoints import Fixpoint
13 from buildings import Building 15 from buildings import Building
14 from crosssectiontracks import CrosssectionTrack 16 from crosssectiontracks import CrosssectionTrack
15 from floodplains import Floodplain 17 from floodplains import Floodplain
16 from boundaries import HydrBoundary, HydrBoundaryPoly 18 from boundaries import HydrBoundary, HydrBoundaryPoly
17 from hws import HWSLines, HWSPoints 19 from hws import HWSLines, HWSPoints
18 from gauges import GaugeLocation 20 from gauges import GaugeLocation
19 from catchments import Catchment 21 from jetties import Jetties
20 22 from dgm import insertRiverDgm
21 23
22 VERBOSE_DEBUG=2 24 logger = logging.getLogger("shpimporter")
23 VERBOSE_INFO=1 25
24 26 os.environ["NLS_LANG"] = ".AL32UTF8"
25 27
26 def DEBUG(msg): 28 def initialize_logging(level):
27 config = getConfig() 29 """Initializes the logging system"""
28 if config.verbose >= VERBOSE_DEBUG: 30 root = logging.getLogger()
29 print "DEBUG: %s" % msg 31 root.setLevel(level)
30 32 hdlr = logging.StreamHandler()
31 def INFO(msg): 33 fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
32 config = getConfig() 34 hdlr.setFormatter(fmt)
33 if config.verbose >= VERBOSE_INFO: 35 root.addHandler(hdlr)
34 print "INFO: %s" % msg 36
35 37 def getImporters(river_id, dbconn, dry_run):
36 def ERROR(msg):
37 config = getConfig()
38 print "ERROR: %s" % msg
39
40
41 def getImporters(config, dbconn):
42 return [ 38 return [
43 Axis(config, dbconn), 39 Axis(river_id, dbconn, dry_run),
44 KM(config, dbconn), 40 KM(river_id, dbconn, dry_run),
45 CrosssectionTrack(config, dbconn), 41 CrosssectionTrack(river_id, dbconn, dry_run),
46 Line(config, dbconn), 42 Fixpoint(river_id, dbconn, dry_run),
47 Fixpoint(config, dbconn), 43 Building(river_id, dbconn, dry_run),
48 Building(config, dbconn), 44 Floodplain(river_id, dbconn, dry_run),
49 Floodplain(config, dbconn), 45 HydrBoundary(river_id, dbconn, dry_run),
50 HydrBoundary(config, dbconn), 46 HydrBoundaryPoly(river_id, dbconn, dry_run),
51 HydrBoundaryPoly(config, dbconn), 47 HWSLines(river_id, dbconn, dry_run),
52 HWSLines(config, dbconn), 48 HWSPoints(river_id, dbconn, dry_run),
53 HWSPoints(config, dbconn), 49 GaugeLocation(river_id, dbconn, dry_run),
54 GaugeLocation(config, dbconn), 50 Jetties(river_id, dbconn, dry_run),
55 Catchment(config, dbconn), 51 UESG(river_id, dbconn, dry_run)
56 UESG(config, dbconn)
57 ] 52 ]
58 53
59 54
60 def getConfig(): 55 def getConfig():
61 parser = optparse.OptionParser() 56 parser = optparse.OptionParser()
62 parser.add_option("--directory", type="string") 57 parser.add_option("--directory", type="string")
63 parser.add_option("--target_srs", type="int") 58 parser.add_option("--target_srs", type="int")
64 parser.add_option("--host", type="string") 59 parser.add_option("--host", type="string")
65 parser.add_option("--user", type="string") 60 parser.add_option("--user", type="string")
66 parser.add_option("--password", type="string") 61 parser.add_option("--password", type="string")
67 parser.add_option("--river_id", type="int") 62 parser.add_option("--river_name", type="string")
68 parser.add_option("--verbose", type="int", default=1) 63 parser.add_option("--verbose", type="int", default=1)
69 parser.add_option("--dry_run", type="int", default=0) 64 parser.add_option("--dry_run", type="int", default=0)
70 parser.add_option("--ogr_connection", type="string") 65 parser.add_option("--ogr_connection", type="string")
71 parser.add_option("--skip_axis", type="int") 66 parser.add_option("--skip_axis", type="int")
72 parser.add_option("--skip_hydr_boundaries", type="int") 67 parser.add_option("--skip_hydr_boundaries", type="int")
73 parser.add_option("--skip_buildings", type="int") 68 parser.add_option("--skip_buildings", type="int")
74 parser.add_option("--skip_crosssections", type="int") 69 parser.add_option("--skip_crosssections", type="int")
75 parser.add_option("--skip_lines", type="int")
76 parser.add_option("--skip_fixpoints", type="int") 70 parser.add_option("--skip_fixpoints", type="int")
77 parser.add_option("--skip_floodplains", type="int") 71 parser.add_option("--skip_floodplains", type="int")
78 parser.add_option("--skip_hws_lines", type="int") 72 parser.add_option("--skip_hws_lines", type="int")
79 parser.add_option("--skip_hws_points", type="int") 73 parser.add_option("--skip_hws_points", type="int")
80 parser.add_option("--skip_gauge_locations", type="int") 74 parser.add_option("--skip_gauge_locations", type="int")
81 parser.add_option("--skip_catchments", type="int")
82 parser.add_option("--skip_kms", type="int") 75 parser.add_option("--skip_kms", type="int")
83 parser.add_option("--skip_uesgs", type="int") 76 parser.add_option("--skip_uesgs", type="int")
77 parser.add_option("--skip_dgm", type="int")
78 parser.add_option("--skip_jetties", type="int")
84 (config, args) = parser.parse_args() 79 (config, args) = parser.parse_args()
85 80
81 if config.verbose > 1:
82 initialize_logging(logging.DEBUG)
83 elif config.verbose == 1:
84 initialize_logging(logging.INFO)
85 else:
86 initialize_logging(logging.WARN)
87
86 if config.directory == None: 88 if config.directory == None:
87 ERROR("No river directory specified!") 89 logger.error("No river directory specified!")
88 raise Exception("Invalid config") 90 raise Exception("Invalid config")
89 if not config.ogr_connection: 91 if not config.ogr_connection:
90 if not config.host: 92 if not config.host:
91 ERROR("No database host specified!") 93 logger.error("No database host specified!")
92 raise Exception("Invalid config") 94 raise Exception("Invalid config")
93 if not config.user: 95 if not config.user:
94 ERROR("No databaser user specified!") 96 logger.error("No databaser user specified!")
95 raise Exception("Invalid config") 97 raise Exception("Invalid config")
96 if not config.password: 98 if not config.password:
97 ERROR("No password specified!") 99 logger.error("No password specified!")
98 raise Exception("Invalid config") 100 raise Exception("Invalid config")
99 if config.river_id == None:
100 ERROR("No river id specified!")
101 raise Exception("Invalid config")
102 101
103 return config 102 return config
104 103
105 104
106 def skip_importer(config, importer): 105 def skip_importer(config, importer):
112 return True 111 return True
113 elif config.skip_buildings == 1 and isinstance(importer, Building): 112 elif config.skip_buildings == 1 and isinstance(importer, Building):
114 return True 113 return True
115 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack): 114 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack):
116 return True 115 return True
117 elif config.skip_lines == 1 and isinstance(importer, Line):
118 return True
119 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint): 116 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint):
120 return True 117 return True
121 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain): 118 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain):
122 return True 119 return True
123 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints):
124 return True
125 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines): 120 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines):
126 return True 121 return True
122 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints) and \
123 not isinstance(importer, HWSLines):
124 return True
127 elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation): 125 elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation):
128 return True 126 return True
129 elif config.skip_catchments == 1 and isinstance(importer, Catchment): 127 elif config.skip_jetties == 1 and isinstance(importer, Jetties):
130 return True 128 return True
131 elif config.skip_kms == 1 and isinstance(importer, KM): 129 elif config.skip_kms == 1 and isinstance(importer, KM):
132 return True 130 return True
133 elif config.skip_uesgs == 1 and isinstance(importer, UESG): 131 elif config.skip_uesgs == 1 and isinstance(importer, UESG):
134 return True 132 return True
135 133
136 return False 134 return False
137
138 135
139 def main(): 136 def main():
140 config=None 137 config=None
141 try: 138 try:
142 config = getConfig() 139 config = getConfig()
143 except: 140 except:
144 return -1 141 return -1
145 142
146 if config == None: 143 if config == None:
147 ERROR("Unable to read config from command line!") 144 logger.error("Unable to read config from command line!")
148 return 145 return
149 146
150 if config.dry_run > 0: 147 if config.dry_run > 0:
151 INFO("You enable 'dry_run'. No database transaction will take place!") 148 logger.info("You enable 'dry_run'. No database transaction will take place!")
152 149
153 if config.ogr_connection: 150 if config.ogr_connection:
154 connstr = config.ogr_connection 151 connstr = config.ogr_connection
155 else: 152 else:
156 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host) 153 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
157 154
155 oracle = False # Marker if oracle is used.
156 if 'OCI:' in connstr:
157 oracle = True
158 try:
159 import cx_Oracle as dbapi
160 raw_connstr=connstr.replace("OCI:", "")
161 except ImportError:
162 logger.error("Module cx_Oracle not found in: %s\n"
163 "Neccessary to connect to a Oracle Database.\n"
164 "Please refer to the installation "
165 "documentation." % sys.path)
166 return -1
167
168 else: # Currently only support for oracle and postgres
169 try:
170 import psycopg2 as dbapi
171 raw_connstr=connstr.replace("PG:", "")
172 except ImportError:
173 logger.error("Module psycopg2 not found in: %s\n"
174 "Neccessary to connect to a Posgresql Database.\n"
175 "Please refer to the installation "
176 "documentation." % sys.path)
177 return -1
178
179 dbconn_raw = dbapi.connect(raw_connstr)
158 dbconn = ogr.Open(connstr) 180 dbconn = ogr.Open(connstr)
159 181
160 if dbconn == None: 182 if dbconn == None:
161 ERROR("Could not connect to database %s" % connstr) 183 logger.error("Could not connect to database %s" % connstr)
162 return -1 184 return -1
163 185
164 importers = getImporters(config, dbconn)
165 types = {} 186 types = {}
166 187
167 for importer in importers: 188 directories = []
168 if skip_importer(config, importer): 189 if not config.river_name:
169 INFO("Skip import of '%s'" % importer.getName()) 190 for file in [os.path.join(config.directory, d) for d in \
191 os.listdir(config.directory)]:
192 if os.path.isdir(file):
193 directories.append(file)
194 else:
195 directories.append(config.directory)
196
197 for directory in directories:
198 if not config.river_name:
199 river_name = utils.getUTF8Path(
200 os.path.basename(os.path.normpath(directory)))
201 else:
202 river_name = config.river_name
203 river_id = utils.getRiverId(dbconn_raw, river_name, oracle)
204
205 if not river_id:
206 logger.info(u"Could not find river in database. Skipping: %s"
207 % unicode(utils.getUTF8(river_name), "UTF-8"))
170 continue 208 continue
171 209 else:
172 INFO("Start import of '%s'" % importer.getName()) 210 logger.info(u"Importing River: %s" % unicode(
173 211 utils.getUTF8(river_name), "UTF-8"))
174 shapes = utils.findShapefiles(importer.getPath(config.directory)) 212
175 DEBUG("Found %i Shapefiles" % len(shapes)) 213 for importer in getImporters(river_id, dbconn, config.dry_run):
176 214 if skip_importer(config, importer):
177 for shpTuple in shapes: 215 logger.info("Skip import of '%s'" % importer.getName())
178 geomType = importer.walkOverShapes(shpTuple) 216 continue
179 try: 217
180 if geomType is not None: 218 logger.info("Start import of '%s'" % importer.getName())
181 num = types[geomType] 219
182 types[geomType] = num+1 220 shapes = utils.findShapefiles(importer.getPath(config.directory))
183 except: 221 logger.debug("Found %i Shapefiles" % len(shapes))
184 types[geomType] = 1 222
185 223 for shpTuple in shapes:
186 for key in types: 224 geomType = importer.walkOverShapes(shpTuple)
187 DEBUG("%i x geometry type %s" % (types[key], key)) 225 try:
188 226 if geomType is not None:
227 num = types[geomType]
228 types[geomType] = num+1
229 except:
230 types[geomType] = 1
231
232 for key in types:
233 logger.debug("%i x geometry type %s" % (types[key], key))
234
235 if not config.skip_dgm:
236 dgmfilename = os.path.join(
237 config.directory, "..", "DGMs.csv")
238 if not os.access(dgmfilename, os.R_OK) or not \
239 os.path.isfile(dgmfilename):
240 logger.info("Could not find or access DGM file: %s \n"
241 "Skipping DGM import." % dgmfilename)
242 else:
243 logger.info("Inserting DGM meta information in 'dem' table.")
244 insertRiverDgm(dbconn_raw, dgmfilename, river_name,
245 config.dry_run, oracle)
246 else:
247 logger.info("Skip import of DGM.")
189 248
190 if __name__ == '__main__': 249 if __name__ == '__main__':
191 main() 250 main()
http://dive4elements.wald.intevation.org