Mercurial > dive4elements > river

comparison backend/contrib/shpimporter/shpimporter.py @ 5838:5aa05a7a34b7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Rename modules to more fitting names.
author Sascha L. Teichmann <teichmann@intevation.de>
date Thu, 25 Apr 2013 15:23:37 +0200
parents flys-backend/contrib/shpimporter/shpimporter.py@ed149d5d7fb7
children e68e414dceb2
comparison
equal deleted inserted replaced
5837:d9901a08d0a6 5838:5aa05a7a34b7
1 try:
2 from osgeo import ogr
3 except ImportError:
4 import ogr
5
6 import utils, optparse
7 import sys
8 import os
9 import logging
10
11 from uesg import UESG
12 from axis import Axis
13 from km import KM
14 from fixpoints import Fixpoint
15 from buildings import Building
16 from crosssectiontracks import CrosssectionTrack
17 from floodplains import Floodplain
18 from boundaries import HydrBoundary, HydrBoundaryPoly
19 from hws import HWSLines, HWSPoints
20 from gauges import GaugeLocation
21 from jetties import Jetties
22 from dgm import insertRiverDgm
23 from floodmarks import Floodmark
24
25 logger = logging.getLogger("shpimporter")
26
27 os.environ["NLS_LANG"] = ".AL32UTF8"
28
29 def initialize_logging(level):
30 """Initializes the logging system"""
31 root = logging.getLogger()
32 root.setLevel(level)
33 hdlr = logging.StreamHandler()
34 fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
35 hdlr.setFormatter(fmt)
36 root.addHandler(hdlr)
37
38 def getImporters(river_id, dbconn, dry_run):
39 return [
40 Axis(river_id, dbconn, dry_run),
41 KM(river_id, dbconn, dry_run),
42 CrosssectionTrack(river_id, dbconn, dry_run),
43 Fixpoint(river_id, dbconn, dry_run),
44 Building(river_id, dbconn, dry_run),
45 Floodplain(river_id, dbconn, dry_run),
46 HydrBoundary(river_id, dbconn, dry_run),
47 HydrBoundaryPoly(river_id, dbconn, dry_run),
48 HWSLines(river_id, dbconn, dry_run),
49 HWSPoints(river_id, dbconn, dry_run),
50 GaugeLocation(river_id, dbconn, dry_run),
51 Jetties(river_id, dbconn, dry_run),
52 Floodmark(river_id, dbconn, dry_run),
53 UESG(river_id, dbconn, dry_run)
54 ]
55
56
57 def getConfig():
58 parser = optparse.OptionParser()
59 parser.add_option("--directory", type="string")
60 parser.add_option("--target_srs", type="int")
61 parser.add_option("--host", type="string")
62 parser.add_option("--user", type="string")
63 parser.add_option("--password", type="string")
64 parser.add_option("--river_name", type="string")
65 parser.add_option("--verbose", type="int", default=1)
66 parser.add_option("--dry_run", type="int", default=0)
67 parser.add_option("--ogr_connection", type="string")
68 parser.add_option("--skip_axis", type="int")
69 parser.add_option("--skip_hydr_boundaries", type="int")
70 parser.add_option("--skip_buildings", type="int")
71 parser.add_option("--skip_crosssections", type="int")
72 parser.add_option("--skip_fixpoints", type="int")
73 parser.add_option("--skip_floodplains", type="int")
74 parser.add_option("--skip_hws_lines", type="int")
75 parser.add_option("--skip_hws_points", type="int")
76 parser.add_option("--skip_gauge_locations", type="int")
77 parser.add_option("--skip_kms", type="int")
78 parser.add_option("--skip_uesgs", type="int")
79 parser.add_option("--skip_dgm", type="int")
80 parser.add_option("--skip_jetties", type="int")
81 parser.add_option("--skip_floodmarks", type="int")
82 (config, args) = parser.parse_args()
83
84 if config.verbose > 1:
85 initialize_logging(logging.DEBUG)
86 elif config.verbose == 1:
87 initialize_logging(logging.INFO)
88 else:
89 initialize_logging(logging.WARN)
90
91 if config.directory == None:
92 logger.error("No river directory specified!")
93 raise Exception("Invalid config")
94 if not config.ogr_connection:
95 if not config.host:
96 logger.error("No database host specified!")
97 raise Exception("Invalid config")
98 if not config.user:
99 logger.error("No databaser user specified!")
100 raise Exception("Invalid config")
101 if not config.password:
102 logger.error("No password specified!")
103 raise Exception("Invalid config")
104
105 return config
106
107
108 def skip_importer(config, importer):
109 if config.skip_axis == 1 and isinstance(importer, Axis):
110 return True
111 elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundary):
112 return True
113 elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundaryPoly):
114 return True
115 elif config.skip_buildings == 1 and isinstance(importer, Building):
116 return True
117 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack):
118 return True
119 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint):
120 return True
121 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain):
122 return True
123 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines):
124 return True
125 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints) and \
126 not isinstance(importer, HWSLines):
127 return True
128 elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation):
129 return True
130 elif config.skip_jetties == 1 and isinstance(importer, Jetties):
131 return True
132 elif config.skip_kms == 1 and isinstance(importer, KM):
133 return True
134 elif config.skip_uesgs == 1 and isinstance(importer, UESG):
135 return True
136 elif config.skip_floodmarks == 1 and isinstance(importer, Floodmark):
137 return True
138
139 return False
140
141 def main():
142 config=None
143 try:
144 config = getConfig()
145 except:
146 return -1
147
148 if config == None:
149 logger.error("Unable to read config from command line!")
150 return
151
152 if config.dry_run > 0:
153 logger.info("You enable 'dry_run'. No database transaction will take place!")
154
155 if config.ogr_connection:
156 connstr = config.ogr_connection
157 else:
158 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
159
160 oracle = False # Marker if oracle is used.
161 if 'OCI:' in connstr:
162 oracle = True
163 try:
164 import cx_Oracle as dbapi
165 raw_connstr=connstr.replace("OCI:", "")
166 except ImportError:
167 logger.error("Module cx_Oracle not found in: %s\n"
168 "Neccessary to connect to a Oracle Database.\n"
169 "Please refer to the installation "
170 "documentation." % sys.path)
171 return -1
172
173 else: # Currently only support for oracle and postgres
174 try:
175 import psycopg2 as dbapi
176 raw_connstr=connstr.replace("PG:", "")
177 except ImportError:
178 logger.error("Module psycopg2 not found in: %s\n"
179 "Neccessary to connect to a Posgresql Database.\n"
180 "Please refer to the installation "
181 "documentation." % sys.path)
182 return -1
183
184 dbconn_raw = dbapi.connect(raw_connstr)
185 dbconn = ogr.Open(connstr)
186
187 if dbconn == None:
188 logger.error("Could not connect to database %s" % connstr)
189 return -1
190
191 types = {}
192
193 directories = []
194 if not config.river_name:
195 for file in [os.path.join(config.directory, d) for d in \
196 os.listdir(config.directory)]:
197 if os.path.isdir(file):
198 directories.append(file)
199 else:
200 directories.append(config.directory)
201
202 for directory in directories:
203 if not config.river_name:
204 river_name = utils.getUTF8Path(
205 os.path.basename(os.path.normpath(directory)))
206 else:
207 river_name = config.river_name
208 river_id = utils.getRiverId(dbconn_raw, river_name, oracle)
209
210 if not river_id:
211 logger.info(u"Could not find river in database. Skipping: %s"
212 % unicode(utils.getUTF8(river_name), "UTF-8"))
213 continue
214 else:
215 logger.info(u"Importing River: %s" % unicode(
216 utils.getUTF8(river_name), "UTF-8"))
217
218 for importer in getImporters(river_id, dbconn, config.dry_run):
219 if skip_importer(config, importer):
220 logger.info("Skip import of '%s'" % importer.getName())
221 continue
222
223 logger.info("Start import of '%s'" % importer.getName())
224
225 shapes = utils.findShapefiles(importer.getPath(config.directory))
226 logger.debug("Found %i Shapefiles" % len(shapes))
227
228 for shpTuple in shapes:
229 geomType = importer.walkOverShapes(shpTuple)
230 try:
231 if geomType is not None:
232 num = types[geomType]
233 types[geomType] = num+1
234 except:
235 types[geomType] = 1
236
237 for key in types:
238 logger.debug("%i x geometry type %s" % (types[key], key))
239
240 if not config.skip_dgm:
241 dgmfilename = os.path.join(
242 config.directory, "..", "DGMs.csv")
243 if not os.access(dgmfilename, os.R_OK) or not \
244 os.path.isfile(dgmfilename):
245 logger.info("Could not find or access DGM file: %s \n"
246 "Skipping DGM import." % dgmfilename)
247 else:
248 logger.info("Inserting DGM meta information in 'dem' table.")
249 insertRiverDgm(dbconn_raw, dgmfilename, river_name,
250 config.dry_run, oracle)
251 else:
252 logger.info("Skip import of DGM.")
253
254 if __name__ == '__main__':
255 main()
http://dive4elements.wald.intevation.org