Mercurial > dive4elements > river

comparison flys-backend/contrib/shpimporter/shpimporter.py @ 5128:a020100ee6a1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SCHEME CHANGE: Merge branch dami into default. A summary on the scheme changes: HWS and Lines tables are dropped and will be replaced by HWS_Lines and HWS_Points. The catchment table removed and will be replaced by a WMS Service. Hydr_boundaries has an added reference to boundary_kind sectie_kind and sobek_kind objects. Dem has a new column srid.
author Andre Heinecke <aheinecke@intevation.de>
date Thu, 28 Feb 2013 11:48:17 +0100
parents c5187ab9f571
children 40dc001594e4
comparison
equal deleted inserted replaced
5126:e37b25628dd4 5128:a020100ee6a1
1 try: 1 try:
2 from osgeo import ogr 2 from osgeo import ogr
3 except ImportErrror: 3 except ImportError:
4 import ogr 4 import ogr
5 5
6 import utils, optparse 6 import utils, optparse
7 import sys
8 import os
9 import logging
7 10
8 from uesg import UESG 11 from uesg import UESG
9 from axis import Axis 12 from axis import Axis
10 from km import KM 13 from km import KM
11 from lines import Line
12 from fixpoints import Fixpoint 14 from fixpoints import Fixpoint
13 from buildings import Building 15 from buildings import Building
14 from crosssectiontracks import CrosssectionTrack 16 from crosssectiontracks import CrosssectionTrack
15 from floodplains import Floodplain 17 from floodplains import Floodplain
16 from boundaries import HydrBoundary, HydrBoundaryPoly 18 from boundaries import HydrBoundary, HydrBoundaryPoly
17 from hws import HWSLines, HWSPoints 19 from hws import HWSLines, HWSPoints
18 from gauges import GaugeLocation 20 from gauges import GaugeLocation
19 from catchments import Catchment 21 from dgm import insertRiverDgm
20 22
21 23 logger = logging.getLogger("shpimporter")
22 VERBOSE_DEBUG=2 24
23 VERBOSE_INFO=1 25 def initialize_logging(level):
24 26 """Initializes the logging system"""
25 27 root = logging.getLogger()
26 def DEBUG(msg): 28 root.setLevel(level)
27 config = getConfig() 29 hdlr = logging.StreamHandler()
28 if config.verbose >= VERBOSE_DEBUG: 30 fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
29 print "DEBUG: %s" % msg 31 hdlr.setFormatter(fmt)
30 32 root.addHandler(hdlr)
31 def INFO(msg): 33
32 config = getConfig() 34 def getImporters(river_id, dbconn, dry_run):
33 if config.verbose >= VERBOSE_INFO:
34 print "INFO: %s" % msg
35
36 def ERROR(msg):
37 config = getConfig()
38 print "ERROR: %s" % msg
39
40
41 def getImporters(config, dbconn):
42 return [ 35 return [
43 Axis(config, dbconn), 36 Axis(river_id, dbconn, dry_run),
44 KM(config, dbconn), 37 KM(river_id, dbconn, dry_run),
45 CrosssectionTrack(config, dbconn), 38 CrosssectionTrack(river_id, dbconn, dry_run),
46 Line(config, dbconn), 39 Fixpoint(river_id, dbconn, dry_run),
47 Fixpoint(config, dbconn), 40 Building(river_id, dbconn, dry_run),
48 Building(config, dbconn), 41 Floodplain(river_id, dbconn, dry_run),
49 Floodplain(config, dbconn), 42 HydrBoundary(river_id, dbconn, dry_run),
50 HydrBoundary(config, dbconn), 43 HydrBoundaryPoly(river_id, dbconn, dry_run),
51 HydrBoundaryPoly(config, dbconn), 44 HWSLines(river_id, dbconn, dry_run),
52 HWSLines(config, dbconn), 45 HWSPoints(river_id, dbconn, dry_run),
53 HWSPoints(config, dbconn), 46 GaugeLocation(river_id, dbconn, dry_run),
54 GaugeLocation(config, dbconn), 47 UESG(river_id, dbconn, dry_run)
55 Catchment(config, dbconn),
56 UESG(config, dbconn)
57 ] 48 ]
58 49
59 50
60 def getConfig(): 51 def getConfig():
61 parser = optparse.OptionParser() 52 parser = optparse.OptionParser()
62 parser.add_option("--directory", type="string") 53 parser.add_option("--directory", type="string")
63 parser.add_option("--target_srs", type="int") 54 parser.add_option("--target_srs", type="int")
64 parser.add_option("--host", type="string") 55 parser.add_option("--host", type="string")
65 parser.add_option("--user", type="string") 56 parser.add_option("--user", type="string")
66 parser.add_option("--password", type="string") 57 parser.add_option("--password", type="string")
67 parser.add_option("--river_id", type="int") 58 parser.add_option("--river_name", type="string")
68 parser.add_option("--verbose", type="int", default=1) 59 parser.add_option("--verbose", type="int", default=1)
69 parser.add_option("--dry_run", type="int", default=0) 60 parser.add_option("--dry_run", type="int", default=0)
70 parser.add_option("--ogr_connection", type="string") 61 parser.add_option("--ogr_connection", type="string")
71 parser.add_option("--skip_axis", type="int") 62 parser.add_option("--skip_axis", type="int")
72 parser.add_option("--skip_hydr_boundaries", type="int") 63 parser.add_option("--skip_hydr_boundaries", type="int")
73 parser.add_option("--skip_buildings", type="int") 64 parser.add_option("--skip_buildings", type="int")
74 parser.add_option("--skip_crosssections", type="int") 65 parser.add_option("--skip_crosssections", type="int")
75 parser.add_option("--skip_lines", type="int")
76 parser.add_option("--skip_fixpoints", type="int") 66 parser.add_option("--skip_fixpoints", type="int")
77 parser.add_option("--skip_floodplains", type="int") 67 parser.add_option("--skip_floodplains", type="int")
78 parser.add_option("--skip_hws_lines", type="int") 68 parser.add_option("--skip_hws_lines", type="int")
79 parser.add_option("--skip_hws_points", type="int") 69 parser.add_option("--skip_hws_points", type="int")
80 parser.add_option("--skip_gauge_locations", type="int") 70 parser.add_option("--skip_gauge_locations", type="int")
81 parser.add_option("--skip_catchments", type="int")
82 parser.add_option("--skip_kms", type="int") 71 parser.add_option("--skip_kms", type="int")
83 parser.add_option("--skip_uesgs", type="int") 72 parser.add_option("--skip_uesgs", type="int")
73 parser.add_option("--skip_dgm", type="int")
84 (config, args) = parser.parse_args() 74 (config, args) = parser.parse_args()
85 75
76 if config.verbose > 1:
77 initialize_logging(logging.DEBUG)
78 elif config.verbose == 1:
79 initialize_logging(logging.INFO)
80 else:
81 initialize_logging(logging.WARN)
82
86 if config.directory == None: 83 if config.directory == None:
87 ERROR("No river directory specified!") 84 logger.error("No river directory specified!")
88 raise Exception("Invalid config") 85 raise Exception("Invalid config")
89 if not config.ogr_connection: 86 if not config.ogr_connection:
90 if not config.host: 87 if not config.host:
91 ERROR("No database host specified!") 88 logger.error("No database host specified!")
92 raise Exception("Invalid config") 89 raise Exception("Invalid config")
93 if not config.user: 90 if not config.user:
94 ERROR("No databaser user specified!") 91 logger.error("No databaser user specified!")
95 raise Exception("Invalid config") 92 raise Exception("Invalid config")
96 if not config.password: 93 if not config.password:
97 ERROR("No password specified!") 94 logger.error("No password specified!")
98 raise Exception("Invalid config") 95 raise Exception("Invalid config")
99 if config.river_id == None:
100 ERROR("No river id specified!")
101 raise Exception("Invalid config")
102 96
103 return config 97 return config
104 98
105 99
106 def skip_importer(config, importer): 100 def skip_importer(config, importer):
112 return True 106 return True
113 elif config.skip_buildings == 1 and isinstance(importer, Building): 107 elif config.skip_buildings == 1 and isinstance(importer, Building):
114 return True 108 return True
115 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack): 109 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack):
116 return True 110 return True
117 elif config.skip_lines == 1 and isinstance(importer, Line):
118 return True
119 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint): 111 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint):
120 return True 112 return True
121 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain): 113 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain):
122 return True 114 return True
123 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints):
124 return True
125 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines): 115 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines):
126 return True 116 return True
117 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints) and \
118 not isinstance(importer, HWSLines):
119 return True
127 elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation): 120 elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation):
128 return True 121 return True
129 elif config.skip_catchments == 1 and isinstance(importer, Catchment):
130 return True
131 elif config.skip_kms == 1 and isinstance(importer, KM): 122 elif config.skip_kms == 1 and isinstance(importer, KM):
132 return True 123 return True
133 elif config.skip_uesgs == 1 and isinstance(importer, UESG): 124 elif config.skip_uesgs == 1 and isinstance(importer, UESG):
134 return True 125 return True
135 126
136 return False 127 return False
137
138 128
139 def main(): 129 def main():
140 config=None 130 config=None
141 try: 131 try:
142 config = getConfig() 132 config = getConfig()
143 except: 133 except:
144 return -1 134 return -1
145 135
146 if config == None: 136 if config == None:
147 ERROR("Unable to read config from command line!") 137 logger.error("Unable to read config from command line!")
148 return 138 return
149 139
150 if config.dry_run > 0: 140 if config.dry_run > 0:
151 INFO("You enable 'dry_run'. No database transaction will take place!") 141 logger.info("You enable 'dry_run'. No database transaction will take place!")
152 142
153 if config.ogr_connection: 143 if config.ogr_connection:
154 connstr = config.ogr_connection 144 connstr = config.ogr_connection
155 else: 145 else:
156 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host) 146 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
157 147
148 oracle = False # Marker if oracle is used.
149 if 'OCI:' in connstr:
150 oracle = True
151 try:
152 import cx_Oracle as dbapi
153 raw_connstr=connstr.replace("OCI:", "")
154 except ImportError:
155 logger.error("Module cx_Oracle not found in: %s\n"
156 "Neccessary to connect to a Oracle Database.\n"
157 "Please refer to the installation "
158 "documentation." % sys.path)
159 return -1
160
161 else: # Currently only support for oracle and postgres
162 try:
163 import psycopg2 as dbapi
164 raw_connstr=connstr.replace("PG:", "")
165 except ImportError:
166 logger.error("Module psycopg2 not found in: %s\n"
167 "Neccessary to connect to a Posgresql Database.\n"
168 "Please refer to the installation "
169 "documentation." % sys.path)
170 return -1
171
172 dbconn_raw = dbapi.connect(raw_connstr)
158 dbconn = ogr.Open(connstr) 173 dbconn = ogr.Open(connstr)
159 174
160 if dbconn == None: 175 if dbconn == None:
161 ERROR("Could not connect to database %s" % connstr) 176 logger.error("Could not connect to database %s" % connstr)
162 return -1 177 return -1
163 178
164 importers = getImporters(config, dbconn)
165 types = {} 179 types = {}
166 180
167 for importer in importers: 181 directories = []
168 if skip_importer(config, importer): 182 if not config.river_name:
169 INFO("Skip import of '%s'" % importer.getName()) 183 for file in [os.path.join(config.directory, d) for d in \
184 os.listdir(config.directory)]:
185 if os.path.isdir(file):
186 directories.append(file)
187 else:
188 directories.append(config.directory)
189
190 for directory in directories:
191 if not config.river_name:
192 river_name = utils.getUTF8Path(
193 os.path.basename(os.path.normpath(directory)))
194 else:
195 river_name = config.river_name
196 river_id = utils.getRiverId(dbconn_raw, river_name, oracle)
197
198 if not river_id:
199 logger.info("Could not find river in database. Skipping: %s"
200 % river_name)
170 continue 201 continue
171 202 else:
172 INFO("Start import of '%s'" % importer.getName()) 203 logger.info("Importing River: %s" % river_name)
173 204
174 shapes = utils.findShapefiles(importer.getPath(config.directory)) 205 for importer in getImporters(river_id, dbconn, config.dry_run):
175 DEBUG("Found %i Shapefiles" % len(shapes)) 206 if skip_importer(config, importer):
176 207 logger.info("Skip import of '%s'" % importer.getName())
177 for shpTuple in shapes: 208 continue
178 geomType = importer.walkOverShapes(shpTuple) 209
179 try: 210 logger.info("Start import of '%s'" % importer.getName())
180 if geomType is not None: 211
181 num = types[geomType] 212 shapes = utils.findShapefiles(importer.getPath(config.directory))
182 types[geomType] = num+1 213 logger.debug("Found %i Shapefiles" % len(shapes))
183 except: 214
184 types[geomType] = 1 215 for shpTuple in shapes:
185 216 geomType = importer.walkOverShapes(shpTuple)
186 for key in types: 217 try:
187 DEBUG("%i x geometry type %s" % (types[key], key)) 218 if geomType is not None:
188 219 num = types[geomType]
220 types[geomType] = num+1
221 except:
222 types[geomType] = 1
223
224 for key in types:
225 logger.debug("%i x geometry type %s" % (types[key], key))
226
227 if not config.skip_dgm:
228 dgmfilename = os.path.join(
229 config.directory, "..", "DGMs.csv")
230 if not os.access(dgmfilename, os.R_OK) or not \
231 os.path.isfile(dgmfilename):
232 logger.info("Could not find or access DGM file: %s \n"
233 "Skipping DGM import." % dgmfilename)
234 else:
235 logger.info("Inserting DGM meta information in 'dem' table.")
236 insertRiverDgm(dbconn_raw, dgmfilename, river_name,
237 config.dry_run, oracle)
238 else:
239 logger.info("Skip import of DGM.")
189 240
190 if __name__ == '__main__': 241 if __name__ == '__main__':
191 main() 242 main()
http://dive4elements.wald.intevation.org