Mercurial > dive4elements > river

comparison flys-backend/contrib/shpimporter/importer.py @ 5006:769593a84606 dami

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Importer: Behold, Logging!
author Andre Heinecke <aheinecke@intevation.de>
date Fri, 15 Feb 2013 16:22:13 +0100
parents f1c01fecf194
children c5187ab9f571
comparison
equal deleted inserted replaced
5005:eb2d6609387c 5006:769593a84606
2 from osgeo import ogr, osr 2 from osgeo import ogr, osr
3 except ImportErrror: 3 except ImportErrror:
4 import ogr, osr 4 import ogr, osr
5 import utils 5 import utils
6 import re 6 import re
7 import logging
8
9 logger = logging.getLogger("importer")
7 10
8 class Importer: 11 class Importer:
9 12
10 def __init__(self, river_id, dbconn, dry_run): 13 def __init__(self, river_id, dbconn, dry_run):
11 self.river_id = river_id 14 self.river_id = river_id
76 def walkOverShapes(self, shape): 79 def walkOverShapes(self, shape):
77 (name, path) = shape 80 (name, path) = shape
78 81
79 shp = ogr.Open(shape[1]) 82 shp = ogr.Open(shape[1])
80 if shp is None: 83 if shp is None:
81 shpimporter.ERROR("Shapefile '%s' could not be opened!" % path) 84 logger.error("Shapefile '%s' could not be opened!" % path)
82 return 85 return
83 86
84 if not self.isShapeRelevant(name, path): 87 if not self.isShapeRelevant(name, path):
85 shpimporter.INFO("Skip shapefile: '%s' of Type: %s" % (path, 88 logger.info("Skip shapefile: '%s' of Type: %s" % (path,
86 utils.getWkbString(shp.GetLayerByName(name).GetGeomType()))) 89 utils.getWkbString(shp.GetLayerByName(name).GetGeomType())))
87 return 90 return
88 91
89 92
90 shpimporter.INFO("Processing shapefile '%s'" % path) 93 logger.info("Processing shapefile '%s'" % path)
91 srcLayer = shp.GetLayerByName(name) 94 srcLayer = shp.GetLayerByName(name)
92 95
93 if srcLayer is None: 96 if srcLayer is None:
94 shpimporter.ERROR("Layer '%s' was not found!" % name) 97 logger.error("Layer '%s' was not found!" % name)
95 return 98 return
96 99
97 return self.shape2Database(srcLayer, name, path) 100 return self.shape2Database(srcLayer, name, path)
98 101
99 def transform(self, feat): 102 def transform(self, feat):
100 geometry = feat.GetGeometryRef() 103 geometry = feat.GetGeometryRef()
101 src_srs = geometry.GetSpatialReference() 104 src_srs = geometry.GetSpatialReference()
102 105
103 if src_srs is None: 106 if src_srs is None:
104 shpimporter.ERROR("No source SRS given! No transformation possible!") 107 logger.error("No source SRS given! No transformation possible!")
105 return feat 108 return feat
106 109
107 transformer = osr.CoordinateTransformation(src_srs, self.dest_srs) 110 transformer = osr.CoordinateTransformation(src_srs, self.dest_srs)
108 if geometry.Transform(transformer): 111 if geometry.Transform(transformer):
109 return None 112 return None
156 159
157 def shape2Database(self, srcLayer, name, path): 160 def shape2Database(self, srcLayer, name, path):
158 destLayer = self.dbconn.GetLayerByName(self.getTablename()) 161 destLayer = self.dbconn.GetLayerByName(self.getTablename())
159 162
160 if srcLayer is None: 163 if srcLayer is None:
161 shpimporter.ERROR("Shapefile is None!") 164 logger.error("Shapefile is None!")
162 return -1 165 return -1
163 166
164 if destLayer is None: 167 if destLayer is None:
165 shpimporter.ERROR("No destination layer given!") 168 logger.error("No destination layer given!")
166 return -1 169 return -1
167 170
168 count = srcLayer.GetFeatureCount() 171 count = srcLayer.GetFeatureCount()
169 shpimporter.DEBUG("Try to add %i features to database." % count) 172 logger.debug("Try to add %i features to database." % count)
170 173
171 srcLayer.ResetReading() 174 srcLayer.ResetReading()
172 self.srcLayer = srcLayer 175 self.srcLayer = srcLayer
173 176
174 geomType = -1 177 geomType = -1
179 182
180 for feat in srcLayer: 183 for feat in srcLayer:
181 geom = feat.GetGeometryRef() 184 geom = feat.GetGeometryRef()
182 185
183 if geom is None: 186 if geom is None:
184 shpimporter.DEBUG("Unkown Geometry reference for feature") 187 logger.debug("Unkown Geometry reference for feature")
185 continue 188 continue
186 189
187 geomType = geom.GetGeometryType() 190 geomType = geom.GetGeometryType()
188 191
189 if self.isGeometryValid(geomType): 192 if self.isGeometryValid(geomType):
196 newFeat.SetField("path", utils.getUTF8Path(path)) 199 newFeat.SetField("path", utils.getUTF8Path(path))
197 newFeat = self.transform(newFeat) 200 newFeat = self.transform(newFeat)
198 if newFeat: 201 if newFeat:
199 res = destLayer.CreateFeature(newFeat) 202 res = destLayer.CreateFeature(newFeat)
200 if res is None or res > 0: 203 if res is None or res > 0:
201 shpimporter.ERROR("Unable to insert feature. Error: %r" % res) 204 logger.error("Unable to insert feature. Error: %r" % res)
202 else: 205 else:
203 success = success + 1 206 success = success + 1
204 else: 207 else:
205 shpimporter.ERROR("Could not transform feature: %s " % feat.GetFID()) 208 logger.error("Could not transform feature: %s " % feat.GetFID())
206 creationFailed += 1 209 creationFailed += 1
207 else: 210 else:
208 creationFailed = creationFailed + 1 211 creationFailed = creationFailed + 1
209 else: 212 else:
210 unsupported[utils.getWkbString(geomType)] = \ 213 unsupported[utils.getWkbString(geomType)] = \
211 unsupported.get(utils.getWkbString(geomType), 0) + 1 214 unsupported.get(utils.getWkbString(geomType), 0) + 1
212 215
213 shpimporter.INFO("Inserted %i features" % success) 216 logger.info("Inserted %i features" % success)
214 shpimporter.INFO("Failed to create %i features" % creationFailed) 217 logger.info("Failed to create %i features" % creationFailed)
215 for key, value in unsupported.items(): 218 for key, value in unsupported.items():
216 shpimporter.INFO("Found %i unsupported features of type: %s" % (value, key)) 219 logger.info("Found %i unsupported features of type: %s" % (value, key))
217 220
218 if self.tracking_import: 221 if self.tracking_import:
219 unhandled = [] 222 unhandled = []
220 for i in range(0, srcLayer.GetLayerDefn().GetFieldCount()): 223 for i in range(0, srcLayer.GetLayerDefn().GetFieldCount()):
221 act_field = srcLayer.GetLayerDefn().GetFieldDefn(i).GetNameRef() 224 act_field = srcLayer.GetLayerDefn().GetFieldDefn(i).GetNameRef()
222 if not act_field in self.handled_fields: 225 if not act_field in self.handled_fields:
223 unhandled.append(act_field) 226 unhandled.append(act_field)
224 227
225 if len(unhandled): 228 if len(unhandled):
226 shpimporter.INFO("Did not import values from fields: %s " % \ 229 logger.info("Did not import values from fields: %s " % \
227 " ".join(unhandled)) 230 " ".join(unhandled))
228 231
229 try: 232 try:
230 if self.dry_run: 233 if self.dry_run:
231 return geomType 234 return geomType
232 destLayer.CommitTransaction() 235 destLayer.CommitTransaction()
233 except e: 236 except e:
234 shpimporter.ERROR("Exception while committing transaction.") 237 logger.error("Exception while committing transaction.")
235 238
236 return geomType 239 return geomType
http://dive4elements.wald.intevation.org