Mercurial > dive4elements > river
annotate flys-backend/contrib/shpimporter/floodmarks.py @ 5779:ebec12def170
Datacage: Add a pool of builders to make it multi threadable.
XML DOM is not thread safe. Therefore the old implementation only allowed one thread
to use the builder at a time. As the complexity of the configuration
has increased over time this has become a bottleneck of the whole application
because it took quiet some time to build a result. Furthermore the builder code path
is visited very frequent. So many concurrent requests were piled up
resulting in long waits for the users.
To mitigate this problem a round robin pool of builders is used now.
Each of the pooled builders has an independent copy of the XML template
and can be run in parallel.
The number of builders is determined by the system property
'flys.datacage.pool.size'. It defaults to 4.
author | Sascha L. Teichmann <teichmann@intevation.de> |
---|---|
date | Sun, 21 Apr 2013 12:48:09 +0200 |
parents | ed149d5d7fb7 |
children |
rev | line source |
---|---|
5545
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
1 # -*- coding: utf-8 -*- |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
2 try: |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
3 from osgeo import ogr |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
4 except ImportError: |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
5 import ogr |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
6 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
7 from importer import Importer |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
8 import logging |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
9 import os |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
10 import re |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
11 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
12 TABLE_NAME="flood_marks" |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
13 PATH="Hydrologie/HW-Marken" |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
14 NAME="Floodmarks" |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
15 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
16 logger = logging.getLogger(NAME) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
17 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
18 BUILDING_KINDS= { |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
19 "sonstige" : 0, |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
20 "brücken" : 1, |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
21 "wehre" : 2, |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
22 "pegel" : 3, |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
23 } |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
24 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
25 class Floodmark(Importer): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
26 fieldmap = { |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
27 "^station$" : "km", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
28 "^km$" : "km", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
29 "^wsv-km$" : "km", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
30 "^FlussKm$" : "km", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
31 "^z$" : "z", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
32 "^z\d*" : "z", # z02, z1890, usw. |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
33 "^m+NHN$" : "z", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
34 "^Ort$" : "location", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
35 "^Pegel$" : "location", |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
36 } |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
37 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
38 def getPath(self, base): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
39 return "%s/%s" % (base, PATH) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
40 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
41 def getTablename(self): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
42 return TABLE_NAME |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
43 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
44 def getName(self): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
45 return NAME |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
46 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
47 def isGeometryValid(self, geomType): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
48 return geomType == ogr.wkbPoint |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
49 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
50 def isShapeRelevant(self, name, path): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
51 return "hw-marken" in name.lower() |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
52 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
53 def createNewFeature(self, featureDef, feat, **args): |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
54 newFeat = ogr.Feature(featureDef) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
55 geometry = feat.GetGeometryRef() |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
56 geometry.SetCoordinateDimension(2) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
57 newFeat.SetGeometry(geometry) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
58 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
59 self.copyFields(feat, newFeat, self.fieldmap) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
60 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
61 newFeat.SetField("river_id", self.river_id) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
62 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
63 filename = os.path.basename(args['path']) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
64 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
65 # Try to extract the year from the filename |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
66 match = re.search(r"([_\-])(\d\d\d\d)([_\-])", filename) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
67 if match: |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
68 year = match.groups()[1] |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
69 year = int(year) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
70 newFeat.SetField("year", year) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
71 else: |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
72 logger.warn(u"Could not extract year from filename: %s " % filename) |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
73 |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
74 return newFeat |
ed149d5d7fb7
Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents:
diff
changeset
|
75 |