Mercurial > dive4elements > river

annotate flys-backend/contrib/shpimporter/dgm.py @ 5200:42bb6ff78d1b 2.9.11

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Directly set the connectionInitSqls on the datasource Somehow the factory fails to set the connectionInitSqls if we add it to the dbcpProperties. So we now set it directly
author Andre Heinecke <aheinecke@intevation.de>
date Fri, 08 Mar 2013 11:48:33 +0100
parents f459911fdbfb
children 908848b74d7e
rev   line source
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
1 # -*- coding: utf-8 -*-
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
2
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
3 import codecs
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
4 import utils
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
5
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
6 def latin(string):
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
7 return unicode(string, "latin1")
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
8
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
9 # <dbfield> : (<csvfield>, conversion function)
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
10 DGM_MAP = {
5156
f459911fdbfb Fixed upper and lower conversions in DGM
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5085
diff changeset
11 "lower" : ("km_von", lambda x: float(x)),
f459911fdbfb Fixed upper and lower conversions in DGM
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5085
diff changeset
12 "upper" : ("km_bis", lambda x: float(x)),
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
13 "year_from" : "Jahr_von",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
14 "year_to" : "Jahr_bis",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
15 "projection" : "Projektion",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
16 "elevation_state" : latin("Höhenstatus"),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
17 "format" : "Format",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
18 "border_break" : ("Bruchkanten",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
19 lambda x: True if x.lower() == "Ja" else False),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
20 "resolution" : (latin("Auflösung"), lambda x: x),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
21 # "description" :
4993
71e1b96f4794 Add srid field
Andre Heinecke <aheinecke@intevation.de>
parents: 4975
diff changeset
22 "srid" : "SRID",
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
23 "path" : ("Pfad_Bestand", lambda x: x),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
24 }
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
25
5085
4f46679e13d0 Put km_from and km_to into name of dgm
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
26 SQL_INSERT_DGT = "INSERT INTO dem (river_id, name, " + ", ".join(DGM_MAP.keys()) + \
4993
71e1b96f4794 Add srid field
Andre Heinecke <aheinecke@intevation.de>
parents: 4975
diff changeset
27 ") VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
5085
4f46679e13d0 Put km_from and km_to into name of dgm
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
28 SQL_INSERT_DGT_ORA = "INSERT INTO dem (river_id, name, " + ", ".join(DGM_MAP.keys()) + \
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
29 ") VALUES (:s, :s, :s, :s, :s, :s, :s, :s, :s, :s, :s, :s, :s)"
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
30
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
31 def insertRiverDgm(dbconn, dgmfile, river_name, dry_run, oracle):
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
32 with codecs.open(dgmfile, "r", "latin1") as csvfile:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
33 firstline = csvfile.readline()
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
34 names = firstline.split(";")
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
35 namedict = {}
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
36 field_nr = 0
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
37 for name in names:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
38 namedict[name] = field_nr
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
39 field_nr += 1
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
40
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
41 river_id = utils.getRiverId(dbconn, river_name, oracle)
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
42 for line in csvfile:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
43 fields = line.split(";")
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
44 if not fields: continue
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
45 if fields[namedict[latin("Gewässer")]] != river_name:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
46 continue
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
47 else:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
48 values=[]
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
49 for key, val in DGM_MAP.items():
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
50 if isinstance(val, tuple):
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
51 values.append(val[1](fields[namedict[val[0]]]))
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
52 else:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
53 values.append(unicode.encode(
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
54 fields[namedict[val]], "UTF-8"))
5085
4f46679e13d0 Put km_from and km_to into name of dgm
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
55 name = "%s KM %s - %s" % (river_name, fields[namedict["km_von"]],
4f46679e13d0 Put km_from and km_to into name of dgm
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
56 fields[namedict["km_bis"]])
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
57 cur = dbconn.cursor()
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
58 if oracle:
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
59 stmt = SQL_INSERT_DGT_ORA
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
60 else:
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
61 stmt = SQL_INSERT_DGT
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
62
5085
4f46679e13d0 Put km_from and km_to into name of dgm
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
63 cur.execute(stmt, [river_id, name] + values)
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
64
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
65 if not dry_run:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
66 dbconn.commit()
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
67
http://dive4elements.wald.intevation.org