Mercurial > dive4elements > river

annotate flys-backend/contrib/shpimporter/dgm.py @ 5278:3ddf5843ede3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
correct session-handling for bed height import (made by S. Teichmann)
author Tom Gottfried <tom.gottfried@intevation.de>
date Wed, 13 Mar 2013 18:53:42 +0100
parents 9245f4b434b3
children 7d29c277000e
rev   line source
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
1 # -*- coding: utf-8 -*-
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
2
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
3 import codecs
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
4 import utils
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
5 import datetime
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
6
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
7 def latin(string):
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
8 return unicode(string, "latin1")
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
9
5213
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
10 import logging
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
11 logger = logging.getLogger("DGM")
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
12
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
13
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
14 # <dbfield> : (<csvfield>, conversion function)
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
15 DGM_MAP = {
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
16 "projection" : "Projektion",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
17 "elevation_state" : latin("Höhenstatus"),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
18 "format" : "Format",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
19 "border_break" : ("Bruchkanten",
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
20 lambda x: True if x.lower() == "Ja" else False),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
21 "resolution" : (latin("Auflösung"), lambda x: x),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
22 # "description" :
4993
71e1b96f4794 Add srid field
Andre Heinecke <aheinecke@intevation.de>
parents: 4975
diff changeset
23 "srid" : "SRID",
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
24 "path" : ("Pfad_Bestand", lambda x: x),
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
25 }
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
26
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
27 SQL_INSERT_DGT = "INSERT INTO dem (river_id, name," \
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
28 " time_interval_id, range_id, " + ", ".join(DGM_MAP.keys()) + \
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
29 ") VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
30 SQL_INSERT_DGT_ORA = "INSERT INTO dem (river_id, name," \
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
31 " time_interval_id, range_id, " + ", ".join(DGM_MAP.keys()) + \
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
32 ") VALUES (:s, :s, :s, :s, :s, :s, :s, :s, :s, :s, :s)"
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
33 SQL_SELECT_TIME_ID = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
34 SELECT id FROM time_intervals WHERE start_time = %s AND stop_time = %s
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
35 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
36 SQL_INSERT_TIME_ID = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
37 INSERT INTO time_intervals (id, start_time, stop_time) VALUES (%s, %s, %s)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
38 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
39 SQL_SELECT_TIME_ID_ORA = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
40 SELECT id FROM time_intervals WHERE start_time = :s AND stop_time = :s
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
41 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
42 SQL_INSERT_TIME_ID_ORA = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
43 INSERT INTO time_intervals (id, start_time, stop_time) VALUES (:s, :s, :s)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
44 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
45 SQL_SELECT_RANGE_ID = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
46 SELECT id FROM ranges WHERE river_id = %s AND a = %s AND b = %s
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
47 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
48 SQL_INSERT_RANGE_ID = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
49 INSERT INTO ranges (id, river_id, a, b) VALUES (%s, %s, %s, %s)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
50 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
51 SQL_SELECT_RANGE_ID_ORA = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
52 SELECT id FROM ranges WHERE river_id = :s AND a = :s AND b = :s
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
53 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
54 SQL_INSERT_RANGE_ID_ORA = """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
55 INSERT INTO ranges (id, river_id, a, b) VALUES (:s, :s, :s, :s)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
56 """
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
57 SQL_NEXT_ID = "SELECT coalesce(max(id), -1) + 1 FROM %s"
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
58
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
59
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
60 NEXT_IDS = {}
5267
9245f4b434b3 Fix syntax error in next_id and remove obsolete oracle parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 5260
diff changeset
61 def next_id(cur, relation):
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
62 idx = NEXT_IDS.get(relation)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
63 if idx is None:
5267
9245f4b434b3 Fix syntax error in next_id and remove obsolete oracle parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 5260
diff changeset
64 cur.execute(SQL_NEXT_ID % relation)
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
65 idx = cur.fetchone()[0]
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
66 NEXT_IDS[relation] = idx + 1
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
67 return idx
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
68
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
69 def get_range_id(cur, river_id, a, b, oracle):
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
70 if oracle:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
71 cur.execute(SQL_SELECT_RANGE_ID_ORA, (river_id, a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
72 else:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
73 cur.execute(SQL_SELECT_RANGE_ID, (river_id, a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
74 row = cur.fetchone()
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
75 if row: return row[0]
5267
9245f4b434b3 Fix syntax error in next_id and remove obsolete oracle parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 5260
diff changeset
76 idx = next_id(cur, "ranges")
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
77 if oracle:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
78 cur.execute(SQL_INSERT_RANGE_ID_ORA, (idx, river_id, a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
79 else:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
80 cur.execute(SQL_INSERT_RANGE_ID, (idx, river_id, a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
81 cur.connection.commit()
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
82 return idx
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
83
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
84 def get_time_interval_id(cur, a, b, oracle):
5213
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
85 if not a or not b:
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
86 return None
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
87 if oracle:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
88 cur.execute(SQL_SELECT_TIME_ID_ORA, (a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
89 else:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
90 cur.execute(SQL_SELECT_TIME_ID, (a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
91 row = cur.fetchone()
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
92 if row: return row[0]
5267
9245f4b434b3 Fix syntax error in next_id and remove obsolete oracle parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 5260
diff changeset
93 idx = next_id(cur, "time_intervals")
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
94 if oracle:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
95 cur.execute(SQL_INSERT_TIME_ID_ORA, (idx, a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
96 else:
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
97 cur.execute(SQL_INSERT_TIME_ID, (idx, a, b))
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
98 cur.connection.commit()
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
99 return idx
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
100
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
101 def insertRiverDgm(dbconn, dgmfile, river_name, dry_run, oracle):
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
102 with codecs.open(dgmfile, "r", "latin1") as csvfile:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
103 firstline = csvfile.readline()
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
104 names = firstline.split(";")
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
105 namedict = {}
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
106 field_nr = 0
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
107 for name in names:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
108 namedict[name] = field_nr
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
109 field_nr += 1
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
110
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
111 river_id = utils.getRiverId(dbconn, river_name, oracle)
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
112 for line in csvfile:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
113 fields = line.split(";")
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
114 if not fields: continue
5222
3b86bf214d53 Fix check for river_name in dgm to handle unicode names
Andre Heinecke <aheinecke@intevation.de>
parents: 5213
diff changeset
115 if fields[namedict[latin("Gewässer")]] != \
3b86bf214d53 Fix check for river_name in dgm to handle unicode names
Andre Heinecke <aheinecke@intevation.de>
parents: 5213
diff changeset
116 unicode(utils.getUTF8(river_name),'UTF-8'):
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
117 continue
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
118 else:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
119 values=[]
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
120 for key, val in DGM_MAP.items():
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
121 if isinstance(val, tuple):
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
122 values.append(val[1](fields[namedict[val[0]]]))
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
123 else:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
124 values.append(unicode.encode(
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
125 fields[namedict[val]], "UTF-8"))
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
126 km_von = fields[namedict["km_von"]]
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
127 km_bis = fields[namedict["km_bis"]]
5213
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
128 year_from = None
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
129 year_to = None
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
130 try:
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
131 year_from = datetime.datetime(
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
132 int(fields[namedict["Jahr_von"]]), 1, 1)
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
133 year_to = datetime.datetime(
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
134 int(fields[namedict["Jahr_bis"]]),1 ,1)
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
135 except ValueError:
76818dc2c937 Importer: Handle empty year fields in dgm csv
Andre Heinecke <aheinecke@intevation.de>
parents: 5210
diff changeset
136 logger.warn("Invalid numbers (or none) found in year_from and year_to")
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
137
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
138 name = "%s KM %s - %s" % (river_name, km_von, km_bis)
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
139 cur = dbconn.cursor()
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
140 range_id = get_range_id(cur, river_id, float(km_von),
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
141 float(km_bis), oracle)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
142 time_interval_id = get_time_interval_id(cur, year_from,
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
143 year_to, oracle)
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
144
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
145 if oracle:
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
146 stmt = SQL_INSERT_DGT_ORA
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
147 else:
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
148 stmt = SQL_INSERT_DGT
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 4993
diff changeset
149
5210
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
150 cur.execute(stmt, [river_id, name, time_interval_id,
908848b74d7e SCHEME CHANGE: dgm now uses time intervals and ranges
Andre Heinecke <aheinecke@intevation.de>
parents: 5156
diff changeset
151 range_id] + values)
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
152
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
153 if not dry_run:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
154 dbconn.commit()
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents:
diff changeset
155
http://dive4elements.wald.intevation.org