Mercurial > dive4elements > river

annotate backend/contrib/shpimporter/shpimporter.py @ 9751:308a0d822d18 3.2.x

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Keep configuration and data in distinct directories This allows having distinct volumes for configuration and data (artifact database, generated mapfiles and shapefiles, etc.). While at it, cleanup MapServer configuration a little bit.
author Tom Gottfried <tom@intevation.de>
date Tue, 11 Oct 2022 11:42:09 +0200
parents 4a13535ed949
children
rev   line source
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
1 try:
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
2 from osgeo import ogr
4977
ee908e4946cd corrected Typo
Tom Gottfried <tom.gottfried@intevation.de>
parents: 4975
diff changeset
3 except ImportError:
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
4 import ogr
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
5
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
6 import utils, optparse
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
7 import sys
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
8 import os
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
9 import logging
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
10
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
11 from uesg import UESG
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
12 from axis import Axis
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
13 from km import KM
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
14 from fixpoints import Fixpoint
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
15 from buildings import Building
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
16 from crosssectiontracks import CrosssectionTrack
2861
b0132e1b9719 Added further shape importers and added the option to reproject shapes during the import process.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2853
diff changeset
17 from floodplains import Floodplain
b0132e1b9719 Added further shape importers and added the option to reproject shapes during the import process.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2853
diff changeset
18 from boundaries import HydrBoundary, HydrBoundaryPoly
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
19 from hws import HWSLines, HWSPoints
5353
d086ce6c13d3 Add jetties importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5221
diff changeset
20 from jetties import Jetties
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
21 from dgm import insertRiverDgm
5545
ed149d5d7fb7 Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5353
diff changeset
22 from floodmarks import Floodmark
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
23
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
24 logger = logging.getLogger("shpimporter")
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
25
5219
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
26 os.environ["NLS_LANG"] = ".AL32UTF8"
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
27
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
28 def initialize_logging(level):
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
29 """Initializes the logging system"""
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
30 root = logging.getLogger()
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
31 root.setLevel(level)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
32 hdlr = logging.StreamHandler()
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
33 fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
34 hdlr.setFormatter(fmt)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
35 root.addHandler(hdlr)
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
36
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
37 def getImporters(river_id, dbconn, dry_run):
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
38 return [
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
39 Axis(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
40 KM(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
41 CrosssectionTrack(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
42 Fixpoint(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
43 Building(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
44 Floodplain(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
45 HydrBoundary(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
46 HydrBoundaryPoly(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
47 HWSLines(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
48 HWSPoints(river_id, dbconn, dry_run),
5353
d086ce6c13d3 Add jetties importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5221
diff changeset
49 Jetties(river_id, dbconn, dry_run),
5545
ed149d5d7fb7 Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5353
diff changeset
50 Floodmark(river_id, dbconn, dry_run),
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
51 UESG(river_id, dbconn, dry_run)
2861
b0132e1b9719 Added further shape importers and added the option to reproject shapes during the import process.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2853
diff changeset
52 ]
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
53
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
54
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
55 def getConfig():
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
56 parser = optparse.OptionParser()
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
57 parser.add_option("--directory", type="string")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
58 parser.add_option("--target_srs", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
59 parser.add_option("--host", type="string")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
60 parser.add_option("--user", type="string")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
61 parser.add_option("--password", type="string")
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
62 parser.add_option("--river_name", type="string")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
63 parser.add_option("--verbose", type="int", default=1)
3655
8654e4480fc6 Shape importer: added command line option dry_run to supress database transactions.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 3654
diff changeset
64 parser.add_option("--dry_run", type="int", default=0)
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
65 parser.add_option("--ogr_connection", type="string")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
66 parser.add_option("--skip_axis", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
67 parser.add_option("--skip_hydr_boundaries", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
68 parser.add_option("--skip_buildings", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
69 parser.add_option("--skip_crosssections", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
70 parser.add_option("--skip_fixpoints", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
71 parser.add_option("--skip_floodplains", type="int")
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
72 parser.add_option("--skip_hws_lines", type="int")
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
73 parser.add_option("--skip_hws_points", type="int")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
74 parser.add_option("--skip_kms", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
75 parser.add_option("--skip_uesgs", type="int")
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
76 parser.add_option("--skip_dgm", type="int")
5353
d086ce6c13d3 Add jetties importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5221
diff changeset
77 parser.add_option("--skip_jetties", type="int")
5545
ed149d5d7fb7 Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5353
diff changeset
78 parser.add_option("--skip_floodmarks", type="int")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
79 (config, args) = parser.parse_args()
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
80
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
81 if config.verbose > 1:
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
82 initialize_logging(logging.DEBUG)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
83 elif config.verbose == 1:
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
84 initialize_logging(logging.INFO)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
85 else:
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
86 initialize_logging(logging.WARN)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
87
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
88 if config.directory == None:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
89 logger.error("No river directory specified!")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
90 raise Exception("Invalid config")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
91 if not config.ogr_connection:
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
92 if not config.host:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
93 logger.error("No database host specified!")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
94 raise Exception("Invalid config")
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
95 if not config.user:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
96 logger.error("No databaser user specified!")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
97 raise Exception("Invalid config")
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
98 if not config.password:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
99 logger.error("No password specified!")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
100 raise Exception("Invalid config")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
101
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
102 return config
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
103
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
104
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
105 def skip_importer(config, importer):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
106 if config.skip_axis == 1 and isinstance(importer, Axis):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
107 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
108 elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundary):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
109 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
110 elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundaryPoly):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
111 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
112 elif config.skip_buildings == 1 and isinstance(importer, Building):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
113 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
114 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
115 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
116 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
117 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
118 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
119 return True
4963
1469066cc7d9 Importer: Fix the order of the config check
Andre Heinecke <aheinecke@intevation.de>
parents: 4955
diff changeset
120 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines):
1469066cc7d9 Importer: Fix the order of the config check
Andre Heinecke <aheinecke@intevation.de>
parents: 4955
diff changeset
121 return True
5011
514948fdae54 Do not skip hws_lines when skip hws_points is true
Andre Heinecke <aheinecke@intevation.de>
parents: 5006
diff changeset
122 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints) and \
514948fdae54 Do not skip hws_lines when skip hws_points is true
Andre Heinecke <aheinecke@intevation.de>
parents: 5006
diff changeset
123 not isinstance(importer, HWSLines):
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
124 return True
5353
d086ce6c13d3 Add jetties importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5221
diff changeset
125 elif config.skip_jetties == 1 and isinstance(importer, Jetties):
d086ce6c13d3 Add jetties importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5221
diff changeset
126 return True
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
127 elif config.skip_kms == 1 and isinstance(importer, KM):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
128 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
129 elif config.skip_uesgs == 1 and isinstance(importer, UESG):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
130 return True
5545
ed149d5d7fb7 Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5353
diff changeset
131 elif config.skip_floodmarks == 1 and isinstance(importer, Floodmark):
ed149d5d7fb7 Add floodmarks importer
Andre Heinecke <aheinecke@intevation.de>
parents: 5353
diff changeset
132 return True
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
133
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
134 return False
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
135
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
136 def main():
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
137 config=None
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
138 try:
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
139 config = getConfig()
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
140 except:
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
141 return -1
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
142
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
143 if config == None:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
144 logger.error("Unable to read config from command line!")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
145 return
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
146
3655
8654e4480fc6 Shape importer: added command line option dry_run to supress database transactions.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 3654
diff changeset
147 if config.dry_run > 0:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
148 logger.info("You enable 'dry_run'. No database transaction will take place!")
3655
8654e4480fc6 Shape importer: added command line option dry_run to supress database transactions.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 3654
diff changeset
149
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
150 if config.ogr_connection:
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
151 connstr = config.ogr_connection
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
152 else:
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
153 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
154
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
155 oracle = False # Marker if oracle is used.
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
156 if 'OCI:' in connstr:
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
157 oracle = True
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
158 try:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
159 import cx_Oracle as dbapi
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
160 raw_connstr=connstr.replace("OCI:", "")
5077
c5187ab9f571 Fix Syntax Errror
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
161 except ImportError:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
162 logger.error("Module cx_Oracle not found in: %s\n"
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
163 "Neccessary to connect to a Oracle Database.\n"
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
164 "Please refer to the installation "
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
165 "documentation." % sys.path)
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
166 return -1
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
167
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
168 else: # Currently only support for oracle and postgres
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
169 try:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
170 import psycopg2 as dbapi
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
171 raw_connstr=connstr.replace("PG:", "")
4977
ee908e4946cd corrected Typo
Tom Gottfried <tom.gottfried@intevation.de>
parents: 4975
diff changeset
172 except ImportError:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
173 logger.error("Module psycopg2 not found in: %s\n"
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
174 "Neccessary to connect to a Posgresql Database.\n"
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
175 "Please refer to the installation "
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
176 "documentation." % sys.path)
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
177 return -1
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
178
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
179 dbconn_raw = dbapi.connect(raw_connstr)
8827
4a13535ed949 Upgrade importer to GDAL 1.11.
Tom Gottfried <tom@intevation.de>
parents: 8611
diff changeset
180 dbconn = ogr.Open(connstr, 1)
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
181
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
182 if dbconn == None:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
183 logger.error("Could not connect to database %s" % connstr)
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
184 return -1
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
185
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
186 types = {}
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
187
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
188 directories = []
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
189 if not config.river_name:
5005
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
190 for file in [os.path.join(config.directory, d) for d in \
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
191 os.listdir(config.directory)]:
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
192 if os.path.isdir(file):
5005
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
193 directories.append(file)
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
194 else:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
195 directories.append(config.directory)
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
196
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
197 for directory in directories:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
198 if not config.river_name:
5005
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
199 river_name = utils.getUTF8Path(
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
200 os.path.basename(os.path.normpath(directory)))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
201 else:
8611
e331a5ad2554 (issue1799) Strip whitespace from river names
Andre Heinecke <andre.heinecke@intevation.de>
parents: 6815
diff changeset
202 river_name = config.river_name.strip()
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
203 river_id = utils.getRiverId(dbconn_raw, river_name, oracle)
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
204
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
205 if not river_id:
5221
aa043641d9c8 Fix syntax error when river id is not found (pylint could have told me this)
Andre Heinecke <aheinecke@intevation.de>
parents: 5219
diff changeset
206 logger.info(u"Could not find river in database. Skipping: %s"
aa043641d9c8 Fix syntax error when river id is not found (pylint could have told me this)
Andre Heinecke <aheinecke@intevation.de>
parents: 5219
diff changeset
207 % unicode(utils.getUTF8(river_name), "UTF-8"))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
208 continue
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
209 else:
5219
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
210 logger.info(u"Importing River: %s" % unicode(
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
211 utils.getUTF8(river_name), "UTF-8"))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
212
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
213 for importer in getImporters(river_id, dbconn, config.dry_run):
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
214 if skip_importer(config, importer):
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
215 logger.info("Skip import of '%s'" % importer.getName())
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
216 continue
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
217
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
218 logger.info("Start import of '%s'" % importer.getName())
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
219
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
220 shapes = utils.findShapefiles(importer.getPath(config.directory))
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
221 logger.debug("Found %i Shapefiles" % len(shapes))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
222
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
223 for shpTuple in shapes:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
224 geomType = importer.walkOverShapes(shpTuple)
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
225 try:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
226 if geomType is not None:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
227 num = types[geomType]
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
228 types[geomType] = num+1
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
229 except:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
230 types[geomType] = 1
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
231
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
232 for key in types:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
233 logger.debug("%i x geometry type %s" % (types[key], key))
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
234
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
235 if not config.skip_dgm:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
236 dgmfilename = os.path.join(
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
237 config.directory, "..", "DGMs.csv")
4989
f4d61ac84b76 If we bother to check if the file is readable we might also
Andre Heinecke <aheinecke@intevation.de>
parents: 4978
diff changeset
238 if not os.access(dgmfilename, os.R_OK) or not \
f4d61ac84b76 If we bother to check if the file is readable we might also
Andre Heinecke <aheinecke@intevation.de>
parents: 4978
diff changeset
239 os.path.isfile(dgmfilename):
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
240 logger.info("Could not find or access DGM file: %s \n"
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
241 "Skipping DGM import." % dgmfilename)
4978
b5ac7588cf4d Use os.access instead of try/except open
Andre Heinecke <aheinecke@intevation.de>
parents: 4977
diff changeset
242 else:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
243 logger.info("Inserting DGM meta information in 'dem' table.")
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
244 insertRiverDgm(dbconn_raw, dgmfilename, river_name,
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
245 config.dry_run, oracle)
5012
a79f6a085ab6 Add message if DGM import is skipped
Andre Heinecke <aheinecke@intevation.de>
parents: 5011
diff changeset
246 else:
a79f6a085ab6 Add message if DGM import is skipped
Andre Heinecke <aheinecke@intevation.de>
parents: 5011
diff changeset
247 logger.info("Skip import of DGM.")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
248
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
249 if __name__ == '__main__':
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
250 main()
http://dive4elements.wald.intevation.org