Mercurial > dive4elements > river

annotate flys-backend/contrib/shpimporter/shpimporter.py @ 5305:e60f65540cc2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added filter to elements in datacage templating. (most work done by Sascha T.).
author Raimund Renkert <rrenkert@intevation.de>
date Thu, 14 Mar 2013 17:09:44 +0100
parents aa043641d9c8
children d086ce6c13d3
rev   line source
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
1 try:
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
2 from osgeo import ogr
4977
ee908e4946cd corrected Typo
Tom Gottfried <tom.gottfried@intevation.de>
parents: 4975
diff changeset
3 except ImportError:
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
4 import ogr
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
5
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
6 import utils, optparse
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
7 import sys
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
8 import os
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
9 import logging
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
10
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
11 from uesg import UESG
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
12 from axis import Axis
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
13 from km import KM
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
14 from fixpoints import Fixpoint
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
15 from buildings import Building
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
16 from crosssectiontracks import CrosssectionTrack
2861
b0132e1b9719 Added further shape importers and added the option to reproject shapes during the import process.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2853
diff changeset
17 from floodplains import Floodplain
b0132e1b9719 Added further shape importers and added the option to reproject shapes during the import process.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2853
diff changeset
18 from boundaries import HydrBoundary, HydrBoundaryPoly
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
19 from hws import HWSLines, HWSPoints
2862
998f1a7dcfde New shape importers and schema modifications.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2861
diff changeset
20 from gauges import GaugeLocation
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
21 from dgm import insertRiverDgm
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
22
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
23 logger = logging.getLogger("shpimporter")
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
24
5219
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
25 os.environ["NLS_LANG"] = ".AL32UTF8"
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
26
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
27 def initialize_logging(level):
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
28 """Initializes the logging system"""
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
29 root = logging.getLogger()
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
30 root.setLevel(level)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
31 hdlr = logging.StreamHandler()
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
32 fmt = logging.Formatter("%(levelname)s %(name)s: %(message)s")
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
33 hdlr.setFormatter(fmt)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
34 root.addHandler(hdlr)
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
35
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
36 def getImporters(river_id, dbconn, dry_run):
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
37 return [
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
38 Axis(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
39 KM(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
40 CrosssectionTrack(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
41 Fixpoint(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
42 Building(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
43 Floodplain(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
44 HydrBoundary(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
45 HydrBoundaryPoly(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
46 HWSLines(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
47 HWSPoints(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
48 GaugeLocation(river_id, dbconn, dry_run),
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
49 UESG(river_id, dbconn, dry_run)
2861
b0132e1b9719 Added further shape importers and added the option to reproject shapes during the import process.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2853
diff changeset
50 ]
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
51
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
52
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
53 def getConfig():
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
54 parser = optparse.OptionParser()
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
55 parser.add_option("--directory", type="string")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
56 parser.add_option("--target_srs", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
57 parser.add_option("--host", type="string")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
58 parser.add_option("--user", type="string")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
59 parser.add_option("--password", type="string")
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
60 parser.add_option("--river_name", type="string")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
61 parser.add_option("--verbose", type="int", default=1)
3655
8654e4480fc6 Shape importer: added command line option dry_run to supress database transactions.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 3654
diff changeset
62 parser.add_option("--dry_run", type="int", default=0)
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
63 parser.add_option("--ogr_connection", type="string")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
64 parser.add_option("--skip_axis", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
65 parser.add_option("--skip_hydr_boundaries", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
66 parser.add_option("--skip_buildings", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
67 parser.add_option("--skip_crosssections", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
68 parser.add_option("--skip_fixpoints", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
69 parser.add_option("--skip_floodplains", type="int")
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
70 parser.add_option("--skip_hws_lines", type="int")
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
71 parser.add_option("--skip_hws_points", type="int")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
72 parser.add_option("--skip_gauge_locations", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
73 parser.add_option("--skip_kms", type="int")
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
74 parser.add_option("--skip_uesgs", type="int")
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
75 parser.add_option("--skip_dgm", type="int")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
76 (config, args) = parser.parse_args()
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
77
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
78 if config.verbose > 1:
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
79 initialize_logging(logging.DEBUG)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
80 elif config.verbose == 1:
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
81 initialize_logging(logging.INFO)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
82 else:
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
83 initialize_logging(logging.WARN)
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
84
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
85 if config.directory == None:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
86 logger.error("No river directory specified!")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
87 raise Exception("Invalid config")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
88 if not config.ogr_connection:
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
89 if not config.host:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
90 logger.error("No database host specified!")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
91 raise Exception("Invalid config")
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
92 if not config.user:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
93 logger.error("No databaser user specified!")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
94 raise Exception("Invalid config")
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
95 if not config.password:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
96 logger.error("No password specified!")
4871
890eaa0a5162 Add ogr_connection parameter to allow directly setting the ogr
Andre Heinecke <aheinecke@intevation.de>
parents: 3655
diff changeset
97 raise Exception("Invalid config")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
98
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
99 return config
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
100
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
101
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
102 def skip_importer(config, importer):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
103 if config.skip_axis == 1 and isinstance(importer, Axis):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
104 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
105 elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundary):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
106 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
107 elif config.skip_hydr_boundaries == 1 and isinstance(importer, HydrBoundaryPoly):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
108 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
109 elif config.skip_buildings == 1 and isinstance(importer, Building):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
110 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
111 elif config.skip_crosssections == 1 and isinstance(importer, CrosssectionTrack):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
112 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
113 elif config.skip_fixpoints == 1 and isinstance(importer, Fixpoint):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
114 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
115 elif config.skip_floodplains == 1 and isinstance(importer, Floodplain):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
116 return True
4963
1469066cc7d9 Importer: Fix the order of the config check
Andre Heinecke <aheinecke@intevation.de>
parents: 4955
diff changeset
117 elif config.skip_hws_lines == 1 and isinstance(importer, HWSLines):
1469066cc7d9 Importer: Fix the order of the config check
Andre Heinecke <aheinecke@intevation.de>
parents: 4955
diff changeset
118 return True
5011
514948fdae54 Do not skip hws_lines when skip hws_points is true
Andre Heinecke <aheinecke@intevation.de>
parents: 5006
diff changeset
119 elif config.skip_hws_points == 1 and isinstance(importer, HWSPoints) and \
514948fdae54 Do not skip hws_lines when skip hws_points is true
Andre Heinecke <aheinecke@intevation.de>
parents: 5006
diff changeset
120 not isinstance(importer, HWSLines):
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
121 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
122 elif config.skip_gauge_locations == 1 and isinstance(importer, GaugeLocation):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
123 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
124 elif config.skip_kms == 1 and isinstance(importer, KM):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
125 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
126 elif config.skip_uesgs == 1 and isinstance(importer, UESG):
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
127 return True
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
128
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
129 return False
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
130
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
131 def main():
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
132 config=None
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
133 try:
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
134 config = getConfig()
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
135 except:
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
136 return -1
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
137
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
138 if config == None:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
139 logger.error("Unable to read config from command line!")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
140 return
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
141
3655
8654e4480fc6 Shape importer: added command line option dry_run to supress database transactions.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 3654
diff changeset
142 if config.dry_run > 0:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
143 logger.info("You enable 'dry_run'. No database transaction will take place!")
3655
8654e4480fc6 Shape importer: added command line option dry_run to supress database transactions.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 3654
diff changeset
144
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
145 if config.ogr_connection:
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
146 connstr = config.ogr_connection
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
147 else:
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
148 connstr = 'OCI:%s/%s@%s' % (config.user, config.password, config.host)
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
149
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
150 oracle = False # Marker if oracle is used.
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
151 if 'OCI:' in connstr:
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
152 oracle = True
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
153 try:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
154 import cx_Oracle as dbapi
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
155 raw_connstr=connstr.replace("OCI:", "")
5077
c5187ab9f571 Fix Syntax Errror
Andre Heinecke <aheinecke@intevation.de>
parents: 5036
diff changeset
156 except ImportError:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
157 logger.error("Module cx_Oracle not found in: %s\n"
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
158 "Neccessary to connect to a Oracle Database.\n"
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
159 "Please refer to the installation "
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
160 "documentation." % sys.path)
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
161 return -1
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
162
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
163 else: # Currently only support for oracle and postgres
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
164 try:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
165 import psycopg2 as dbapi
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
166 raw_connstr=connstr.replace("PG:", "")
4977
ee908e4946cd corrected Typo
Tom Gottfried <tom.gottfried@intevation.de>
parents: 4975
diff changeset
167 except ImportError:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
168 logger.error("Module psycopg2 not found in: %s\n"
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
169 "Neccessary to connect to a Posgresql Database.\n"
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
170 "Please refer to the installation "
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
171 "documentation." % sys.path)
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
172 return -1
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
173
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
174 dbconn_raw = dbapi.connect(raw_connstr)
4879
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
175 dbconn = ogr.Open(connstr)
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
176
63f82bdd45f8 (importer) Enable hws_* importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4872
diff changeset
177 if dbconn == None:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
178 logger.error("Could not connect to database %s" % connstr)
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
179 return -1
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
180
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
181 types = {}
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
182
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
183 directories = []
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
184 if not config.river_name:
5005
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
185 for file in [os.path.join(config.directory, d) for d in \
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
186 os.listdir(config.directory)]:
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
187 if os.path.isdir(file):
5005
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
188 directories.append(file)
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
189 else:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
190 directories.append(config.directory)
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
191
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
192 for directory in directories:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
193 if not config.river_name:
5005
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
194 river_name = utils.getUTF8Path(
eb2d6609387c Importer: Fix directory paths and river name encoding
Andre Heinecke <aheinecke@intevation.de>
parents: 4998
diff changeset
195 os.path.basename(os.path.normpath(directory)))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
196 else:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
197 river_name = config.river_name
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
198 river_id = utils.getRiverId(dbconn_raw, river_name, oracle)
2853
bd9e76e0b55d Improved the python shapefile importer.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2798
diff changeset
199
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
200 if not river_id:
5221
aa043641d9c8 Fix syntax error when river id is not found (pylint could have told me this)
Andre Heinecke <aheinecke@intevation.de>
parents: 5219
diff changeset
201 logger.info(u"Could not find river in database. Skipping: %s"
aa043641d9c8 Fix syntax error when river id is not found (pylint could have told me this)
Andre Heinecke <aheinecke@intevation.de>
parents: 5219
diff changeset
202 % unicode(utils.getUTF8(river_name), "UTF-8"))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
203 continue
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
204 else:
5219
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
205 logger.info(u"Importing River: %s" % unicode(
40dc001594e4 Allow unicode river names
Andre Heinecke <aheinecke@intevation.de>
parents: 5077
diff changeset
206 utils.getUTF8(river_name), "UTF-8"))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
207
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
208 for importer in getImporters(river_id, dbconn, config.dry_run):
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
209 if skip_importer(config, importer):
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
210 logger.info("Skip import of '%s'" % importer.getName())
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
211 continue
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
212
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
213 logger.info("Start import of '%s'" % importer.getName())
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
214
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
215 shapes = utils.findShapefiles(importer.getPath(config.directory))
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
216 logger.debug("Found %i Shapefiles" % len(shapes))
4970
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
217
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
218 for shpTuple in shapes:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
219 geomType = importer.walkOverShapes(shpTuple)
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
220 try:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
221 if geomType is not None:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
222 num = types[geomType]
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
223 types[geomType] = num+1
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
224 except:
174fbaa3d344 Add handling of River Names and remove target_src parameter
Andre Heinecke <aheinecke@intevation.de>
parents: 4963
diff changeset
225 types[geomType] = 1
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
226
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
227 for key in types:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
228 logger.debug("%i x geometry type %s" % (types[key], key))
2798
5a654f2e35bc Added a python tool to import shapefiles into database.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
diff changeset
229
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
230 if not config.skip_dgm:
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
231 dgmfilename = os.path.join(
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
232 config.directory, "..", "DGMs.csv")
4989
f4d61ac84b76 If we bother to check if the file is readable we might also
Andre Heinecke <aheinecke@intevation.de>
parents: 4978
diff changeset
233 if not os.access(dgmfilename, os.R_OK) or not \
f4d61ac84b76 If we bother to check if the file is readable we might also
Andre Heinecke <aheinecke@intevation.de>
parents: 4978
diff changeset
234 os.path.isfile(dgmfilename):
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
235 logger.info("Could not find or access DGM file: %s \n"
4975
9eea3cd22ee7 Add dgm file reader although an awk script should have been more appropiate
Andre Heinecke <aheinecke@intevation.de>
parents: 4970
diff changeset
236 "Skipping DGM import." % dgmfilename)
4978
b5ac7588cf4d Use os.access instead of try/except open
Andre Heinecke <aheinecke@intevation.de>
parents: 4977
diff changeset
237 else:
5006
769593a84606 Importer: Behold, Logging!
Andre Heinecke <aheinecke@intevation.de>
parents: 5005
diff changeset
238 logger.info("Inserting DGM meta information in 'dem' table.")
5036
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
239 insertRiverDgm(dbconn_raw, dgmfilename, river_name,
d6860fca89e4 Pull information about the backend into dgm.
Andre Heinecke <aheinecke@intevation.de>
parents: 5028
diff changeset
240 config.dry_run, oracle)
5012
a79f6a085ab6 Add message if DGM import is skipped
Andre Heinecke <aheinecke@intevation.de>
parents: 5011
diff changeset
241 else:
a79f6a085ab6 Add message if DGM import is skipped
Andre Heinecke <aheinecke@intevation.de>
parents: 5011
diff changeset
242 logger.info("Skip import of DGM.")
3654
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
243
59ca5dab2782 Shape importer: use python's OptionParse to read user specific configuration from command line.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2862
diff changeset
244 if __name__ == '__main__':
4872
a563e9f58f93 Improve error handling and unify dbconn for all importers
Andre Heinecke <aheinecke@intevation.de>
parents: 4871
diff changeset
245 main()
http://dive4elements.wald.intevation.org