Mercurial > dive4elements > river
annotate backend/src/main/java/org/dive4elements/river/importer/Importer.java @ 9650:a2a42a6bac6b
Importer (s/u-info) extensions:
outer try/catch for parse and log of line no,
catching parsing exception if not enough value fields,
parsing error and warning log messages with line number,
detecting and rejecting duplicate data series,
better differentiation between error and warning log messages
author | mschaefer |
---|---|
date | Mon, 23 Mar 2020 14:57:03 +0100 |
parents | a44200bf0547 |
children |
rev | line source |
---|---|
5844
4dd33b86dc61
Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5838
diff
changeset
|
1 /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde |
4dd33b86dc61
Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5838
diff
changeset
|
2 * Software engineering by Intevation GmbH |
4dd33b86dc61
Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5838
diff
changeset
|
3 * |
5992
4c3ccf2b0304
Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5844
diff
changeset
|
4 * This file is Free Software under the GNU AGPL (>=v3) |
5844
4dd33b86dc61
Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5838
diff
changeset
|
5 * and comes with ABSOLUTELY NO WARRANTY! Check out the |
5992
4c3ccf2b0304
Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5844
diff
changeset
|
6 * documentation coming with Dive4Elements River for details. |
5844
4dd33b86dc61
Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5838
diff
changeset
|
7 */ |
4dd33b86dc61
Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5838
diff
changeset
|
8 |
5829
18619c1e7c2a
Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents:
5828
diff
changeset
|
9 package org.dive4elements.river.importer; |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
10 |
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
11 import java.io.File; |
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
12 import java.io.IOException; |
8970 | 13 import java.net.MalformedURLException; |
14 import java.sql.SQLException; | |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
15 import java.util.HashMap; |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
16 import java.util.List; |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
17 import java.util.Map; |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
18 |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
19 import org.apache.log4j.Logger; |
8970 | 20 import org.apache.log4j.PropertyConfigurator; |
21 import org.dive4elements.artifacts.common.utils.XMLUtils; | |
22 import org.dive4elements.river.backend.utils.StringUtil; | |
23 import org.dive4elements.river.importer.parsers.AnnotationClassifier; | |
24 import org.dive4elements.river.importer.parsers.BundesWasserStrassenParser; | |
25 import org.dive4elements.river.importer.parsers.InfoGewParser; | |
26 import org.hibernate.HibernateException; | |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
27 import org.hibernate.Transaction; |
8970 | 28 import org.slf4j.bridge.SLF4JBridgeHandler; |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
29 import org.w3c.dom.Document; |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
30 |
5016
255898799ed9
Minimal whitespace and doc cosmetics.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
4708
diff
changeset
|
31 /** Data Importer. Further processing happens per-river. */ |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
32 public class Importer |
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
33 { |
9011 | 34 static final String VERSION = "3.3.0"; |
35 | |
8200
9d2e69f971f5
sed -i src/**/*.java 's/logger/log/g'
Sascha L. Teichmann <teichmann@intevation.de>
parents:
8187
diff
changeset
|
36 /** Private log. */ |
8843 | 37 private static final Logger log = Logger.getLogger(Importer.class); |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
38 |
8843 | 39 private static final String BWASTR_ID_CSV_FILE = "BWASTR_ID.csv"; |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
40 |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
41 protected List<ImportRiver> rivers; |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
42 |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
43 public Importer() { |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
44 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
45 |
8970 | 46 public Importer(final List<ImportRiver> rivers) { |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
47 this.rivers = rivers; |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
48 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
49 |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
50 public List<ImportRiver> getRivers() { |
8970 | 51 return this.rivers; |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
52 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
53 |
8970 | 54 public void setRivers(final List<ImportRiver> rivers) { |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
55 this.rivers = rivers; |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
56 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
57 |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
58 /** Write rivers and their dependencies/dependants to db. */ |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
59 public void writeRivers() { |
9650 | 60 log.info("write rivers started"); |
188
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
61 |
8970 | 62 for (final ImportRiver river: this.rivers) { |
188
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
63 log.debug("writing river '" + river.getName() + "'"); |
190
d40da430d2fe
Added stub code to write gauges
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
189
diff
changeset
|
64 river.storeDependencies(); |
497
67fd63e4ef66
Importer: centralized caching
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
199
diff
changeset
|
65 ImporterSession.getInstance().getDatabaseSession().flush(); |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
66 } |
188
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
67 |
9650 | 68 log.info("write rivers finished"); |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
69 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
70 |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
71 public void writeToDatabase() { |
183
222f4db3430a
Importer: Used thread local pattern to make sharing of session easier.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
182
diff
changeset
|
72 |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
73 Transaction tx = null; |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
74 |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
75 try { |
8970 | 76 tx = ImporterSession.getInstance().getDatabaseSession().beginTransaction(); |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
77 |
188
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
78 try { |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
79 writeRivers(); |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
80 } |
8970 | 81 catch (final HibernateException he) { |
188
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
82 Throwable t = he.getCause(); |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
83 while (t instanceof SQLException) { |
8970 | 84 final SQLException sqle = (SQLException) t; |
188
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
85 log.error("SQL exeception chain:", sqle); |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
86 t = sqle.getNextException(); |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
87 } |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
88 throw he; |
003ac16812dd
Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
183
diff
changeset
|
89 } |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
90 |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
91 tx.commit(); |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
92 } |
8970 | 93 catch (final RuntimeException re) { |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
94 if (tx != null) { |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
95 tx.rollback(); |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
96 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
97 throw re; |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
98 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
99 } |
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
100 |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
101 public static AnnotationClassifier getAnnotationClassifier() { |
8970 | 102 final String annotationTypes = Config.INSTANCE.getAnnotationTypes(); |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
103 |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
104 if (annotationTypes == null) { |
766
aa9e3da95c31
Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
765
diff
changeset
|
105 log.info("no annotation types file configured."); |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
106 return null; |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
107 } |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
108 |
8970 | 109 final File file = new File(annotationTypes); |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
110 |
766
aa9e3da95c31
Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
765
diff
changeset
|
111 log.info("use annotation types file '" + file + "'"); |
aa9e3da95c31
Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
765
diff
changeset
|
112 |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
113 if (!(file.isFile() && file.canRead())) { |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
114 log.warn("annotation type file '" + file + "' is not readable."); |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
115 return null; |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
116 } |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
117 |
8970 | 118 final Document rules = XMLUtils.parseDocument(file, false, null); |
765
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
119 |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
120 if (rules == null) { |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
121 log.warn("cannot parse annotation types file."); |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
122 return null; |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
123 } |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
124 |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
125 return new AnnotationClassifier(rules); |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
126 } |
763c4137d6e1
Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
497
diff
changeset
|
127 |
4707
2b0426b79a92
Documentation.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
3659
diff
changeset
|
128 |
2b0426b79a92
Documentation.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
3659
diff
changeset
|
129 /** Starting point for importing river data. */ |
8970 | 130 public static void main(final String [] args) { |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
131 |
8970 | 132 configureLogging(); |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
133 |
9011 | 134 log.info("IMPORTER version " + VERSION); |
8970 | 135 log.info("START parsing rivers..."); |
136 | |
137 final InfoGewParser infoGewParser = new InfoGewParser(getAnnotationClassifier()); | |
2806
33f40b23edd8
Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
2804
diff
changeset
|
138 |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
139 File bwastrFile = null; |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
140 |
8970 | 141 // Main parsing loop for all river gew file paths in args |
142 // FIXME: Multiple rivers lead to reparsing the already parsed rivers again in InfoGewParser.parse... | |
143 for (final String gew : args) { | |
144 log.info("Parsing info gew file: " + gew); | |
145 final File gewFile = new File(gew); | |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
146 if (bwastrFile == null) { |
8970 | 147 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE); |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
148 } |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
149 try { |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
150 infoGewParser.parse(gewFile); |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
151 } |
9650 | 152 catch (final Exception ioe) { |
8970 | 153 log.error("error parsing gew: " + gew, ioe); |
5106
d16398d65a59
Importer: Print a decent error when failing to parse a gew and exit.
Andre Heinecke <aheinecke@intevation.de>
parents:
5080
diff
changeset
|
154 System.exit(1); |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
155 } |
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
156 } |
180
3908bfb2aa43
Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
177
diff
changeset
|
157 |
8970 | 158 // Parse a single river gew file specified in the flys.backend.importer.infogew.file property |
159 // (seems to be an alternative to the args way) | |
160 final String gew = Config.INSTANCE.getInfoGewFile(); | |
2804
68d1661a57cf
Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
1223
diff
changeset
|
161 if (gew != null && gew.length() > 0) { |
8970 | 162 log.info("Parsing info gew file: " + gew); |
163 final File gewFile = new File(gew); | |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
164 if (bwastrFile == null) { |
8970 | 165 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE); |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
166 } |
2804
68d1661a57cf
Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
1223
diff
changeset
|
167 try { |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
168 infoGewParser.parse(gewFile); |
2804
68d1661a57cf
Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
1223
diff
changeset
|
169 } |
9650 | 170 catch (final Exception ioe) { |
8970 | 171 log.error("error parsing gew: " + gew, ioe); |
5106
d16398d65a59
Importer: Print a decent error when failing to parse a gew and exit.
Andre Heinecke <aheinecke@intevation.de>
parents:
5080
diff
changeset
|
172 System.exit(1); |
2804
68d1661a57cf
Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
1223
diff
changeset
|
173 } |
68d1661a57cf
Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
1223
diff
changeset
|
174 } |
68d1661a57cf
Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents:
1223
diff
changeset
|
175 |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
176 // Look for official numbers. |
8970 | 177 final BundesWasserStrassenParser bwastrIdParser = new BundesWasserStrassenParser(); |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
178 |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
179 // Read bwastFile (river-dir + BWASTR_ID_CSV_FILE). |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
180 if (!Config.INSTANCE.skipBWASTR()) { |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
181 try{ |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
182 bwastrIdParser.parse(bwastrFile); |
8970 | 183 final HashMap<String,Long> map = bwastrIdParser.getMap(); |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
184 |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
185 // Now link rivers with official numbers. |
8970 | 186 for(final ImportRiver river: infoGewParser.getRivers()) { |
187 for(final Map.Entry<String, Long> entry: map.entrySet()) { | |
188 if (StringUtil.containsIgnoreCase(river.getName(), entry.getKey())) { | |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
189 river.setOfficialNumber(entry.getValue()); |
8970 | 190 log.debug(river.getName() + " is mapped to bwastr " + entry.getValue()); |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
191 } |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
192 } |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
193 } |
8970 | 194 } |
195 catch (final IOException ioe) { | |
196 log.warn("BWASTR-file could not be loaded: " + ioe.getMessage()); | |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
197 } |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
198 } |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
199 else { |
8970 | 200 log.debug("Skip reading BWASTR_ID.csv"); |
5027
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
201 } |
25c180fa6162
Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
5016
diff
changeset
|
202 |
8970 | 203 // Write all parsed objects to the database |
1223
268f8da412e3
Importer: Added a central configuration to allow skipping of parsing/storing individual sub systems.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
1211
diff
changeset
|
204 if (!Config.INSTANCE.dryRun()) { |
199
ed38839a6b08
Ported over some WST parsing stuff from desktop flys
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
190
diff
changeset
|
205 new Importer(infoGewParser.getRivers()).writeToDatabase(); |
ed38839a6b08
Ported over some WST parsing stuff from desktop flys
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
190
diff
changeset
|
206 } |
4708
45d05402b2a6
Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
4707
diff
changeset
|
207 else { |
45d05402b2a6
Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
4707
diff
changeset
|
208 log.info("Dry run, not writing to database."); |
45d05402b2a6
Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents:
4707
diff
changeset
|
209 } |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
210 } |
8970 | 211 |
212 /** | |
213 * Tries to load the Log4j configuration from the property 'log4j.configuration'. | |
214 */ | |
215 private static final void configureLogging() { | |
216 final String configPath = System.getProperty("log4j.configuration"); | |
217 try { | |
218 final File propFile = new File(configPath); | |
219 if (propFile.isFile() && propFile.canRead()) { | |
220 try { | |
221 PropertyConfigurator.configure(propFile.toURI().toURL()); | |
222 SLF4JBridgeHandler.install(); | |
223 } | |
224 catch (final MalformedURLException mue) { | |
225 mue.printStackTrace(System.err); | |
226 } | |
227 } | |
228 } | |
229 catch (final Exception e) { | |
230 e.printStackTrace(System.err); | |
231 } | |
232 } | |
233 | |
177
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
234 } |
31895d24387e
Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff
changeset
|
235 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : |