Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/Importer.java @ 8970:da5dc7446652

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SLF4J added and configureLogging in main
author mschaefer
date Tue, 03 Apr 2018 10:02:01 +0200
parents 5e38e2924c07
children a44200bf0547
rev   line source
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
1 /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
2 * Software engineering by Intevation GmbH
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
3 *
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
4 * This file is Free Software under the GNU AGPL (>=v3)
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
5 * and comes with ABSOLUTELY NO WARRANTY! Check out the
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
6 * documentation coming with Dive4Elements River for details.
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
7 */
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
8
5829
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
9 package org.dive4elements.river.importer;
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
10
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
11 import java.io.File;
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
12 import java.io.IOException;
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
13 import java.net.MalformedURLException;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
14 import java.sql.SQLException;
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
15 import java.util.HashMap;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
16 import java.util.List;
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
17 import java.util.Map;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
18
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
19 import org.apache.log4j.Logger;
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
20 import org.apache.log4j.PropertyConfigurator;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
21 import org.dive4elements.artifacts.common.utils.XMLUtils;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
22 import org.dive4elements.river.backend.utils.StringUtil;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
23 import org.dive4elements.river.importer.parsers.AnnotationClassifier;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
24 import org.dive4elements.river.importer.parsers.BundesWasserStrassenParser;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
25 import org.dive4elements.river.importer.parsers.InfoGewParser;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
26 import org.hibernate.HibernateException;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
27 import org.hibernate.Transaction;
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
28 import org.slf4j.bridge.SLF4JBridgeHandler;
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
29 import org.w3c.dom.Document;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
30
5016
255898799ed9 Minimal whitespace and doc cosmetics.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4708
diff changeset
31 /** Data Importer. Further processing happens per-river. */
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
32 public class Importer
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
33 {
8200
9d2e69f971f5 sed -i src/**/*.java 's/logger/log/g'
Sascha L. Teichmann <teichmann@intevation.de>
parents: 8187
diff changeset
34 /** Private log. */
8843
3ecf1f76b2b8 Mark constants final.
Tom Gottfried <tom@intevation.de>
parents: 8821
diff changeset
35 private static final Logger log = Logger.getLogger(Importer.class);
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
36
8843
3ecf1f76b2b8 Mark constants final.
Tom Gottfried <tom@intevation.de>
parents: 8821
diff changeset
37 private static final String BWASTR_ID_CSV_FILE = "BWASTR_ID.csv";
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
38
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
39 protected List<ImportRiver> rivers;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
40
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
41 public Importer() {
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
42 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
43
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
44 public Importer(final List<ImportRiver> rivers) {
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
45 this.rivers = rivers;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
46 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
47
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
48 public List<ImportRiver> getRivers() {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
49 return this.rivers;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
50 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
51
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
52 public void setRivers(final List<ImportRiver> rivers) {
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
53 this.rivers = rivers;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
54 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
55
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
56 /** Write rivers and their dependencies/dependants to db. */
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
57 public void writeRivers() {
182
1c0afb01bc93 Importer: Fixed error in HQL statement.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 180
diff changeset
58 log.debug("write rivers started");
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
59
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
60 for (final ImportRiver river: this.rivers) {
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
61 log.debug("writing river '" + river.getName() + "'");
190
d40da430d2fe Added stub code to write gauges
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 189
diff changeset
62 river.storeDependencies();
497
67fd63e4ef66 Importer: centralized caching
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 199
diff changeset
63 ImporterSession.getInstance().getDatabaseSession().flush();
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
64 }
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
65
182
1c0afb01bc93 Importer: Fixed error in HQL statement.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 180
diff changeset
66 log.debug("write rivers finished");
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
67 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
68
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
69 public void writeToDatabase() {
183
222f4db3430a Importer: Used thread local pattern to make sharing of session easier.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 182
diff changeset
70
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
71 Transaction tx = null;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
72
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
73 try {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
74 tx = ImporterSession.getInstance().getDatabaseSession().beginTransaction();
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
75
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
76 try {
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
77 writeRivers();
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
78 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
79 catch (final HibernateException he) {
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
80 Throwable t = he.getCause();
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
81 while (t instanceof SQLException) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
82 final SQLException sqle = (SQLException) t;
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
83 log.error("SQL exeception chain:", sqle);
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
84 t = sqle.getNextException();
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
85 }
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
86 throw he;
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
87 }
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
88
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
89 tx.commit();
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
90 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
91 catch (final RuntimeException re) {
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
92 if (tx != null) {
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
93 tx.rollback();
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
94 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
95 throw re;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
96 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
97 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
98
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
99 public static AnnotationClassifier getAnnotationClassifier() {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
100 final String annotationTypes = Config.INSTANCE.getAnnotationTypes();
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
101
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
102 if (annotationTypes == null) {
766
aa9e3da95c31 Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 765
diff changeset
103 log.info("no annotation types file configured.");
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
104 return null;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
105 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
106
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
107 final File file = new File(annotationTypes);
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
108
766
aa9e3da95c31 Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 765
diff changeset
109 log.info("use annotation types file '" + file + "'");
aa9e3da95c31 Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 765
diff changeset
110
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
111 if (!(file.isFile() && file.canRead())) {
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
112 log.warn("annotation type file '" + file + "' is not readable.");
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
113 return null;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
114 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
115
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
116 final Document rules = XMLUtils.parseDocument(file, false, null);
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
117
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
118 if (rules == null) {
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
119 log.warn("cannot parse annotation types file.");
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
120 return null;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
121 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
122
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
123 return new AnnotationClassifier(rules);
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
124 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
125
4707
2b0426b79a92 Documentation.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3659
diff changeset
126
2b0426b79a92 Documentation.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3659
diff changeset
127 /** Starting point for importing river data. */
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
128 public static void main(final String [] args) {
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
129
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
130 configureLogging();
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
131
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
132 log.info("START parsing rivers...");
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
133
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
134 final InfoGewParser infoGewParser = new InfoGewParser(getAnnotationClassifier());
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2804
diff changeset
135
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
136 File bwastrFile = null;
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
137
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
138 // Main parsing loop for all river gew file paths in args
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
139 // FIXME: Multiple rivers lead to reparsing the already parsed rivers again in InfoGewParser.parse...
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
140 for (final String gew : args) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
141 log.info("Parsing info gew file: " + gew);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
142 final File gewFile = new File(gew);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
143 if (bwastrFile == null) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
144 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
145 }
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
146 try {
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
147 infoGewParser.parse(gewFile);
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
148 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
149 catch (final IOException ioe) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
150 log.error("error parsing gew: " + gew, ioe);
5106
d16398d65a59 Importer: Print a decent error when failing to parse a gew and exit.
Andre Heinecke <aheinecke@intevation.de>
parents: 5080
diff changeset
151 System.exit(1);
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
152 }
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
153 }
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
154
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
155 // Parse a single river gew file specified in the flys.backend.importer.infogew.file property
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
156 // (seems to be an alternative to the args way)
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
157 final String gew = Config.INSTANCE.getInfoGewFile();
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
158 if (gew != null && gew.length() > 0) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
159 log.info("Parsing info gew file: " + gew);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
160 final File gewFile = new File(gew);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
161 if (bwastrFile == null) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
162 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
163 }
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
164 try {
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
165 infoGewParser.parse(gewFile);
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
166 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
167 catch (final IOException ioe) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
168 log.error("error parsing gew: " + gew, ioe);
5106
d16398d65a59 Importer: Print a decent error when failing to parse a gew and exit.
Andre Heinecke <aheinecke@intevation.de>
parents: 5080
diff changeset
169 System.exit(1);
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
170 }
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
171 }
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
172
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
173 // Look for official numbers.
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
174 final BundesWasserStrassenParser bwastrIdParser = new BundesWasserStrassenParser();
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
175
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
176 // Read bwastFile (river-dir + BWASTR_ID_CSV_FILE).
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
177 if (!Config.INSTANCE.skipBWASTR()) {
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
178 try{
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
179 bwastrIdParser.parse(bwastrFile);
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
180 final HashMap<String,Long> map = bwastrIdParser.getMap();
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
181
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
182 // Now link rivers with official numbers.
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
183 for(final ImportRiver river: infoGewParser.getRivers()) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
184 for(final Map.Entry<String, Long> entry: map.entrySet()) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
185 if (StringUtil.containsIgnoreCase(river.getName(), entry.getKey())) {
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
186 river.setOfficialNumber(entry.getValue());
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
187 log.debug(river.getName() + " is mapped to bwastr " + entry.getValue());
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
188 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
189 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
190 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
191 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
192 catch (final IOException ioe) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
193 log.warn("BWASTR-file could not be loaded: " + ioe.getMessage());
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
194 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
195 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
196 else {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
197 log.debug("Skip reading BWASTR_ID.csv");
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
198 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
199
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
200 // Write all parsed objects to the database
1223
268f8da412e3 Importer: Added a central configuration to allow skipping of parsing/storing individual sub systems.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1211
diff changeset
201 if (!Config.INSTANCE.dryRun()) {
199
ed38839a6b08 Ported over some WST parsing stuff from desktop flys
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 190
diff changeset
202 new Importer(infoGewParser.getRivers()).writeToDatabase();
ed38839a6b08 Ported over some WST parsing stuff from desktop flys
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 190
diff changeset
203 }
4708
45d05402b2a6 Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4707
diff changeset
204 else {
45d05402b2a6 Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4707
diff changeset
205 log.info("Dry run, not writing to database.");
45d05402b2a6 Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4707
diff changeset
206 }
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
207 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
208
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
209 /**
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
210 * Tries to load the Log4j configuration from the property 'log4j.configuration'.
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
211 */
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
212 private static final void configureLogging() {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
213 final String configPath = System.getProperty("log4j.configuration");
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
214 try {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
215 final File propFile = new File(configPath);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
216 if (propFile.isFile() && propFile.canRead()) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
217 try {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
218 PropertyConfigurator.configure(propFile.toURI().toURL());
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
219 SLF4JBridgeHandler.install();
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
220 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
221 catch (final MalformedURLException mue) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
222 mue.printStackTrace(System.err);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
223 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
224 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
225 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
226 catch (final Exception e) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
227 e.printStackTrace(System.err);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
228 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
229 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
230
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
231 }
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
232 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org