Mercurial > dive4elements > river

annotate backend/src/main/java/org/dive4elements/river/importer/Importer.java @ 9011:a44200bf0547

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
A few directory and file name changes
author mschaefer
date Wed, 18 Apr 2018 07:43:04 +0200
parents da5dc7446652
children a2a42a6bac6b
rev   line source
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
1 /* Copyright (C) 2011, 2012, 2013 by Bundesanstalt für Gewässerkunde
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
2 * Software engineering by Intevation GmbH
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
3 *
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
4 * This file is Free Software under the GNU AGPL (>=v3)
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
5 * and comes with ABSOLUTELY NO WARRANTY! Check out the
5992
4c3ccf2b0304 Removed trailing whitespace.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5844
diff changeset
6 * documentation coming with Dive4Elements River for details.
5844
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
7 */
4dd33b86dc61 Added header to river backend.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5838
diff changeset
8
5829
18619c1e7c2a Repaired internal references.
Sascha L. Teichmann <teichmann@intevation.de>
parents: 5828
diff changeset
9 package org.dive4elements.river.importer;
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
10
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
11 import java.io.File;
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
12 import java.io.IOException;
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
13 import java.net.MalformedURLException;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
14 import java.sql.SQLException;
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
15 import java.util.HashMap;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
16 import java.util.List;
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
17 import java.util.Map;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
18
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
19 import org.apache.log4j.Logger;
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
20 import org.apache.log4j.PropertyConfigurator;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
21 import org.dive4elements.artifacts.common.utils.XMLUtils;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
22 import org.dive4elements.river.backend.utils.StringUtil;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
23 import org.dive4elements.river.importer.parsers.AnnotationClassifier;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
24 import org.dive4elements.river.importer.parsers.BundesWasserStrassenParser;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
25 import org.dive4elements.river.importer.parsers.InfoGewParser;
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
26 import org.hibernate.HibernateException;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
27 import org.hibernate.Transaction;
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
28 import org.slf4j.bridge.SLF4JBridgeHandler;
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
29 import org.w3c.dom.Document;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
30
5016
255898799ed9 Minimal whitespace and doc cosmetics.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4708
diff changeset
31 /** Data Importer. Further processing happens per-river. */
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
32 public class Importer
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
33 {
9011
a44200bf0547 A few directory and file name changes
mschaefer
parents: 8970
diff changeset
34 static final String VERSION = "3.3.0";
a44200bf0547 A few directory and file name changes
mschaefer
parents: 8970
diff changeset
35
8200
9d2e69f971f5 sed -i src/**/*.java 's/logger/log/g'
Sascha L. Teichmann <teichmann@intevation.de>
parents: 8187
diff changeset
36 /** Private log. */
8843
3ecf1f76b2b8 Mark constants final.
Tom Gottfried <tom@intevation.de>
parents: 8821
diff changeset
37 private static final Logger log = Logger.getLogger(Importer.class);
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
38
8843
3ecf1f76b2b8 Mark constants final.
Tom Gottfried <tom@intevation.de>
parents: 8821
diff changeset
39 private static final String BWASTR_ID_CSV_FILE = "BWASTR_ID.csv";
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
40
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
41 protected List<ImportRiver> rivers;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
42
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
43 public Importer() {
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
44 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
45
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
46 public Importer(final List<ImportRiver> rivers) {
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
47 this.rivers = rivers;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
48 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
49
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
50 public List<ImportRiver> getRivers() {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
51 return this.rivers;
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
52 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
53
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
54 public void setRivers(final List<ImportRiver> rivers) {
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
55 this.rivers = rivers;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
56 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
57
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
58 /** Write rivers and their dependencies/dependants to db. */
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
59 public void writeRivers() {
182
1c0afb01bc93 Importer: Fixed error in HQL statement.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 180
diff changeset
60 log.debug("write rivers started");
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
61
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
62 for (final ImportRiver river: this.rivers) {
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
63 log.debug("writing river '" + river.getName() + "'");
190
d40da430d2fe Added stub code to write gauges
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 189
diff changeset
64 river.storeDependencies();
497
67fd63e4ef66 Importer: centralized caching
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 199
diff changeset
65 ImporterSession.getInstance().getDatabaseSession().flush();
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
66 }
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
67
182
1c0afb01bc93 Importer: Fixed error in HQL statement.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 180
diff changeset
68 log.debug("write rivers finished");
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
69 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
70
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
71 public void writeToDatabase() {
183
222f4db3430a Importer: Used thread local pattern to make sharing of session easier.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 182
diff changeset
72
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
73 Transaction tx = null;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
74
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
75 try {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
76 tx = ImporterSession.getInstance().getDatabaseSession().beginTransaction();
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
77
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
78 try {
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
79 writeRivers();
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
80 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
81 catch (final HibernateException he) {
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
82 Throwable t = he.getCause();
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
83 while (t instanceof SQLException) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
84 final SQLException sqle = (SQLException) t;
188
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
85 log.error("SQL exeception chain:", sqle);
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
86 t = sqle.getNextException();
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
87 }
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
88 throw he;
003ac16812dd Store annotations in backend.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 183
diff changeset
89 }
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
90
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
91 tx.commit();
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
92 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
93 catch (final RuntimeException re) {
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
94 if (tx != null) {
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
95 tx.rollback();
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
96 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
97 throw re;
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
98 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
99 }
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
100
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
101 public static AnnotationClassifier getAnnotationClassifier() {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
102 final String annotationTypes = Config.INSTANCE.getAnnotationTypes();
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
103
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
104 if (annotationTypes == null) {
766
aa9e3da95c31 Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 765
diff changeset
105 log.info("no annotation types file configured.");
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
106 return null;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
107 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
108
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
109 final File file = new File(annotationTypes);
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
110
766
aa9e3da95c31 Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 765
diff changeset
111 log.info("use annotation types file '" + file + "'");
aa9e3da95c31 Importer: Fix import of annotation type classification.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 765
diff changeset
112
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
113 if (!(file.isFile() && file.canRead())) {
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
114 log.warn("annotation type file '" + file + "' is not readable.");
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
115 return null;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
116 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
117
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
118 final Document rules = XMLUtils.parseDocument(file, false, null);
765
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
119
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
120 if (rules == null) {
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
121 log.warn("cannot parse annotation types file.");
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
122 return null;
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
123 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
124
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
125 return new AnnotationClassifier(rules);
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
126 }
763c4137d6e1 Added classification of annotation types. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 497
diff changeset
127
4707
2b0426b79a92 Documentation.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3659
diff changeset
128
2b0426b79a92 Documentation.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 3659
diff changeset
129 /** Starting point for importing river data. */
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
130 public static void main(final String [] args) {
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
131
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
132 configureLogging();
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
133
9011
a44200bf0547 A few directory and file name changes
mschaefer
parents: 8970
diff changeset
134 log.info("IMPORTER version " + VERSION);
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
135 log.info("START parsing rivers...");
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
136
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
137 final InfoGewParser infoGewParser = new InfoGewParser(getAnnotationClassifier());
2806
33f40b23edd8 Initial checkin for parsing MINFO bed heights.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 2804
diff changeset
138
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
139 File bwastrFile = null;
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
140
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
141 // Main parsing loop for all river gew file paths in args
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
142 // FIXME: Multiple rivers lead to reparsing the already parsed rivers again in InfoGewParser.parse...
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
143 for (final String gew : args) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
144 log.info("Parsing info gew file: " + gew);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
145 final File gewFile = new File(gew);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
146 if (bwastrFile == null) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
147 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
148 }
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
149 try {
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
150 infoGewParser.parse(gewFile);
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
151 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
152 catch (final IOException ioe) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
153 log.error("error parsing gew: " + gew, ioe);
5106
d16398d65a59 Importer: Print a decent error when failing to parse a gew and exit.
Andre Heinecke <aheinecke@intevation.de>
parents: 5080
diff changeset
154 System.exit(1);
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
155 }
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
156 }
180
3908bfb2aa43 Store imported rivers into database. Needs testing!
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 177
diff changeset
157
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
158 // Parse a single river gew file specified in the flys.backend.importer.infogew.file property
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
159 // (seems to be an alternative to the args way)
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
160 final String gew = Config.INSTANCE.getInfoGewFile();
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
161 if (gew != null && gew.length() > 0) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
162 log.info("Parsing info gew file: " + gew);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
163 final File gewFile = new File(gew);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
164 if (bwastrFile == null) {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
165 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
166 }
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
167 try {
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
168 infoGewParser.parse(gewFile);
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
169 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
170 catch (final IOException ioe) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
171 log.error("error parsing gew: " + gew, ioe);
5106
d16398d65a59 Importer: Print a decent error when failing to parse a gew and exit.
Andre Heinecke <aheinecke@intevation.de>
parents: 5080
diff changeset
172 System.exit(1);
2804
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
173 }
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
174 }
68d1661a57cf Added a new config option to specify a INFO.gew file by system property.
Ingo Weinzierl <ingo.weinzierl@intevation.de>
parents: 1223
diff changeset
175
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
176 // Look for official numbers.
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
177 final BundesWasserStrassenParser bwastrIdParser = new BundesWasserStrassenParser();
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
178
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
179 // Read bwastFile (river-dir + BWASTR_ID_CSV_FILE).
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
180 if (!Config.INSTANCE.skipBWASTR()) {
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
181 try{
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
182 bwastrIdParser.parse(bwastrFile);
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
183 final HashMap<String,Long> map = bwastrIdParser.getMap();
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
184
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
185 // Now link rivers with official numbers.
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
186 for(final ImportRiver river: infoGewParser.getRivers()) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
187 for(final Map.Entry<String, Long> entry: map.entrySet()) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
188 if (StringUtil.containsIgnoreCase(river.getName(), entry.getKey())) {
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
189 river.setOfficialNumber(entry.getValue());
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
190 log.debug(river.getName() + " is mapped to bwastr " + entry.getValue());
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
191 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
192 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
193 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
194 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
195 catch (final IOException ioe) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
196 log.warn("BWASTR-file could not be loaded: " + ioe.getMessage());
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
197 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
198 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
199 else {
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
200 log.debug("Skip reading BWASTR_ID.csv");
5027
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
201 }
25c180fa6162 Use BWASTRParser to set official numbers on rivers.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 5016
diff changeset
202
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
203 // Write all parsed objects to the database
1223
268f8da412e3 Importer: Added a central configuration to allow skipping of parsing/storing individual sub systems.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 1211
diff changeset
204 if (!Config.INSTANCE.dryRun()) {
199
ed38839a6b08 Ported over some WST parsing stuff from desktop flys
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 190
diff changeset
205 new Importer(infoGewParser.getRivers()).writeToDatabase();
ed38839a6b08 Ported over some WST parsing stuff from desktop flys
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents: 190
diff changeset
206 }
4708
45d05402b2a6 Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4707
diff changeset
207 else {
45d05402b2a6 Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4707
diff changeset
208 log.info("Dry run, not writing to database.");
45d05402b2a6 Importer: Emit log message when a dry run was run.
Felix Wolfsteller <felix.wolfsteller@intevation.de>
parents: 4707
diff changeset
209 }
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
210 }
8970
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
211
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
212 /**
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
213 * Tries to load the Log4j configuration from the property 'log4j.configuration'.
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
214 */
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
215 private static final void configureLogging() {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
216 final String configPath = System.getProperty("log4j.configuration");
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
217 try {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
218 final File propFile = new File(configPath);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
219 if (propFile.isFile() && propFile.canRead()) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
220 try {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
221 PropertyConfigurator.configure(propFile.toURI().toURL());
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
222 SLF4JBridgeHandler.install();
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
223 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
224 catch (final MalformedURLException mue) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
225 mue.printStackTrace(System.err);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
226 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
227 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
228 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
229 catch (final Exception e) {
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
230 e.printStackTrace(System.err);
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
231 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
232 }
da5dc7446652 SLF4J added and configureLogging in main
mschaefer
parents: 8856
diff changeset
233
177
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
234 }
31895d24387e Importer: Added info gew parser.
Sascha L. Teichmann <sascha.teichmann@intevation.de>
parents:
diff changeset
235 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org