Mercurial > dive4elements > river

comparison backend/src/main/java/org/dive4elements/river/importer/Importer.java @ 8970:da5dc7446652

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SLF4J added and configureLogging in main
author mschaefer
date Tue, 03 Apr 2018 10:02:01 +0200
parents 5e38e2924c07
children a44200bf0547
comparison
equal deleted inserted replaced
8969:fe81eb39080c 8970:da5dc7446652
6 * documentation coming with Dive4Elements River for details. 6 * documentation coming with Dive4Elements River for details.
7 */ 7 */
8 8
9 package org.dive4elements.river.importer; 9 package org.dive4elements.river.importer;
10 10
11 import java.io.File;
12 import java.io.IOException;
13 import java.net.MalformedURLException;
14 import java.sql.SQLException;
15 import java.util.HashMap;
16 import java.util.List;
17 import java.util.Map;
18
19 import org.apache.log4j.Logger;
20 import org.apache.log4j.PropertyConfigurator;
11 import org.dive4elements.artifacts.common.utils.XMLUtils; 21 import org.dive4elements.artifacts.common.utils.XMLUtils;
12 22 import org.dive4elements.river.backend.utils.StringUtil;
13 import org.dive4elements.river.importer.parsers.AnnotationClassifier; 23 import org.dive4elements.river.importer.parsers.AnnotationClassifier;
14 import org.dive4elements.river.importer.parsers.BundesWasserStrassenParser; 24 import org.dive4elements.river.importer.parsers.BundesWasserStrassenParser;
15 import org.dive4elements.river.importer.parsers.InfoGewParser; 25 import org.dive4elements.river.importer.parsers.InfoGewParser;
16 26 import org.hibernate.HibernateException;
17 import java.io.File;
18 import java.io.IOException;
19
20 import java.util.HashMap;
21 import java.util.List;
22 import java.util.Map;
23
24 import java.sql.SQLException;
25
26 import org.apache.log4j.Logger;
27
28 import org.hibernate.Transaction; 27 import org.hibernate.Transaction;
29 import org.hibernate.HibernateException; 28 import org.slf4j.bridge.SLF4JBridgeHandler;
30
31 import org.w3c.dom.Document; 29 import org.w3c.dom.Document;
32
33 import org.dive4elements.river.backend.utils.StringUtil;
34 30
35 /** Data Importer. Further processing happens per-river. */ 31 /** Data Importer. Further processing happens per-river. */
36 public class Importer 32 public class Importer
37 { 33 {
38 /** Private log. */ 34 /** Private log. */
43 protected List<ImportRiver> rivers; 39 protected List<ImportRiver> rivers;
44 40
45 public Importer() { 41 public Importer() {
46 } 42 }
47 43
48 public Importer(List<ImportRiver> rivers) { 44 public Importer(final List<ImportRiver> rivers) {
49 this.rivers = rivers; 45 this.rivers = rivers;
50 } 46 }
51 47
52 public List<ImportRiver> getRivers() { 48 public List<ImportRiver> getRivers() {
53 return rivers; 49 return this.rivers;
54 } 50 }
55 51
56 public void setRivers(List<ImportRiver> rivers) { 52 public void setRivers(final List<ImportRiver> rivers) {
57 this.rivers = rivers; 53 this.rivers = rivers;
58 } 54 }
59 55
60 /** Write rivers and their dependencies/dependants to db. */ 56 /** Write rivers and their dependencies/dependants to db. */
61 public void writeRivers() { 57 public void writeRivers() {
62 log.debug("write rivers started"); 58 log.debug("write rivers started");
63 59
64 for (ImportRiver river: rivers) { 60 for (final ImportRiver river: this.rivers) {
65 log.debug("writing river '" + river.getName() + "'"); 61 log.debug("writing river '" + river.getName() + "'");
66 river.storeDependencies(); 62 river.storeDependencies();
67 ImporterSession.getInstance().getDatabaseSession().flush(); 63 ImporterSession.getInstance().getDatabaseSession().flush();
68 } 64 }
69 65
73 public void writeToDatabase() { 69 public void writeToDatabase() {
74 70
75 Transaction tx = null; 71 Transaction tx = null;
76 72
77 try { 73 try {
78 tx = ImporterSession.getInstance() 74 tx = ImporterSession.getInstance().getDatabaseSession().beginTransaction();
79 .getDatabaseSession().beginTransaction();
80 75
81 try { 76 try {
82 writeRivers(); 77 writeRivers();
83 } 78 }
84 catch (HibernateException he) { 79 catch (final HibernateException he) {
85 Throwable t = he.getCause(); 80 Throwable t = he.getCause();
86 while (t instanceof SQLException) { 81 while (t instanceof SQLException) {
87 SQLException sqle = (SQLException) t; 82 final SQLException sqle = (SQLException) t;
88 log.error("SQL exeception chain:", sqle); 83 log.error("SQL exeception chain:", sqle);
89 t = sqle.getNextException(); 84 t = sqle.getNextException();
90 } 85 }
91 throw he; 86 throw he;
92 } 87 }
93 88
94 tx.commit(); 89 tx.commit();
95 } 90 }
96 catch (RuntimeException re) { 91 catch (final RuntimeException re) {
97 if (tx != null) { 92 if (tx != null) {
98 tx.rollback(); 93 tx.rollback();
99 } 94 }
100 throw re; 95 throw re;
101 } 96 }
102 } 97 }
103 98
104 public static AnnotationClassifier getAnnotationClassifier() { 99 public static AnnotationClassifier getAnnotationClassifier() {
105 String annotationTypes = Config.INSTANCE.getAnnotationTypes(); 100 final String annotationTypes = Config.INSTANCE.getAnnotationTypes();
106 101
107 if (annotationTypes == null) { 102 if (annotationTypes == null) {
108 log.info("no annotation types file configured."); 103 log.info("no annotation types file configured.");
109 return null; 104 return null;
110 } 105 }
111 106
112 File file = new File(annotationTypes); 107 final File file = new File(annotationTypes);
113 108
114 log.info("use annotation types file '" + file + "'"); 109 log.info("use annotation types file '" + file + "'");
115 110
116 if (!(file.isFile() && file.canRead())) { 111 if (!(file.isFile() && file.canRead())) {
117 log.warn("annotation type file '" + file + "' is not readable."); 112 log.warn("annotation type file '" + file + "' is not readable.");
118 return null; 113 return null;
119 } 114 }
120 115
121 Document rules = XMLUtils.parseDocument(file, false, null); 116 final Document rules = XMLUtils.parseDocument(file, false, null);
122 117
123 if (rules == null) { 118 if (rules == null) {
124 log.warn("cannot parse annotation types file."); 119 log.warn("cannot parse annotation types file.");
125 return null; 120 return null;
126 } 121 }
128 return new AnnotationClassifier(rules); 123 return new AnnotationClassifier(rules);
129 } 124 }
130 125
131 126
132 /** Starting point for importing river data. */ 127 /** Starting point for importing river data. */
133 public static void main(String [] args) { 128 public static void main(final String [] args) {
134 129
135 InfoGewParser infoGewParser = new InfoGewParser( 130 configureLogging();
136 getAnnotationClassifier()); 131
137 132 log.info("START parsing rivers...");
138 log.info("Start parsing rivers..."); 133
134 final InfoGewParser infoGewParser = new InfoGewParser(getAnnotationClassifier());
139 135
140 File bwastrFile = null; 136 File bwastrFile = null;
141 137
142 for (String gew: args) { 138 // Main parsing loop for all river gew file paths in args
143 log.info("parsing info gew file: " + gew); 139 // FIXME: Multiple rivers lead to reparsing the already parsed rivers again in InfoGewParser.parse...
144 File gewFile = new File(gew); 140 for (final String gew : args) {
141 log.info("Parsing info gew file: " + gew);
142 final File gewFile = new File(gew);
145 if (bwastrFile == null) { 143 if (bwastrFile == null) {
146 bwastrFile = new File( 144 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
147 gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
148 } 145 }
149 try { 146 try {
150 infoGewParser.parse(gewFile); 147 infoGewParser.parse(gewFile);
151 } 148 }
152 catch (IOException ioe) { 149 catch (final IOException ioe) {
153 log.error("error while parsing gew: " + gew, ioe); 150 log.error("error parsing gew: " + gew, ioe);
154 System.exit(1); 151 System.exit(1);
155 } 152 }
156 } 153 }
157 154
158 String gew = Config.INSTANCE.getInfoGewFile(); 155 // Parse a single river gew file specified in the flys.backend.importer.infogew.file property
156 // (seems to be an alternative to the args way)
157 final String gew = Config.INSTANCE.getInfoGewFile();
159 if (gew != null && gew.length() > 0) { 158 if (gew != null && gew.length() > 0) {
160 log.info("parsing info gew file: " + gew); 159 log.info("Parsing info gew file: " + gew);
161 File gewFile = new File(gew); 160 final File gewFile = new File(gew);
162 if (bwastrFile == null) { 161 if (bwastrFile == null) {
163 bwastrFile = new File( 162 bwastrFile = new File(gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
164 gewFile.getParentFile(), BWASTR_ID_CSV_FILE);
165 } 163 }
166 try { 164 try {
167 infoGewParser.parse(gewFile); 165 infoGewParser.parse(gewFile);
168 } 166 }
169 catch (IOException ioe) { 167 catch (final IOException ioe) {
170 log.error("error while parsing gew: " + gew, ioe); 168 log.error("error parsing gew: " + gew, ioe);
171 System.exit(1); 169 System.exit(1);
172 } 170 }
173 } 171 }
174 172
175 // Look for official numbers. 173 // Look for official numbers.
176 BundesWasserStrassenParser bwastrIdParser = 174 final BundesWasserStrassenParser bwastrIdParser = new BundesWasserStrassenParser();
177 new BundesWasserStrassenParser();
178 175
179 // Read bwastFile (river-dir + BWASTR_ID_CSV_FILE). 176 // Read bwastFile (river-dir + BWASTR_ID_CSV_FILE).
180 if (!Config.INSTANCE.skipBWASTR()) { 177 if (!Config.INSTANCE.skipBWASTR()) {
181 try{ 178 try{
182 bwastrIdParser.parse(bwastrFile); 179 bwastrIdParser.parse(bwastrFile);
183 HashMap<String,Long> map = bwastrIdParser.getMap(); 180 final HashMap<String,Long> map = bwastrIdParser.getMap();
184 181
185 // Now link rivers with official numbers. 182 // Now link rivers with official numbers.
186 for(ImportRiver river: infoGewParser.getRivers()) { 183 for(final ImportRiver river: infoGewParser.getRivers()) {
187 for(Map.Entry<String, Long> entry: map.entrySet()) { 184 for(final Map.Entry<String, Long> entry: map.entrySet()) {
188 if (StringUtil.containsIgnoreCase( 185 if (StringUtil.containsIgnoreCase(river.getName(), entry.getKey())) {
189 river.getName(), entry.getKey())) {
190 river.setOfficialNumber(entry.getValue()); 186 river.setOfficialNumber(entry.getValue());
191 log.debug(river.getName() 187 log.debug(river.getName() + " is mapped to bwastr " + entry.getValue());
192 + " is mapped to bwastr " + entry.getValue());
193 } 188 }
194 } 189 }
195 } 190 }
196 } catch (IOException ioe) { 191 }
197 log.warn("BWASTR-file could not be loaded."); 192 catch (final IOException ioe) {
193 log.warn("BWASTR-file could not be loaded: " + ioe.getMessage());
198 } 194 }
199 } 195 }
200 else { 196 else {
201 log.debug("skip reading BWASTR_ID.csv"); 197 log.debug("Skip reading BWASTR_ID.csv");
202 } 198 }
203 199
200 // Write all parsed objects to the database
204 if (!Config.INSTANCE.dryRun()) { 201 if (!Config.INSTANCE.dryRun()) {
205 new Importer(infoGewParser.getRivers()).writeToDatabase(); 202 new Importer(infoGewParser.getRivers()).writeToDatabase();
206 } 203 }
207 else { 204 else {
208 log.info("Dry run, not writing to database."); 205 log.info("Dry run, not writing to database.");
209 } 206 }
210 } 207 }
208
209 /**
210 * Tries to load the Log4j configuration from the property 'log4j.configuration'.
211 */
212 private static final void configureLogging() {
213 final String configPath = System.getProperty("log4j.configuration");
214 try {
215 final File propFile = new File(configPath);
216 if (propFile.isFile() && propFile.canRead()) {
217 try {
218 PropertyConfigurator.configure(propFile.toURI().toURL());
219 SLF4JBridgeHandler.install();
220 }
221 catch (final MalformedURLException mue) {
222 mue.printStackTrace(System.err);
223 }
224 }
225 }
226 catch (final Exception e) {
227 e.printStackTrace(System.err);
228 }
229 }
230
211 } 231 }
212 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 : 232 // vim:set ts=4 sw=4 si et sta sts=4 fenc=utf8 :
http://dive4elements.wald.intevation.org